commit

.gitea/workflows/build-an-push.yaml

@@ -1,35 +0,0 @@
-name: 'build-and-push'
-
-on:
-  push:
-    branches:
-      - master
-jobs:
-  push:
-    runs-on: ubuntu-latest
-    steps:
-      - name: 'checkout'
-        uses: actions/checkout@v4
-      - name: Set up Docker Context for Buildx
-        run: docker context create builders
-      - name: 'create.env'
-        run: touch .env
-      - name: 'setup buildx'
-        uses: docker/setup-buildx-action@v3
-        with:
-          config-inline: |
-            [registry."192.168.0.151:32000"]
-            http = true
-            insecure = true
-      - name: 'setup Go Version'
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.22'
-      - run: make build
-      - name: 'Build and push Docker image'
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile
-          push: true
-          tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest"

.gitea/workflows/demo.yaml

@@ -1,19 +0,0 @@
-name: Gitea Actions Demo
-run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
-on: [push]
-
-jobs:
-  Explore-Gitea-Actions:
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo "🎉 The job was automatically and automagically  triggered by a ${{ gitea.event_name }} event."
-      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
-      - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
-      - name: Check out repository code
-        uses: actions/checkout@v4
-      - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
-      - run: echo "🖥️ The workflow is now ready to test your code on the runner."
-      - name: List files in the repository
-        run: |
-          ls ${{ gitea.workspace }}
-      - run: echo "🍏 This job's status is ${{ job.status }}."

.gitea/workflows/test.yaml

@@ -1,27 +0,0 @@
-name: 'Go test code'
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: 'Checkout gitea action'
-        uses: actions/checkout@v4
-        
-      - name: 'Setup Go'
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: 'go.mod'
-          
-      - name: 'Go version'
-        run: go version
-          
-      - name: 'Run Test'
-        run: |
-          export GOPATH="$HOME/go/"
-          export GOBIN=$PATH:GOPATH/bin
-          go test -v ./... 

.github/workflows/build_push_and_deploy.yaml

@@ -0,0 +1,113 @@
+name: ${{ github.actor }} executed Build Push Prod
+on:
+  push:
+    tags:
+      - 'v[0-9]+\.[0-9]+\.[0-9]+'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      TOKEN: ${{ secrets.TOKEN }}
+      USERNAME: ${{ vars.USERNAME }}
+      TAG_VERSION: ${{ github.ref_name }}
+    steps:
+      - name: echo repo dir
+        id: repo-url
+        run: |
+             echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Quemu
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set Up BuildX
+        uses: docker/setup-buildx-action@v3
+              
+      - name: Login to Docker
+        uses: docker/login-action@v3
+        with:
+           registry: ${{ github.server_url }}
+           username: ${{ vars.USERNAME }}
+           password: ${{ secrets.PWD }}
+
+      - name: calculate short sha
+        id: calculate-sha
+        run: |
+          echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
+          build-args: |
+            version=${{ github.ref_name }}
+            SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }}
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    env:
+      TOKEN: ${{ secrets.TOKEN }}
+      TAG: ${{ github.ref_name }}
+      PASSWORD: $PWD
+      USERNAME: ${{ vars.USERNAME }}
+      TAG_VERSION: ${{ github.ref_name }}
+      NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}"
+      ENV: "production"
+      APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}"
+      DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}"
+      DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}"
+      APA: "${{ vars.APA }}"
+      ACAP: "${{ vars.ACAP }}"
+      ANAP: "${{ vars.ANAP }}"
+      BCCRB: "${{ vars.BCCRB }}"
+      BCD: "${{ vars.BCD }}"
+      BDR: "${{ vars.BDR }}"
+      BHD: "${{ vars.BHD }}"
+      BNC: "${{ vars.BNC }}"
+      BNSC: "${{ vars.BNSC }}"
+      BPD: "${{ vars.BPD }}"
+      GENERAL: "${{ vars.GENERAL }}"
+      SCOTIA: "${{ vars.SCOTIA }}"
+      VIMECA: "${{ vars.VIMENCA }}"
+    steps:
+      - name: echo repo dir
+        id: repo-url
+        run: |
+             echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
+             echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: install envsubst
+        run: |
+             apt update
+             apt install -y gettext-base
+             apt clean
+
+      - name:  create deployment files
+        env:
+          IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
+          REPONAME: "${{ steps.repo-url.outputs.repoName }}"
+        run: |
+             envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml
+      - name: Download kubectl
+        run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+
+      - name: install kubectl
+        run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+      - name: insert kube config
+        env:
+          REPONAME: "${{ steps.repo-url.outputs.repoName }}"
+        run: |
+             mkdir ~/.kube
+             export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}"
+             echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
+      - name: apply config
+        run: kubectl apply -f k8s/cronjobs.ymlf

.github/workflows/test_project.yml

@@ -0,0 +1,68 @@
+name: "dev test"
+on:
+  push:
+    branches:
+      - master
+      - dev
+      - 'feature/**'
+    paths:
+      - '**.go'
+      - '**.yml'
+      - '**.yaml'      
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      TOKEN: ${{ secrets.TOKEN }}
+      USERNAME: ${{ vars.USERNAME }}
+      TAG_VERSION: ${{ github.ref_name }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+      - name: run tidy
+        run: |
+             go mod tidy
+      - name: test with go
+        run: go test -v ./...
+
+  vulnCheck:
+    runs-on: ubuntu-latest
+    needs: test
+    env:
+      TOKEN: ${{ secrets.TOKEN }}
+      USERNAME: ${{ vars.USERNAME }}
+      TAG_VERSION: ${{ github.ref_name }}
+    steps:
+      - name: checkout Code
+        uses: actions/checkout@v4
+        
+      - name: go vuln check
+        uses: golang/govulncheck-action@v1
+        with:
+          go-version-file: 'go.mod'
+          go-package: ./...
+
+  Ci-Lint:
+        runs-on: ubuntu-latest
+        needs: vulnCheck
+        env:
+          TOKEN: ${{ secrets.TOKEN }}
+          USERNAME: ${{ vars.USERNAME }}
+          TAG_VERSION: ${{ github.ref_name }}
+        steps:
+          - name: Checkout code
+            uses: actions/checkout@v4
+
+          - name: Setup Go
+            uses: actions/setup-go@v5
+            with:
+                go-version-file: 'go.mod'
+          - name: CI lint Check 
+            uses: golangci/golangci-lint-action@v6
+            with:
+              only-new-issues: true

.gitignore

@@ -1,4 +1,5 @@
 .env
 k8s/deployment.yml
 bin/*
-dolardb/*
+dolardb/*
+k8s/cronjobs.yml

Dockerfile

@@ -8,5 +8,4 @@ RUN ls
 COPY ./bin/${BINAME} /usr/bin/crawler
 RUN mkdir /app
 WORKDIR /app
-
 ENTRYPOINT crawler

Makefile

@@ -15,7 +15,7 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION)
 CONTAINERS=docker
 COMPOSE=$(CONTAINERS)-compose
 # Configure local registry
-REGADDR=192.168.0.151:32000
+REGADDR=git.maximotejeda.com
 #K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
 .phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local
 

cmd/crawler/main.go

@@ -15,8 +15,7 @@ func main() {
 	log := slog.Default()
 	var opts []grpc.DialOption
 	opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
-	conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...)
+	conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...)
-
 	if err != nil {
 		log.Error("creating gerpc conn", "error", err)
 		panic(err)

config/config.go

@@ -18,8 +18,8 @@ func GetBCDURL() string {
 	return getEnvValue("BCD")
 }
 
-func GetBDRURL() string {
+func GetBRDURL() string {
-	return getEnvValue("BDR")
+	return getEnvValue("BRD")
 }
 
 func GetBHDURL() string {

docker-compose.yaml

@@ -10,5 +10,5 @@ services:
       watch:
         - action: rebuild
           path: ./
-      
+
   

internal/adapters/crawler/bdr.go

@@ -13,15 +13,15 @@ import (
 	"github.com/playwright-community/playwright-go"
 )
 
-type bdr struct{}
+type brd struct{}
 
-func NewBDR() ports.APIPorts {
+func NewBRD() ports.APIPorts {
-	return &bdr{}
+	return &brd{}
 }
-func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
+func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
 	tout := 120000.00
-	log = log.With("scrapper", "bdr")
+	log = log.With("scrapper", "brd")
-	if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{
+	if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{
 		Timeout:   &tout,
 		WaitUntil: playwright.WaitUntilStateLoad,
 	}); err != nil {

internal/adapters/crawler/bhd.go

@@ -86,10 +86,6 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger
 				place := strings.ReplaceAll(n[0], "US$RD$", "")
 				place = strings.ToLower(place)
 				price := helpers.Normalize(val)
-				if err != nil {
-					log.Error("parsing value", "where", place, "err", err)
-					return nil, err
-				}
 				switch place {
 				case "compramos":
 					inst.Compra = price

internal/adapters/crawler/crawler.go

@@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
 	case "bcd":
 		parser = NewBCD()
 	case "brd":
-		parser = NewBDR()
+		parser = NewBRD()
 	case "bhd":
 		parser = NewBHD()
 	case "bnc":

k8s/cronjobs.yml.template

@@ -0,0 +1,255 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-inf-$REPONAME
+spec:
+  schedule: "*/20 8-19 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-inf-$REPONAME
+            image: $IMAGE
+            env:
+              - name: WHO
+                value: inf
+              - name: DOLLAR_SERVICE_URL
+                value: $DOLLAR_SERVICE_URL
+              - name: GENERAL
+                value: $GENERAL
+              - name: NATSURI
+                value: $NATS_SERVICE_URL
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-bcd-$REPONAME
+spec:
+  schedule: "3,33 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-bcd-$REPONAME
+            image: $IMAGE
+            env:
+             - name: BCD
+               value: $BCD
+             - name: NATSURI
+               value: $NATS_SERVICE_URL
+             - name: WHO
+               value: bcd
+             - name: DOLLAR_SERVICE_URL
+               value: $DOLLAR_SERVICE_URL
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-bpd-$REPONAME
+spec:
+  schedule: "5,35 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-bpd-$REPONAME
+            image: $IMAGE
+            env:
+             - name: BPD
+               value: $BPD
+             - name: NATSURI
+               value: $NATS_SERVICE_URL
+             - name: WHO
+               value: bpd
+             - name: DOLLAR_SERVICE_URL
+               value: $DOLLAR_SERVICE_URL
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-bhd-$REPONAME
+spec:
+  schedule: "8,38 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure         
+          containers:
+          - name: crawler-bhd-$REPONAME
+            image: $IMAGE
+            env:
+             - name: BHD
+               value: $BHD
+             - name: NATSURI
+               value: $NATS_SERVICE_URL
+             - name: WHO
+               value: bhd
+             - name: DOLLAR_SERVICE_URL
+               value: $DOLLAR_SERVICE_URL
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-brd-$REPONAME
+spec:
+  schedule: "22,52 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-brd-$REPONAME
+            image: $IMAGE
+            env:
+             - name: BDR
+               value: $BDR
+             - name: NATSURI
+               value: $NATS_SERVICE_URL
+             - name: WHO
+               value: brd
+             - name: DOLLAR_SERVICE_URL
+               value: $DOLLAR_SERVICE_URL
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-apap-$REPONAME
+spec:
+  schedule: "25,55 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1 
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-apap-$REPONAME
+            image: $IMAGE
+            env:
+              - name: APA
+                value: $APA
+              - name: NATSURI
+                value: $NATS_SERVICE_URL
+              - name: WHO
+                value: apap
+              - name: DOLLAR_SERVICE_URL
+                value: $DOLLAR_SERVICE_URL
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-bnc-$REPONAME
+spec:
+  schedule: "11,41 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1 
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-bnc-$REPONAME
+            image: $IMAGE
+            env:
+              - name: BNC
+                value: $BNC
+              - name: NATSURI
+                value: $NATS_SERVICE_URL
+              - name: WHO
+                value: bnc
+              - name: DOLLAR_SERVICE_URL
+                value: $DOLLAR_SERVICE_URL
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-vimenca-$REPONAME
+spec:
+  schedule: "14,44 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1 
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-vimenca-$REPONAME
+            image: $IMAGE
+            env:
+              - name: VIMENCA
+                value: $VIMENCA
+              - name: NATSURI
+                value: $NATS_SERVICE_URL
+              - name: WHO
+                value: vimenca
+              - name: DOLLAR_SERVICE_URL
+                value: $DOLLAR_SERVICE_URL
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: job-scotia-$REPONAME
+spec:
+  schedule: "17,47 8-22 * * 1-6"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      parallelism: 1
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+          - name: crawler-scotia-$REPONAME
+            image: $IMAGE
+            env:
+              - name: SCOTIA
+                value: $SCOTIA
+              - name: NATSURI
+                value: $NATS_SERVICE_URL
+              - name: WHO
+                value: scotia
+              - name: DOLLAR_SERVICE_URL
+                value: $DOLLAR_SERVICE_URL