From 69a5c43d2f618610874afe2d5fee85c0636b1a8b Mon Sep 17 00:00:00 2001 From: maximo tejeda Date: Mon, 2 Dec 2024 15:13:52 -0400 Subject: [PATCH] commit --- .gitea/workflows/build-an-push.yaml | 35 --- .gitea/workflows/demo.yaml | 19 -- .gitea/workflows/test.yaml | 27 -- .github/workflows/build_push_and_deploy.yaml | 113 ++++++++ .github/workflows/test_project.yml | 68 +++++ .gitignore | 3 +- Dockerfile | 1 - Makefile | 2 +- cmd/crawler/main.go | 3 +- config/config.go | 4 +- docker-compose.yaml | 2 +- internal/adapters/crawler/bdr.go | 12 +- internal/adapters/crawler/bhd.go | 4 - internal/adapters/crawler/crawler.go | 2 +- k8s/cronjobs.yml.template | 255 +++++++++++++++++++ 15 files changed, 450 insertions(+), 100 deletions(-) delete mode 100644 .gitea/workflows/build-an-push.yaml delete mode 100644 .gitea/workflows/demo.yaml delete mode 100644 .gitea/workflows/test.yaml create mode 100644 .github/workflows/build_push_and_deploy.yaml create mode 100644 .github/workflows/test_project.yml create mode 100644 k8s/cronjobs.yml.template diff --git a/.gitea/workflows/build-an-push.yaml b/.gitea/workflows/build-an-push.yaml deleted file mode 100644 index 3bd7558..0000000 --- a/.gitea/workflows/build-an-push.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: 'build-and-push' - -on: - push: - branches: - - master -jobs: - push: - runs-on: ubuntu-latest - steps: - - name: 'checkout' - uses: actions/checkout@v4 - - name: Set up Docker Context for Buildx - run: docker context create builders - - name: 'create.env' - run: touch .env - - name: 'setup buildx' - uses: docker/setup-buildx-action@v3 - with: - config-inline: | - [registry."192.168.0.151:32000"] - http = true - insecure = true - - name: 'setup Go Version' - uses: actions/setup-go@v2 - with: - go-version: '1.22' - - run: make build - - name: 'Build and push Docker image' - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile - push: true - tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest" diff --git a/.gitea/workflows/demo.yaml b/.gitea/workflows/demo.yaml deleted file mode 100644 index e942159..0000000 --- a/.gitea/workflows/demo.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: Gitea Actions Demo -run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀 -on: [push] - -jobs: - Explore-Gitea-Actions: - runs-on: ubuntu-latest - steps: - - run: echo "🎉 The job was automatically and automagically triggered by a ${{ gitea.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" - - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}." - - name: Check out repository code - uses: actions/checkout@v4 - - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner." - - run: echo "🖥️ The workflow is now ready to test your code on the runner." - - name: List files in the repository - run: | - ls ${{ gitea.workspace }} - - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.gitea/workflows/test.yaml b/.gitea/workflows/test.yaml deleted file mode 100644 index 43baad7..0000000 --- a/.gitea/workflows/test.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: 'Go test code' - -on: - push: - branches: - - master - -jobs: - test: - runs-on: ubuntu-latest - steps: - - name: 'Checkout gitea action' - uses: actions/checkout@v4 - - - name: 'Setup Go' - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - - - name: 'Go version' - run: go version - - - name: 'Run Test' - run: | - export GOPATH="$HOME/go/" - export GOBIN=$PATH:GOPATH/bin - go test -v ./... diff --git a/.github/workflows/build_push_and_deploy.yaml b/.github/workflows/build_push_and_deploy.yaml new file mode 100644 index 0000000..9f52ded --- /dev/null +++ b/.github/workflows/build_push_and_deploy.yaml @@ -0,0 +1,113 @@ +name: ${{ github.actor }} executed Build Push Prod +on: + push: + tags: + - 'v[0-9]+\.[0-9]+\.[0-9]+' +jobs: + build: + runs-on: ubuntu-latest + env: + TOKEN: ${{ secrets.TOKEN }} + USERNAME: ${{ vars.USERNAME }} + TAG_VERSION: ${{ github.ref_name }} + steps: + - name: echo repo dir + id: repo-url + run: | + echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Quemu + uses: docker/setup-qemu-action@v3 + + - name: Set Up BuildX + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker + uses: docker/login-action@v3 + with: + registry: ${{ github.server_url }} + username: ${{ vars.USERNAME }} + password: ${{ secrets.PWD }} + + - name: calculate short sha + id: calculate-sha + run: | + echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}" + build-args: | + version=${{ github.ref_name }} + SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }} + + deploy: + runs-on: ubuntu-latest + needs: build + env: + TOKEN: ${{ secrets.TOKEN }} + TAG: ${{ github.ref_name }} + PASSWORD: $PWD + USERNAME: ${{ vars.USERNAME }} + TAG_VERSION: ${{ github.ref_name }} + NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}" + ENV: "production" + APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}" + DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}" + DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}" + APA: "${{ vars.APA }}" + ACAP: "${{ vars.ACAP }}" + ANAP: "${{ vars.ANAP }}" + BCCRB: "${{ vars.BCCRB }}" + BCD: "${{ vars.BCD }}" + BDR: "${{ vars.BDR }}" + BHD: "${{ vars.BHD }}" + BNC: "${{ vars.BNC }}" + BNSC: "${{ vars.BNSC }}" + BPD: "${{ vars.BPD }}" + GENERAL: "${{ vars.GENERAL }}" + SCOTIA: "${{ vars.SCOTIA }}" + VIMECA: "${{ vars.VIMENCA }}" + steps: + - name: echo repo dir + id: repo-url + run: | + echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT + echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT + - name: Checkout code + uses: actions/checkout@v4 + + - name: install envsubst + run: | + apt update + apt install -y gettext-base + apt clean + + - name: create deployment files + env: + IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}" + REPONAME: "${{ steps.repo-url.outputs.repoName }}" + run: | + envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml + - name: Download kubectl + run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + + - name: install kubectl + run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + + - name: insert kube config + env: + REPONAME: "${{ steps.repo-url.outputs.repoName }}" + run: | + mkdir ~/.kube + export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}" + echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config + - name: apply config + run: kubectl apply -f k8s/cronjobs.ymlf diff --git a/.github/workflows/test_project.yml b/.github/workflows/test_project.yml new file mode 100644 index 0000000..5c61a57 --- /dev/null +++ b/.github/workflows/test_project.yml @@ -0,0 +1,68 @@ +name: "dev test" +on: + push: + branches: + - master + - dev + - 'feature/**' + paths: + - '**.go' + - '**.yml' + - '**.yaml' +jobs: + test: + runs-on: ubuntu-latest + env: + TOKEN: ${{ secrets.TOKEN }} + USERNAME: ${{ vars.USERNAME }} + TAG_VERSION: ${{ github.ref_name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: run tidy + run: | + go mod tidy + - name: test with go + run: go test -v ./... + + vulnCheck: + runs-on: ubuntu-latest + needs: test + env: + TOKEN: ${{ secrets.TOKEN }} + USERNAME: ${{ vars.USERNAME }} + TAG_VERSION: ${{ github.ref_name }} + steps: + - name: checkout Code + uses: actions/checkout@v4 + + - name: go vuln check + uses: golang/govulncheck-action@v1 + with: + go-version-file: 'go.mod' + go-package: ./... + + Ci-Lint: + runs-on: ubuntu-latest + needs: vulnCheck + env: + TOKEN: ${{ secrets.TOKEN }} + USERNAME: ${{ vars.USERNAME }} + TAG_VERSION: ${{ github.ref_name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + - name: CI lint Check + uses: golangci/golangci-lint-action@v6 + with: + only-new-issues: true diff --git a/.gitignore b/.gitignore index cba9d71..a9a774b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env k8s/deployment.yml bin/* -dolardb/* \ No newline at end of file +dolardb/* +k8s/cronjobs.yml diff --git a/Dockerfile b/Dockerfile index 2e9659a..cd757c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,5 +8,4 @@ RUN ls COPY ./bin/${BINAME} /usr/bin/crawler RUN mkdir /app WORKDIR /app - ENTRYPOINT crawler diff --git a/Makefile b/Makefile index 8124375..c7d0fe1 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION) CONTAINERS=docker COMPOSE=$(CONTAINERS)-compose # Configure local registry -REGADDR=192.168.0.151:32000 +REGADDR=git.maximotejeda.com #K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank) .phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index 6ba3836..567ef98 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -15,8 +15,7 @@ func main() { log := slog.Default() var opts []grpc.DialOption opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...) - + conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...) if err != nil { log.Error("creating gerpc conn", "error", err) panic(err) diff --git a/config/config.go b/config/config.go index 6bc675f..1088d7c 100644 --- a/config/config.go +++ b/config/config.go @@ -18,8 +18,8 @@ func GetBCDURL() string { return getEnvValue("BCD") } -func GetBDRURL() string { - return getEnvValue("BDR") +func GetBRDURL() string { + return getEnvValue("BRD") } func GetBHDURL() string { diff --git a/docker-compose.yaml b/docker-compose.yaml index 1a46bab..1eac457 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,5 +10,5 @@ services: watch: - action: rebuild path: ./ - + diff --git a/internal/adapters/crawler/bdr.go b/internal/adapters/crawler/bdr.go index 2591a14..4d85c80 100644 --- a/internal/adapters/crawler/bdr.go +++ b/internal/adapters/crawler/bdr.go @@ -13,15 +13,15 @@ import ( "github.com/playwright-community/playwright-go" ) -type bdr struct{} +type brd struct{} -func NewBDR() ports.APIPorts { - return &bdr{} +func NewBRD() ports.APIPorts { + return &brd{} } -func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { +func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 - log = log.With("scrapper", "bdr") - if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{ + log = log.With("scrapper", "brd") + if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{ Timeout: &tout, WaitUntil: playwright.WaitUntilStateLoad, }); err != nil { diff --git a/internal/adapters/crawler/bhd.go b/internal/adapters/crawler/bhd.go index 6f0d0f2..7986728 100644 --- a/internal/adapters/crawler/bhd.go +++ b/internal/adapters/crawler/bhd.go @@ -86,10 +86,6 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger place := strings.ReplaceAll(n[0], "US$RD$", "") place = strings.ToLower(place) price := helpers.Normalize(val) - if err != nil { - log.Error("parsing value", "where", place, "err", err) - return nil, err - } switch place { case "compramos": inst.Compra = price diff --git a/internal/adapters/crawler/crawler.go b/internal/adapters/crawler/crawler.go index f6648b5..c4af8cb 100644 --- a/internal/adapters/crawler/crawler.go +++ b/internal/adapters/crawler/crawler.go @@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) { case "bcd": parser = NewBCD() case "brd": - parser = NewBDR() + parser = NewBRD() case "bhd": parser = NewBHD() case "bnc": diff --git a/k8s/cronjobs.yml.template b/k8s/cronjobs.yml.template new file mode 100644 index 0000000..f0e0ea3 --- /dev/null +++ b/k8s/cronjobs.yml.template @@ -0,0 +1,255 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-inf-$REPONAME +spec: + schedule: "*/20 8-19 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-inf-$REPONAME + image: $IMAGE + env: + - name: WHO + value: inf + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL + - name: GENERAL + value: $GENERAL + - name: NATSURI + value: $NATS_SERVICE_URL +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-bcd-$REPONAME +spec: + schedule: "3,33 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-bcd-$REPONAME + image: $IMAGE + env: + - name: BCD + value: $BCD + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: bcd + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-bpd-$REPONAME +spec: + schedule: "5,35 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-bpd-$REPONAME + image: $IMAGE + env: + - name: BPD + value: $BPD + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: bpd + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-bhd-$REPONAME +spec: + schedule: "8,38 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-bhd-$REPONAME + image: $IMAGE + env: + - name: BHD + value: $BHD + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: bhd + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-brd-$REPONAME +spec: + schedule: "22,52 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-brd-$REPONAME + image: $IMAGE + env: + - name: BDR + value: $BDR + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: brd + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-apap-$REPONAME +spec: + schedule: "25,55 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-apap-$REPONAME + image: $IMAGE + env: + - name: APA + value: $APA + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: apap + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-bnc-$REPONAME +spec: + schedule: "11,41 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-bnc-$REPONAME + image: $IMAGE + env: + - name: BNC + value: $BNC + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: bnc + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-vimenca-$REPONAME +spec: + schedule: "14,44 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-vimenca-$REPONAME + image: $IMAGE + env: + - name: VIMENCA + value: $VIMENCA + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: vimenca + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: job-scotia-$REPONAME +spec: + schedule: "17,47 8-22 * * 1-6" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 2 + parallelism: 1 + template: + spec: + restartPolicy: OnFailure + containers: + - name: crawler-scotia-$REPONAME + image: $IMAGE + env: + - name: SCOTIA + value: $SCOTIA + - name: NATSURI + value: $NATS_SERVICE_URL + - name: WHO + value: scotia + - name: DOLLAR_SERVICE_URL + value: $DOLLAR_SERVICE_URL