Compare commits

...

10 Commits

Author SHA1 Message Date
b655097bbc private
Some checks failed
dev test / test (push) Failing after 1m10s
dev test / vulnCheck (push) Has been skipped
dev test / Ci-Lint (push) Has been skipped
2024-12-02 15:57:25 -04:00
121d8799a7 go proxy direct 2024-12-02 15:55:46 -04:00
3f5366770d dolar from gitea 2024-12-02 15:24:35 -04:00
1113c9a597 FIX docker and protos 2024-12-02 15:19:15 -04:00
69a5c43d2f commit 2024-12-02 15:13:52 -04:00
5a184a2199 Remove kubectl from make file 2024-07-21 18:35:04 -04:00
714bc88b07 touch env actions 2024-07-21 18:33:18 -04:00
75f1ce19f9 Add build step Actions 2024-07-21 18:29:23 -04:00
855a347eac ADD make build step 2024-07-21 13:31:33 -04:00
c610ac5e15 list files 2024-07-21 13:23:56 -04:00
18 changed files with 479 additions and 131 deletions

View File

@ -1,31 +0,0 @@
name: 'build-and-push'
on:
push:
branches:
- master
jobs:
push:
runs-on: ubuntu-latest
steps:
- name: 'checkout'
uses: actions/checkout@v4
- name: Set up Docker Context for Buildx
run: docker context create builders
- name: 'setup buildx'
uses: docker/setup-buildx-action@v3
with:
config-inline: |
[registry."192.168.0.151:32000"]
http = true
insecure = true
- name: 'Build and push Docker image'
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest"

View File

@ -1,19 +0,0 @@
name: Gitea Actions Demo
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
on: [push]
jobs:
Explore-Gitea-Actions:
runs-on: ubuntu-latest
steps:
- run: echo "🎉 The job was automatically and automagically triggered by a ${{ gitea.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
- run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
- name: Check out repository code
uses: actions/checkout@v4
- run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ gitea.workspace }}
- run: echo "🍏 This job's status is ${{ job.status }}."

View File

@ -1,27 +0,0 @@
name: 'Go test code'
on:
push:
branches:
- master
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: 'Checkout gitea action'
uses: actions/checkout@v4
- name: 'Setup Go'
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: 'Go version'
run: go version
- name: 'Run Test'
run: |
export GOPATH="$HOME/go/"
export GOBIN=$PATH:GOPATH/bin
go test -v ./...

View File

@ -0,0 +1,114 @@
name: ${{ github.actor }} executed Build Push Prod
on:
push:
tags:
- 'v[0-9]+\.[0-9]+\.[0-9]+'
jobs:
build:
runs-on: ubuntu-latest
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
GOPROXY: direct
steps:
- name: echo repo dir
id: repo-url
run: |
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Quemu
uses: docker/setup-qemu-action@v3
- name: Set Up BuildX
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@v3
with:
registry: ${{ github.server_url }}
username: ${{ vars.USERNAME }}
password: ${{ secrets.PWD }}
- name: calculate short sha
id: calculate-sha
run: |
echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
build-args: |
version=${{ github.ref_name }}
SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }}
deploy:
runs-on: ubuntu-latest
needs: build
env:
TOKEN: ${{ secrets.TOKEN }}
TAG: ${{ github.ref_name }}
PASSWORD: $PWD
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}"
ENV: "production"
APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}"
DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}"
DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}"
APA: "${{ vars.APA }}"
ACAP: "${{ vars.ACAP }}"
ANAP: "${{ vars.ANAP }}"
BCCRB: "${{ vars.BCCRB }}"
BCD: "${{ vars.BCD }}"
BDR: "${{ vars.BDR }}"
BHD: "${{ vars.BHD }}"
BNC: "${{ vars.BNC }}"
BNSC: "${{ vars.BNSC }}"
BPD: "${{ vars.BPD }}"
GENERAL: "${{ vars.GENERAL }}"
SCOTIA: "${{ vars.SCOTIA }}"
VIMECA: "${{ vars.VIMENCA }}"
steps:
- name: echo repo dir
id: repo-url
run: |
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT
- name: Checkout code
uses: actions/checkout@v4
- name: install envsubst
run: |
apt update
apt install -y gettext-base
apt clean
- name: create deployment files
env:
IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
run: |
envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml
- name: Download kubectl
run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
- name: install kubectl
run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
- name: insert kube config
env:
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
run: |
mkdir ~/.kube
export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}"
echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
- name: apply config
run: kubectl apply -f k8s/cronjobs.ymlf

69
.github/workflows/test_project.yml vendored Normal file
View File

@ -0,0 +1,69 @@
name: "dev test"
on:
push:
branches:
- master
- dev
- 'feature/**'
paths:
- '**.go'
- '**.yml'
- '**.yaml'
jobs:
test:
runs-on: ubuntu-latest
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
GOPROXY: "direct"
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: run tidy
run: |
GOPROXY=direct GOPRIVATE=git.maximotejeda.com/maximo go mod tidy
- name: test with go
run: go test -v ./...
vulnCheck:
runs-on: ubuntu-latest
needs: test
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: checkout Code
uses: actions/checkout@v4
- name: go vuln check
uses: golang/govulncheck-action@v1
with:
go-version-file: 'go.mod'
go-package: ./...
Ci-Lint:
runs-on: ubuntu-latest
needs: vulnCheck
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: CI lint Check
uses: golangci/golangci-lint-action@v6
with:
only-new-issues: true

2
.gitignore vendored
View File

@ -2,3 +2,5 @@
k8s/deployment.yml k8s/deployment.yml
bin/* bin/*
dolardb/* dolardb/*
k8s/cronjobs.yml
k8s/cronjobs.yml

View File

@ -1,11 +1,16 @@
FROM golang:latest FROM golang:alpine AS builder
ARG BINAME=crawler-linux-arm64-0.0.0_1 ARG TARGETARCH
RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps ARG version=not-set
#RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps ARG SHORTSHA=not-set
#RUN playwright install --with-deps
RUN make build
COPY ./bin/${BINAME} /usr/bin/crawler
RUN mkdir /app
WORKDIR /app WORKDIR /app
COPY . .
# https://stackoverflow.com/questions/70369368/check-architecture-in-dockerfile-to-get-amd-arm
RUN GOPROXY=direct go build -o bin/crawler \
-ldflags "-X main.Shortsha=${SHORTSHA} \
-X main.Version=${version} \
-X main.Aarch=${TARGETARCH}" ./cmd
ENTRYPOINT crawler FROM alpine AS runner
COPY --from=builder /app/bin/crawler /usr/bin/
WORKDIR /app
ENTRYPOINT /usr/bin/crawler

View File

@ -15,8 +15,8 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION)
CONTAINERS=docker CONTAINERS=docker
COMPOSE=$(CONTAINERS)-compose COMPOSE=$(CONTAINERS)-compose
# Configure local registry # Configure local registry
REGADDR=192.168.0.151:32000 REGADDR=git.maximotejeda.com
K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank) #K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
.phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local .phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local
@ -26,7 +26,7 @@ build-image: build
# Here we upload it to local # Here we upload it to local
build-test-image: build-test-image:
@$(CONTAINERS) buildx build --platform linux/arm64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old . @$(CONTAINERS) buildx build --platform linux/arm64,linux/amd64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old .
run-image: build-image run-image: build-image
@$(CONTAINERS) compose -f docker-compose.yaml up @$(CONTAINERS) compose -f docker-compose.yaml up

View File

@ -15,8 +15,7 @@ func main() {
log := slog.Default() log := slog.Default()
var opts []grpc.DialOption var opts []grpc.DialOption
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...) conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...)
if err != nil { if err != nil {
log.Error("creating gerpc conn", "error", err) log.Error("creating gerpc conn", "error", err)
panic(err) panic(err)

View File

@ -18,8 +18,8 @@ func GetBCDURL() string {
return getEnvValue("BCD") return getEnvValue("BCD")
} }
func GetBDRURL() string { func GetBRDURL() string {
return getEnvValue("BDR") return getEnvValue("BRD")
} }
func GetBHDURL() string { func GetBHDURL() string {

7
go.mod
View File

@ -1,10 +1,11 @@
module github.com/maximotejeda/us_dop_scrapper module github.com/maximotejeda/us_dop_scrapper
go 1.22.0 go 1.23
toolchain go1.23.2
require ( require (
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3 github.com/nats-io/nats.go v1.34.1
github.com/nats-io/nats.go v1.33.1
github.com/playwright-community/playwright-go v0.4001.0 github.com/playwright-community/playwright-go v0.4001.0
golang.org/x/text v0.16.0 golang.org/x/text v0.16.0
google.golang.org/grpc v1.65.0 google.golang.org/grpc v1.65.0

20
go.sum
View File

@ -12,12 +12,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3 h1:J2qbFWgrnQvcVeHGoqBz5YtfLZW2uG18xYRk09BXIeI=
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3/go.mod h1:bAs0mlC1Vyn/BkHONL2Ik8ox9px9s9bhbJWgUQFMMWo=
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
@ -34,37 +32,23 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A= google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -13,15 +13,15 @@ import (
"github.com/playwright-community/playwright-go" "github.com/playwright-community/playwright-go"
) )
type bdr struct{} type brd struct{}
func NewBDR() ports.APIPorts { func NewBRD() ports.APIPorts {
return &bdr{} return &brd{}
} }
func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00 tout := 120000.00
log = log.With("scrapper", "bdr") log = log.With("scrapper", "brd")
if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{ if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{
Timeout: &tout, Timeout: &tout,
WaitUntil: playwright.WaitUntilStateLoad, WaitUntil: playwright.WaitUntilStateLoad,
}); err != nil { }); err != nil {

View File

@ -86,10 +86,6 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger
place := strings.ReplaceAll(n[0], "US$RD$", "") place := strings.ReplaceAll(n[0], "US$RD$", "")
place = strings.ToLower(place) place = strings.ToLower(place)
price := helpers.Normalize(val) price := helpers.Normalize(val)
if err != nil {
log.Error("parsing value", "where", place, "err", err)
return nil, err
}
switch place { switch place {
case "compramos": case "compramos":
inst.Compra = price inst.Compra = price

View File

@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
case "bcd": case "bcd":
parser = NewBCD() parser = NewBCD()
case "brd": case "brd":
parser = NewBDR() parser = NewBRD()
case "bhd": case "bhd":
parser = NewBHD() parser = NewBHD()
case "bnc": case "bnc":

View File

@ -3,13 +3,13 @@ package dolar
import ( import (
"context" "context"
"github.com/maximotejeda/msvc-proto/golang/dolar" dl "git.maximotejeda.com/maximo/dolar/proto/golang/dolar"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
type Adapter struct { type Adapter struct {
dolar dolar.DollarClient dolar dl.DollarClient
conn *grpc.ClientConn conn *grpc.ClientConn
} }
@ -20,8 +20,8 @@ func NewAdapter(conn *grpc.ClientConn) (*Adapter, error) {
func (a *Adapter) NewHistory(history *domain.History) error { func (a *Adapter) NewHistory(history *domain.History) error {
_, err := a.dolar.NewHistory(context.Background(), _, err := a.dolar.NewHistory(context.Background(),
&dolar.AddDolarRequest{ &dl.AddDolarRequest{
Institution: &dolar.History{ Institution: &dl.History{
Name: history.Name, Name: history.Name,
Compra: float32(history.Compra), Compra: float32(history.Compra),
Venta: float32(history.Venta), Venta: float32(history.Venta),

255
k8s/cronjobs.yml.template Normal file
View File

@ -0,0 +1,255 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-inf-$REPONAME
spec:
schedule: "*/20 8-19 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-inf-$REPONAME
image: $IMAGE
env:
- name: WHO
value: inf
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
- name: GENERAL
value: $GENERAL
- name: NATSURI
value: $NATS_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bcd-$REPONAME
spec:
schedule: "3,33 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bcd-$REPONAME
image: $IMAGE
env:
- name: BCD
value: $BCD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bcd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bpd-$REPONAME
spec:
schedule: "5,35 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bpd-$REPONAME
image: $IMAGE
env:
- name: BPD
value: $BPD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bpd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bhd-$REPONAME
spec:
schedule: "8,38 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bhd-$REPONAME
image: $IMAGE
env:
- name: BHD
value: $BHD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bhd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-brd-$REPONAME
spec:
schedule: "22,52 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-brd-$REPONAME
image: $IMAGE
env:
- name: BDR
value: $BDR
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: brd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-apap-$REPONAME
spec:
schedule: "25,55 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-apap-$REPONAME
image: $IMAGE
env:
- name: APA
value: $APA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: apap
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bnc-$REPONAME
spec:
schedule: "11,41 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bnc-$REPONAME
image: $IMAGE
env:
- name: BNC
value: $BNC
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bnc
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-vimenca-$REPONAME
spec:
schedule: "14,44 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-vimenca-$REPONAME
image: $IMAGE
env:
- name: VIMENCA
value: $VIMENCA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: vimenca
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-scotia-$REPONAME
spec:
schedule: "17,47 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-scotia-$REPONAME
image: $IMAGE
env:
- name: SCOTIA
value: $SCOTIA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: scotia
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL