Compare commits
No commits in common. "b655097bbcef81eeb8e15c9d0927adcdb6c23a21" and "8749915e811e5c0322354353421e986655dbe6ea" have entirely different histories.
b655097bbc
...
8749915e81
31
.gitea/workflows/build-an-push.yaml
Normal file
31
.gitea/workflows/build-an-push.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
name: 'build-and-push'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
push:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: 'checkout'
|
||||
uses: actions/checkout@v4
|
||||
- name: Set up Docker Context for Buildx
|
||||
run: docker context create builders
|
||||
- name: 'setup buildx'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
config-inline: |
|
||||
[registry."192.168.0.151:32000"]
|
||||
http = true
|
||||
insecure = true
|
||||
|
||||
- name: 'Build and push Docker image'
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest"
|
||||
19
.gitea/workflows/demo.yaml
Normal file
19
.gitea/workflows/demo.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
name: Gitea Actions Demo
|
||||
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
Explore-Gitea-Actions:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: echo "🎉 The job was automatically and automagically triggered by a ${{ gitea.event_name }} event."
|
||||
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
|
||||
- run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v4
|
||||
- run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
|
||||
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
|
||||
- name: List files in the repository
|
||||
run: |
|
||||
ls ${{ gitea.workspace }}
|
||||
- run: echo "🍏 This job's status is ${{ job.status }}."
|
||||
27
.gitea/workflows/test.yaml
Normal file
27
.gitea/workflows/test.yaml
Normal file
@ -0,0 +1,27 @@
|
||||
name: 'Go test code'
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout gitea action'
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 'Setup Go'
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: 'Go version'
|
||||
run: go version
|
||||
|
||||
- name: 'Run Test'
|
||||
run: |
|
||||
export GOPATH="$HOME/go/"
|
||||
export GOBIN=$PATH:GOPATH/bin
|
||||
go test -v ./...
|
||||
114
.github/workflows/build_push_and_deploy.yaml
vendored
114
.github/workflows/build_push_and_deploy.yaml
vendored
@ -1,114 +0,0 @@
|
||||
name: ${{ github.actor }} executed Build Push Prod
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v[0-9]+\.[0-9]+\.[0-9]+'
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
TOKEN: ${{ secrets.TOKEN }}
|
||||
USERNAME: ${{ vars.USERNAME }}
|
||||
TAG_VERSION: ${{ github.ref_name }}
|
||||
GOPROXY: direct
|
||||
steps:
|
||||
- name: echo repo dir
|
||||
id: repo-url
|
||||
run: |
|
||||
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Quemu
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up BuildX
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ github.server_url }}
|
||||
username: ${{ vars.USERNAME }}
|
||||
password: ${{ secrets.PWD }}
|
||||
|
||||
- name: calculate short sha
|
||||
id: calculate-sha
|
||||
run: |
|
||||
echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
|
||||
build-args: |
|
||||
version=${{ github.ref_name }}
|
||||
SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }}
|
||||
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
env:
|
||||
TOKEN: ${{ secrets.TOKEN }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
PASSWORD: $PWD
|
||||
USERNAME: ${{ vars.USERNAME }}
|
||||
TAG_VERSION: ${{ github.ref_name }}
|
||||
NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}"
|
||||
ENV: "production"
|
||||
APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}"
|
||||
DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}"
|
||||
DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}"
|
||||
APA: "${{ vars.APA }}"
|
||||
ACAP: "${{ vars.ACAP }}"
|
||||
ANAP: "${{ vars.ANAP }}"
|
||||
BCCRB: "${{ vars.BCCRB }}"
|
||||
BCD: "${{ vars.BCD }}"
|
||||
BDR: "${{ vars.BDR }}"
|
||||
BHD: "${{ vars.BHD }}"
|
||||
BNC: "${{ vars.BNC }}"
|
||||
BNSC: "${{ vars.BNSC }}"
|
||||
BPD: "${{ vars.BPD }}"
|
||||
GENERAL: "${{ vars.GENERAL }}"
|
||||
SCOTIA: "${{ vars.SCOTIA }}"
|
||||
VIMECA: "${{ vars.VIMENCA }}"
|
||||
steps:
|
||||
- name: echo repo dir
|
||||
id: repo-url
|
||||
run: |
|
||||
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
|
||||
echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: install envsubst
|
||||
run: |
|
||||
apt update
|
||||
apt install -y gettext-base
|
||||
apt clean
|
||||
|
||||
- name: create deployment files
|
||||
env:
|
||||
IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
|
||||
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
|
||||
run: |
|
||||
envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml
|
||||
- name: Download kubectl
|
||||
run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
||||
|
||||
- name: install kubectl
|
||||
run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||
|
||||
- name: insert kube config
|
||||
env:
|
||||
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
|
||||
run: |
|
||||
mkdir ~/.kube
|
||||
export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}"
|
||||
echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
|
||||
- name: apply config
|
||||
run: kubectl apply -f k8s/cronjobs.ymlf
|
||||
69
.github/workflows/test_project.yml
vendored
69
.github/workflows/test_project.yml
vendored
@ -1,69 +0,0 @@
|
||||
name: "dev test"
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- dev
|
||||
- 'feature/**'
|
||||
paths:
|
||||
- '**.go'
|
||||
- '**.yml'
|
||||
- '**.yaml'
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
TOKEN: ${{ secrets.TOKEN }}
|
||||
USERNAME: ${{ vars.USERNAME }}
|
||||
TAG_VERSION: ${{ github.ref_name }}
|
||||
GOPROXY: "direct"
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: run tidy
|
||||
run: |
|
||||
GOPROXY=direct GOPRIVATE=git.maximotejeda.com/maximo go mod tidy
|
||||
- name: test with go
|
||||
run: go test -v ./...
|
||||
|
||||
vulnCheck:
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
env:
|
||||
TOKEN: ${{ secrets.TOKEN }}
|
||||
USERNAME: ${{ vars.USERNAME }}
|
||||
TAG_VERSION: ${{ github.ref_name }}
|
||||
steps:
|
||||
- name: checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: go vuln check
|
||||
uses: golang/govulncheck-action@v1
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-package: ./...
|
||||
|
||||
Ci-Lint:
|
||||
runs-on: ubuntu-latest
|
||||
needs: vulnCheck
|
||||
env:
|
||||
TOKEN: ${{ secrets.TOKEN }}
|
||||
USERNAME: ${{ vars.USERNAME }}
|
||||
TAG_VERSION: ${{ github.ref_name }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- name: CI lint Check
|
||||
uses: golangci/golangci-lint-action@v6
|
||||
with:
|
||||
only-new-issues: true
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -1,6 +1,4 @@
|
||||
.env
|
||||
k8s/deployment.yml
|
||||
bin/*
|
||||
dolardb/*
|
||||
k8s/cronjobs.yml
|
||||
k8s/cronjobs.yml
|
||||
dolardb/*
|
||||
23
Dockerfile
23
Dockerfile
@ -1,16 +1,11 @@
|
||||
FROM golang:alpine AS builder
|
||||
ARG TARGETARCH
|
||||
ARG version=not-set
|
||||
ARG SHORTSHA=not-set
|
||||
FROM golang:latest
|
||||
ARG BINAME=crawler-linux-arm64-0.0.0_1
|
||||
RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps
|
||||
#RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps
|
||||
#RUN playwright install --with-deps
|
||||
RUN make build
|
||||
COPY ./bin/${BINAME} /usr/bin/crawler
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
# https://stackoverflow.com/questions/70369368/check-architecture-in-dockerfile-to-get-amd-arm
|
||||
RUN GOPROXY=direct go build -o bin/crawler \
|
||||
-ldflags "-X main.Shortsha=${SHORTSHA} \
|
||||
-X main.Version=${version} \
|
||||
-X main.Aarch=${TARGETARCH}" ./cmd
|
||||
|
||||
FROM alpine AS runner
|
||||
COPY --from=builder /app/bin/crawler /usr/bin/
|
||||
WORKDIR /app
|
||||
ENTRYPOINT /usr/bin/crawler
|
||||
ENTRYPOINT crawler
|
||||
|
||||
6
Makefile
6
Makefile
@ -15,8 +15,8 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION)
|
||||
CONTAINERS=docker
|
||||
COMPOSE=$(CONTAINERS)-compose
|
||||
# Configure local registry
|
||||
REGADDR=git.maximotejeda.com
|
||||
#K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
|
||||
REGADDR=192.168.0.151:32000
|
||||
K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
|
||||
.phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ build-image: build
|
||||
|
||||
# Here we upload it to local
|
||||
build-test-image:
|
||||
@$(CONTAINERS) buildx build --platform linux/arm64,linux/amd64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old .
|
||||
@$(CONTAINERS) buildx build --platform linux/arm64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old .
|
||||
|
||||
run-image: build-image
|
||||
@$(CONTAINERS) compose -f docker-compose.yaml up
|
||||
|
||||
@ -15,7 +15,8 @@ func main() {
|
||||
log := slog.Default()
|
||||
var opts []grpc.DialOption
|
||||
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...)
|
||||
conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...)
|
||||
|
||||
if err != nil {
|
||||
log.Error("creating gerpc conn", "error", err)
|
||||
panic(err)
|
||||
|
||||
@ -18,8 +18,8 @@ func GetBCDURL() string {
|
||||
return getEnvValue("BCD")
|
||||
}
|
||||
|
||||
func GetBRDURL() string {
|
||||
return getEnvValue("BRD")
|
||||
func GetBDRURL() string {
|
||||
return getEnvValue("BDR")
|
||||
}
|
||||
|
||||
func GetBHDURL() string {
|
||||
|
||||
@ -10,5 +10,5 @@ services:
|
||||
watch:
|
||||
- action: rebuild
|
||||
path: ./
|
||||
|
||||
|
||||
|
||||
|
||||
7
go.mod
7
go.mod
@ -1,11 +1,10 @@
|
||||
module github.com/maximotejeda/us_dop_scrapper
|
||||
|
||||
go 1.23
|
||||
|
||||
toolchain go1.23.2
|
||||
go 1.22.0
|
||||
|
||||
require (
|
||||
github.com/nats-io/nats.go v1.34.1
|
||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3
|
||||
github.com/nats-io/nats.go v1.33.1
|
||||
github.com/playwright-community/playwright-go v0.4001.0
|
||||
golang.org/x/text v0.16.0
|
||||
google.golang.org/grpc v1.65.0
|
||||
|
||||
20
go.sum
20
go.sum
@ -12,10 +12,12 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3 h1:J2qbFWgrnQvcVeHGoqBz5YtfLZW2uG18xYRk09BXIeI=
|
||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3/go.mod h1:bAs0mlC1Vyn/BkHONL2Ik8ox9px9s9bhbJWgUQFMMWo=
|
||||
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
||||
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
||||
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
|
||||
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||
github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
|
||||
github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
@ -32,23 +34,37 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
|
||||
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
|
||||
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
|
||||
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
|
||||
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
@ -13,15 +13,15 @@ import (
|
||||
"github.com/playwright-community/playwright-go"
|
||||
)
|
||||
|
||||
type brd struct{}
|
||||
type bdr struct{}
|
||||
|
||||
func NewBRD() ports.APIPorts {
|
||||
return &brd{}
|
||||
func NewBDR() ports.APIPorts {
|
||||
return &bdr{}
|
||||
}
|
||||
func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
|
||||
func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
|
||||
tout := 120000.00
|
||||
log = log.With("scrapper", "brd")
|
||||
if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{
|
||||
log = log.With("scrapper", "bdr")
|
||||
if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{
|
||||
Timeout: &tout,
|
||||
WaitUntil: playwright.WaitUntilStateLoad,
|
||||
}); err != nil {
|
||||
|
||||
@ -86,6 +86,10 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger
|
||||
place := strings.ReplaceAll(n[0], "US$RD$", "")
|
||||
place = strings.ToLower(place)
|
||||
price := helpers.Normalize(val)
|
||||
if err != nil {
|
||||
log.Error("parsing value", "where", place, "err", err)
|
||||
return nil, err
|
||||
}
|
||||
switch place {
|
||||
case "compramos":
|
||||
inst.Compra = price
|
||||
|
||||
@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
|
||||
case "bcd":
|
||||
parser = NewBCD()
|
||||
case "brd":
|
||||
parser = NewBRD()
|
||||
parser = NewBDR()
|
||||
case "bhd":
|
||||
parser = NewBHD()
|
||||
case "bnc":
|
||||
|
||||
@ -3,13 +3,13 @@ package dolar
|
||||
import (
|
||||
"context"
|
||||
|
||||
dl "git.maximotejeda.com/maximo/dolar/proto/golang/dolar"
|
||||
"github.com/maximotejeda/msvc-proto/golang/dolar"
|
||||
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type Adapter struct {
|
||||
dolar dl.DollarClient
|
||||
dolar dolar.DollarClient
|
||||
conn *grpc.ClientConn
|
||||
}
|
||||
|
||||
@ -20,8 +20,8 @@ func NewAdapter(conn *grpc.ClientConn) (*Adapter, error) {
|
||||
|
||||
func (a *Adapter) NewHistory(history *domain.History) error {
|
||||
_, err := a.dolar.NewHistory(context.Background(),
|
||||
&dl.AddDolarRequest{
|
||||
Institution: &dl.History{
|
||||
&dolar.AddDolarRequest{
|
||||
Institution: &dolar.History{
|
||||
Name: history.Name,
|
||||
Compra: float32(history.Compra),
|
||||
Venta: float32(history.Venta),
|
||||
|
||||
@ -1,255 +0,0 @@
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-inf-$REPONAME
|
||||
spec:
|
||||
schedule: "*/20 8-19 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-inf-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: WHO
|
||||
value: inf
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
- name: GENERAL
|
||||
value: $GENERAL
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-bcd-$REPONAME
|
||||
spec:
|
||||
schedule: "3,33 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-bcd-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: BCD
|
||||
value: $BCD
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: bcd
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-bpd-$REPONAME
|
||||
spec:
|
||||
schedule: "5,35 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-bpd-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: BPD
|
||||
value: $BPD
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: bpd
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-bhd-$REPONAME
|
||||
spec:
|
||||
schedule: "8,38 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-bhd-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: BHD
|
||||
value: $BHD
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: bhd
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-brd-$REPONAME
|
||||
spec:
|
||||
schedule: "22,52 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-brd-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: BDR
|
||||
value: $BDR
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: brd
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-apap-$REPONAME
|
||||
spec:
|
||||
schedule: "25,55 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-apap-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: APA
|
||||
value: $APA
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: apap
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-bnc-$REPONAME
|
||||
spec:
|
||||
schedule: "11,41 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-bnc-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: BNC
|
||||
value: $BNC
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: bnc
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-vimenca-$REPONAME
|
||||
spec:
|
||||
schedule: "14,44 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-vimenca-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: VIMENCA
|
||||
value: $VIMENCA
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: vimenca
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: job-scotia-$REPONAME
|
||||
spec:
|
||||
schedule: "17,47 8-22 * * 1-6"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
parallelism: 1
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: crawler-scotia-$REPONAME
|
||||
image: $IMAGE
|
||||
env:
|
||||
- name: SCOTIA
|
||||
value: $SCOTIA
|
||||
- name: NATSURI
|
||||
value: $NATS_SERVICE_URL
|
||||
- name: WHO
|
||||
value: scotia
|
||||
- name: DOLLAR_SERVICE_URL
|
||||
value: $DOLLAR_SERVICE_URL
|
||||
Loading…
x
Reference in New Issue
Block a user