Compare commits
10 Commits
8749915e81
...
b655097bbc
| Author | SHA1 | Date | |
|---|---|---|---|
| b655097bbc | |||
| 121d8799a7 | |||
| 3f5366770d | |||
| 1113c9a597 | |||
| 69a5c43d2f | |||
| 5a184a2199 | |||
| 714bc88b07 | |||
| 75f1ce19f9 | |||
| 855a347eac | |||
| c610ac5e15 |
@ -1,31 +0,0 @@
|
|||||||
name: 'build-and-push'
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
push:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: 'checkout'
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Set up Docker Context for Buildx
|
|
||||||
run: docker context create builders
|
|
||||||
- name: 'setup buildx'
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
with:
|
|
||||||
config-inline: |
|
|
||||||
[registry."192.168.0.151:32000"]
|
|
||||||
http = true
|
|
||||||
insecure = true
|
|
||||||
|
|
||||||
- name: 'Build and push Docker image'
|
|
||||||
uses: docker/build-push-action@v5
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
push: true
|
|
||||||
tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest"
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
name: Gitea Actions Demo
|
|
||||||
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
|
|
||||||
on: [push]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
Explore-Gitea-Actions:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- run: echo "🎉 The job was automatically and automagically triggered by a ${{ gitea.event_name }} event."
|
|
||||||
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
|
|
||||||
- run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
|
|
||||||
- name: Check out repository code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
|
|
||||||
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
|
|
||||||
- name: List files in the repository
|
|
||||||
run: |
|
|
||||||
ls ${{ gitea.workspace }}
|
|
||||||
- run: echo "🍏 This job's status is ${{ job.status }}."
|
|
||||||
@ -1,27 +0,0 @@
|
|||||||
name: 'Go test code'
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
test:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: 'Checkout gitea action'
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: 'Setup Go'
|
|
||||||
uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version-file: 'go.mod'
|
|
||||||
|
|
||||||
- name: 'Go version'
|
|
||||||
run: go version
|
|
||||||
|
|
||||||
- name: 'Run Test'
|
|
||||||
run: |
|
|
||||||
export GOPATH="$HOME/go/"
|
|
||||||
export GOBIN=$PATH:GOPATH/bin
|
|
||||||
go test -v ./...
|
|
||||||
114
.github/workflows/build_push_and_deploy.yaml
vendored
Normal file
114
.github/workflows/build_push_and_deploy.yaml
vendored
Normal file
@ -0,0 +1,114 @@
|
|||||||
|
name: ${{ github.actor }} executed Build Push Prod
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v[0-9]+\.[0-9]+\.[0-9]+'
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.TOKEN }}
|
||||||
|
USERNAME: ${{ vars.USERNAME }}
|
||||||
|
TAG_VERSION: ${{ github.ref_name }}
|
||||||
|
GOPROXY: direct
|
||||||
|
steps:
|
||||||
|
- name: echo repo dir
|
||||||
|
id: repo-url
|
||||||
|
run: |
|
||||||
|
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Quemu
|
||||||
|
uses: docker/setup-qemu-action@v3
|
||||||
|
|
||||||
|
- name: Set Up BuildX
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Login to Docker
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
registry: ${{ github.server_url }}
|
||||||
|
username: ${{ vars.USERNAME }}
|
||||||
|
password: ${{ secrets.PWD }}
|
||||||
|
|
||||||
|
- name: calculate short sha
|
||||||
|
id: calculate-sha
|
||||||
|
run: |
|
||||||
|
echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: true
|
||||||
|
tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
|
||||||
|
build-args: |
|
||||||
|
version=${{ github.ref_name }}
|
||||||
|
SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }}
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.TOKEN }}
|
||||||
|
TAG: ${{ github.ref_name }}
|
||||||
|
PASSWORD: $PWD
|
||||||
|
USERNAME: ${{ vars.USERNAME }}
|
||||||
|
TAG_VERSION: ${{ github.ref_name }}
|
||||||
|
NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}"
|
||||||
|
ENV: "production"
|
||||||
|
APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}"
|
||||||
|
DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}"
|
||||||
|
DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}"
|
||||||
|
APA: "${{ vars.APA }}"
|
||||||
|
ACAP: "${{ vars.ACAP }}"
|
||||||
|
ANAP: "${{ vars.ANAP }}"
|
||||||
|
BCCRB: "${{ vars.BCCRB }}"
|
||||||
|
BCD: "${{ vars.BCD }}"
|
||||||
|
BDR: "${{ vars.BDR }}"
|
||||||
|
BHD: "${{ vars.BHD }}"
|
||||||
|
BNC: "${{ vars.BNC }}"
|
||||||
|
BNSC: "${{ vars.BNSC }}"
|
||||||
|
BPD: "${{ vars.BPD }}"
|
||||||
|
GENERAL: "${{ vars.GENERAL }}"
|
||||||
|
SCOTIA: "${{ vars.SCOTIA }}"
|
||||||
|
VIMECA: "${{ vars.VIMENCA }}"
|
||||||
|
steps:
|
||||||
|
- name: echo repo dir
|
||||||
|
id: repo-url
|
||||||
|
run: |
|
||||||
|
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
|
||||||
|
echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: install envsubst
|
||||||
|
run: |
|
||||||
|
apt update
|
||||||
|
apt install -y gettext-base
|
||||||
|
apt clean
|
||||||
|
|
||||||
|
- name: create deployment files
|
||||||
|
env:
|
||||||
|
IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
|
||||||
|
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
|
||||||
|
run: |
|
||||||
|
envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml
|
||||||
|
- name: Download kubectl
|
||||||
|
run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
||||||
|
|
||||||
|
- name: install kubectl
|
||||||
|
run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||||
|
|
||||||
|
- name: insert kube config
|
||||||
|
env:
|
||||||
|
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
|
||||||
|
run: |
|
||||||
|
mkdir ~/.kube
|
||||||
|
export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}"
|
||||||
|
echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
|
||||||
|
- name: apply config
|
||||||
|
run: kubectl apply -f k8s/cronjobs.ymlf
|
||||||
69
.github/workflows/test_project.yml
vendored
Normal file
69
.github/workflows/test_project.yml
vendored
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
name: "dev test"
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- dev
|
||||||
|
- 'feature/**'
|
||||||
|
paths:
|
||||||
|
- '**.go'
|
||||||
|
- '**.yml'
|
||||||
|
- '**.yaml'
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.TOKEN }}
|
||||||
|
USERNAME: ${{ vars.USERNAME }}
|
||||||
|
TAG_VERSION: ${{ github.ref_name }}
|
||||||
|
GOPROXY: "direct"
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- name: run tidy
|
||||||
|
run: |
|
||||||
|
GOPROXY=direct GOPRIVATE=git.maximotejeda.com/maximo go mod tidy
|
||||||
|
- name: test with go
|
||||||
|
run: go test -v ./...
|
||||||
|
|
||||||
|
vulnCheck:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: test
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.TOKEN }}
|
||||||
|
USERNAME: ${{ vars.USERNAME }}
|
||||||
|
TAG_VERSION: ${{ github.ref_name }}
|
||||||
|
steps:
|
||||||
|
- name: checkout Code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: go vuln check
|
||||||
|
uses: golang/govulncheck-action@v1
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
go-package: ./...
|
||||||
|
|
||||||
|
Ci-Lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: vulnCheck
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.TOKEN }}
|
||||||
|
USERNAME: ${{ vars.USERNAME }}
|
||||||
|
TAG_VERSION: ${{ github.ref_name }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Setup Go
|
||||||
|
uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- name: CI lint Check
|
||||||
|
uses: golangci/golangci-lint-action@v6
|
||||||
|
with:
|
||||||
|
only-new-issues: true
|
||||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,3 +2,5 @@
|
|||||||
k8s/deployment.yml
|
k8s/deployment.yml
|
||||||
bin/*
|
bin/*
|
||||||
dolardb/*
|
dolardb/*
|
||||||
|
k8s/cronjobs.yml
|
||||||
|
k8s/cronjobs.yml
|
||||||
|
|||||||
23
Dockerfile
23
Dockerfile
@ -1,11 +1,16 @@
|
|||||||
FROM golang:latest
|
FROM golang:alpine AS builder
|
||||||
ARG BINAME=crawler-linux-arm64-0.0.0_1
|
ARG TARGETARCH
|
||||||
RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps
|
ARG version=not-set
|
||||||
#RUN go run github.com/playwright-community/playwright-go/cmd/playwright@v0.4001.0 install --with-deps
|
ARG SHORTSHA=not-set
|
||||||
#RUN playwright install --with-deps
|
|
||||||
RUN make build
|
|
||||||
COPY ./bin/${BINAME} /usr/bin/crawler
|
|
||||||
RUN mkdir /app
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
COPY . .
|
||||||
|
# https://stackoverflow.com/questions/70369368/check-architecture-in-dockerfile-to-get-amd-arm
|
||||||
|
RUN GOPROXY=direct go build -o bin/crawler \
|
||||||
|
-ldflags "-X main.Shortsha=${SHORTSHA} \
|
||||||
|
-X main.Version=${version} \
|
||||||
|
-X main.Aarch=${TARGETARCH}" ./cmd
|
||||||
|
|
||||||
ENTRYPOINT crawler
|
FROM alpine AS runner
|
||||||
|
COPY --from=builder /app/bin/crawler /usr/bin/
|
||||||
|
WORKDIR /app
|
||||||
|
ENTRYPOINT /usr/bin/crawler
|
||||||
|
|||||||
6
Makefile
6
Makefile
@ -15,8 +15,8 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION)
|
|||||||
CONTAINERS=docker
|
CONTAINERS=docker
|
||||||
COMPOSE=$(CONTAINERS)-compose
|
COMPOSE=$(CONTAINERS)-compose
|
||||||
# Configure local registry
|
# Configure local registry
|
||||||
REGADDR=192.168.0.151:32000
|
REGADDR=git.maximotejeda.com
|
||||||
K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
|
#K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
|
||||||
.phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local
|
.phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local
|
||||||
|
|
||||||
|
|
||||||
@ -26,7 +26,7 @@ build-image: build
|
|||||||
|
|
||||||
# Here we upload it to local
|
# Here we upload it to local
|
||||||
build-test-image:
|
build-test-image:
|
||||||
@$(CONTAINERS) buildx build --platform linux/arm64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old .
|
@$(CONTAINERS) buildx build --platform linux/arm64,linux/amd64 --push -t $(REGADDR)/crawler:latest -f Dockerfile.old .
|
||||||
|
|
||||||
run-image: build-image
|
run-image: build-image
|
||||||
@$(CONTAINERS) compose -f docker-compose.yaml up
|
@$(CONTAINERS) compose -f docker-compose.yaml up
|
||||||
|
|||||||
@ -15,8 +15,7 @@ func main() {
|
|||||||
log := slog.Default()
|
log := slog.Default()
|
||||||
var opts []grpc.DialOption
|
var opts []grpc.DialOption
|
||||||
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||||
conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...)
|
conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("creating gerpc conn", "error", err)
|
log.Error("creating gerpc conn", "error", err)
|
||||||
panic(err)
|
panic(err)
|
||||||
|
|||||||
@ -18,8 +18,8 @@ func GetBCDURL() string {
|
|||||||
return getEnvValue("BCD")
|
return getEnvValue("BCD")
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetBDRURL() string {
|
func GetBRDURL() string {
|
||||||
return getEnvValue("BDR")
|
return getEnvValue("BRD")
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetBHDURL() string {
|
func GetBHDURL() string {
|
||||||
|
|||||||
7
go.mod
7
go.mod
@ -1,10 +1,11 @@
|
|||||||
module github.com/maximotejeda/us_dop_scrapper
|
module github.com/maximotejeda/us_dop_scrapper
|
||||||
|
|
||||||
go 1.22.0
|
go 1.23
|
||||||
|
|
||||||
|
toolchain go1.23.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3
|
github.com/nats-io/nats.go v1.34.1
|
||||||
github.com/nats-io/nats.go v1.33.1
|
|
||||||
github.com/playwright-community/playwright-go v0.4001.0
|
github.com/playwright-community/playwright-go v0.4001.0
|
||||||
golang.org/x/text v0.16.0
|
golang.org/x/text v0.16.0
|
||||||
google.golang.org/grpc v1.65.0
|
google.golang.org/grpc v1.65.0
|
||||||
|
|||||||
20
go.sum
20
go.sum
@ -12,12 +12,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
|||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
|
||||||
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3 h1:J2qbFWgrnQvcVeHGoqBz5YtfLZW2uG18xYRk09BXIeI=
|
|
||||||
github.com/maximotejeda/msvc-proto/golang/dolar v0.0.0-3/go.mod h1:bAs0mlC1Vyn/BkHONL2Ik8ox9px9s9bhbJWgUQFMMWo=
|
|
||||||
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc=
|
||||||
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
|
||||||
github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
|
github.com/nats-io/nats.go v1.34.1 h1:syWey5xaNHZgicYBemv0nohUPPmaLteiBEUT6Q5+F/4=
|
||||||
github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
github.com/nats-io/nats.go v1.34.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
|
||||||
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
|
||||||
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
|
||||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||||
@ -34,37 +32,23 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
|||||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
|
||||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
|
||||||
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
||||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
|
||||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
|
||||||
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
||||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
|
||||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
|
||||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
|
||||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
|
||||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A=
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A=
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
|
||||||
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
|
|
||||||
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
|
|
||||||
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
|
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
|
||||||
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
|
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
|
||||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
|
||||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
|
||||||
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
|
||||||
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
|||||||
@ -13,15 +13,15 @@ import (
|
|||||||
"github.com/playwright-community/playwright-go"
|
"github.com/playwright-community/playwright-go"
|
||||||
)
|
)
|
||||||
|
|
||||||
type bdr struct{}
|
type brd struct{}
|
||||||
|
|
||||||
func NewBDR() ports.APIPorts {
|
func NewBRD() ports.APIPorts {
|
||||||
return &bdr{}
|
return &brd{}
|
||||||
}
|
}
|
||||||
func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
|
func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
|
||||||
tout := 120000.00
|
tout := 120000.00
|
||||||
log = log.With("scrapper", "bdr")
|
log = log.With("scrapper", "brd")
|
||||||
if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{
|
if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{
|
||||||
Timeout: &tout,
|
Timeout: &tout,
|
||||||
WaitUntil: playwright.WaitUntilStateLoad,
|
WaitUntil: playwright.WaitUntilStateLoad,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
|
|||||||
@ -86,10 +86,6 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger
|
|||||||
place := strings.ReplaceAll(n[0], "US$RD$", "")
|
place := strings.ReplaceAll(n[0], "US$RD$", "")
|
||||||
place = strings.ToLower(place)
|
place = strings.ToLower(place)
|
||||||
price := helpers.Normalize(val)
|
price := helpers.Normalize(val)
|
||||||
if err != nil {
|
|
||||||
log.Error("parsing value", "where", place, "err", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
switch place {
|
switch place {
|
||||||
case "compramos":
|
case "compramos":
|
||||||
inst.Compra = price
|
inst.Compra = price
|
||||||
|
|||||||
@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
|
|||||||
case "bcd":
|
case "bcd":
|
||||||
parser = NewBCD()
|
parser = NewBCD()
|
||||||
case "brd":
|
case "brd":
|
||||||
parser = NewBDR()
|
parser = NewBRD()
|
||||||
case "bhd":
|
case "bhd":
|
||||||
parser = NewBHD()
|
parser = NewBHD()
|
||||||
case "bnc":
|
case "bnc":
|
||||||
|
|||||||
@ -3,13 +3,13 @@ package dolar
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/maximotejeda/msvc-proto/golang/dolar"
|
dl "git.maximotejeda.com/maximo/dolar/proto/golang/dolar"
|
||||||
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
|
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Adapter struct {
|
type Adapter struct {
|
||||||
dolar dolar.DollarClient
|
dolar dl.DollarClient
|
||||||
conn *grpc.ClientConn
|
conn *grpc.ClientConn
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -20,8 +20,8 @@ func NewAdapter(conn *grpc.ClientConn) (*Adapter, error) {
|
|||||||
|
|
||||||
func (a *Adapter) NewHistory(history *domain.History) error {
|
func (a *Adapter) NewHistory(history *domain.History) error {
|
||||||
_, err := a.dolar.NewHistory(context.Background(),
|
_, err := a.dolar.NewHistory(context.Background(),
|
||||||
&dolar.AddDolarRequest{
|
&dl.AddDolarRequest{
|
||||||
Institution: &dolar.History{
|
Institution: &dl.History{
|
||||||
Name: history.Name,
|
Name: history.Name,
|
||||||
Compra: float32(history.Compra),
|
Compra: float32(history.Compra),
|
||||||
Venta: float32(history.Venta),
|
Venta: float32(history.Venta),
|
||||||
|
|||||||
255
k8s/cronjobs.yml.template
Normal file
255
k8s/cronjobs.yml.template
Normal file
@ -0,0 +1,255 @@
|
|||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-inf-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "*/20 8-19 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-inf-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: WHO
|
||||||
|
value: inf
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
- name: GENERAL
|
||||||
|
value: $GENERAL
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-bcd-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "3,33 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-bcd-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: BCD
|
||||||
|
value: $BCD
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: bcd
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-bpd-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "5,35 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-bpd-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: BPD
|
||||||
|
value: $BPD
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: bpd
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-bhd-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "8,38 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-bhd-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: BHD
|
||||||
|
value: $BHD
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: bhd
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-brd-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "22,52 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-brd-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: BDR
|
||||||
|
value: $BDR
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: brd
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-apap-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "25,55 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-apap-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: APA
|
||||||
|
value: $APA
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: apap
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-bnc-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "11,41 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-bnc-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: BNC
|
||||||
|
value: $BNC
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: bnc
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-vimenca-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "14,44 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-vimenca-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: VIMENCA
|
||||||
|
value: $VIMENCA
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: vimenca
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: job-scotia-$REPONAME
|
||||||
|
spec:
|
||||||
|
schedule: "17,47 8-22 * * 1-6"
|
||||||
|
successfulJobsHistoryLimit: 1
|
||||||
|
failedJobsHistoryLimit: 1
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
backoffLimit: 2
|
||||||
|
parallelism: 1
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: OnFailure
|
||||||
|
containers:
|
||||||
|
- name: crawler-scotia-$REPONAME
|
||||||
|
image: $IMAGE
|
||||||
|
env:
|
||||||
|
- name: SCOTIA
|
||||||
|
value: $SCOTIA
|
||||||
|
- name: NATSURI
|
||||||
|
value: $NATS_SERVICE_URL
|
||||||
|
- name: WHO
|
||||||
|
value: scotia
|
||||||
|
- name: DOLLAR_SERVICE_URL
|
||||||
|
value: $DOLLAR_SERVICE_URL
|
||||||
Loading…
x
Reference in New Issue
Block a user