This commit is contained in:
maximo tejeda 2024-12-02 15:13:52 -04:00
parent 5a184a2199
commit 69a5c43d2f
15 changed files with 450 additions and 100 deletions

View File

@ -1,35 +0,0 @@
name: 'build-and-push'
on:
push:
branches:
- master
jobs:
push:
runs-on: ubuntu-latest
steps:
- name: 'checkout'
uses: actions/checkout@v4
- name: Set up Docker Context for Buildx
run: docker context create builders
- name: 'create.env'
run: touch .env
- name: 'setup buildx'
uses: docker/setup-buildx-action@v3
with:
config-inline: |
[registry."192.168.0.151:32000"]
http = true
insecure = true
- name: 'setup Go Version'
uses: actions/setup-go@v2
with:
go-version: '1.22'
- run: make build
- name: 'Build and push Docker image'
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
tags: "192.168.0.151:32000/crawler:${{gitea.sha}},192.168.0.151:32000/crawler:latest"

View File

@ -1,19 +0,0 @@
name: Gitea Actions Demo
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
on: [push]
jobs:
Explore-Gitea-Actions:
runs-on: ubuntu-latest
steps:
- run: echo "🎉 The job was automatically and automagically triggered by a ${{ gitea.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
- run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
- name: Check out repository code
uses: actions/checkout@v4
- run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ gitea.workspace }}
- run: echo "🍏 This job's status is ${{ job.status }}."

View File

@ -1,27 +0,0 @@
name: 'Go test code'
on:
push:
branches:
- master
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: 'Checkout gitea action'
uses: actions/checkout@v4
- name: 'Setup Go'
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: 'Go version'
run: go version
- name: 'Run Test'
run: |
export GOPATH="$HOME/go/"
export GOBIN=$PATH:GOPATH/bin
go test -v ./...

View File

@ -0,0 +1,113 @@
name: ${{ github.actor }} executed Build Push Prod
on:
push:
tags:
- 'v[0-9]+\.[0-9]+\.[0-9]+'
jobs:
build:
runs-on: ubuntu-latest
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: echo repo dir
id: repo-url
run: |
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Quemu
uses: docker/setup-qemu-action@v3
- name: Set Up BuildX
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@v3
with:
registry: ${{ github.server_url }}
username: ${{ vars.USERNAME }}
password: ${{ secrets.PWD }}
- name: calculate short sha
id: calculate-sha
run: |
echo "shortsha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
build-args: |
version=${{ github.ref_name }}
SHORTSHA=${{ steps.calculate-sha.outputs.shortsha }}
deploy:
runs-on: ubuntu-latest
needs: build
env:
TOKEN: ${{ secrets.TOKEN }}
TAG: ${{ github.ref_name }}
PASSWORD: $PWD
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
NATS_SERVICE_URL: "${{ vars.NATS_SERVICE_URL }}"
ENV: "production"
APPLICATION_PORT: "${{ vars.APPLICATION_PORT }}"
DATA_SOURCE_URL: "${{ vars.DATA_SOURCE_URL }}"
DOLLAR_SERVICE_URL: "${{ vars.DOLLAR_SERVICE_URL }}"
APA: "${{ vars.APA }}"
ACAP: "${{ vars.ACAP }}"
ANAP: "${{ vars.ANAP }}"
BCCRB: "${{ vars.BCCRB }}"
BCD: "${{ vars.BCD }}"
BDR: "${{ vars.BDR }}"
BHD: "${{ vars.BHD }}"
BNC: "${{ vars.BNC }}"
BNSC: "${{ vars.BNSC }}"
BPD: "${{ vars.BPD }}"
GENERAL: "${{ vars.GENERAL }}"
SCOTIA: "${{ vars.SCOTIA }}"
VIMECA: "${{ vars.VIMENCA }}"
steps:
- name: echo repo dir
id: repo-url
run: |
echo "repourl=$(echo ${{ github.server_url }} | sed -e 's;https://;;')" >> $GITHUB_OUTPUT
echo "repoName=$(echo ${{ github.repository }} | sed -e 's;${{ github.actor }}/;;')" >> $GITHUB_OUTPUT
- name: Checkout code
uses: actions/checkout@v4
- name: install envsubst
run: |
apt update
apt install -y gettext-base
apt clean
- name: create deployment files
env:
IMAGE: "${{ steps.repo-url.outputs.repourl }}/${{ github.repository }}:${{ github.ref_name }}"
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
run: |
envsubst < k8s/cronjobs.yml.template > k8s/cronjobs.yml
- name: Download kubectl
run: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
- name: install kubectl
run: sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
- name: insert kube config
env:
REPONAME: "${{ steps.repo-url.outputs.repoName }}"
run: |
mkdir ~/.kube
export IMAGE="$( echo ${{ github.server_url }} | sed -e 's;https://;;')/${{ github.repository }}:${{ github.ref_name }}"
echo "${{ secrets.KUBECONFIG }}" > ~/.kube/config
- name: apply config
run: kubectl apply -f k8s/cronjobs.ymlf

68
.github/workflows/test_project.yml vendored Normal file
View File

@ -0,0 +1,68 @@
name: "dev test"
on:
push:
branches:
- master
- dev
- 'feature/**'
paths:
- '**.go'
- '**.yml'
- '**.yaml'
jobs:
test:
runs-on: ubuntu-latest
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: run tidy
run: |
go mod tidy
- name: test with go
run: go test -v ./...
vulnCheck:
runs-on: ubuntu-latest
needs: test
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: checkout Code
uses: actions/checkout@v4
- name: go vuln check
uses: golang/govulncheck-action@v1
with:
go-version-file: 'go.mod'
go-package: ./...
Ci-Lint:
runs-on: ubuntu-latest
needs: vulnCheck
env:
TOKEN: ${{ secrets.TOKEN }}
USERNAME: ${{ vars.USERNAME }}
TAG_VERSION: ${{ github.ref_name }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: CI lint Check
uses: golangci/golangci-lint-action@v6
with:
only-new-issues: true

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
.env
k8s/deployment.yml
bin/*
dolardb/*
dolardb/*
k8s/cronjobs.yml

View File

@ -8,5 +8,4 @@ RUN ls
COPY ./bin/${BINAME} /usr/bin/crawler
RUN mkdir /app
WORKDIR /app
ENTRYPOINT crawler

View File

@ -15,7 +15,7 @@ BINAMEARM=$(SERVICE)-$(OS)-arm64-$(VERSION)
CONTAINERS=docker
COMPOSE=$(CONTAINERS)-compose
# Configure local registry
REGADDR=192.168.0.151:32000
REGADDR=git.maximotejeda.com
#K8SRSNAME=$(shell kubectl get rs --no-headers -o custom-columns=":metadata.name" | grep bank)
.phony: all clean build test clean-image build-image build-image-debug run-image run-image-debug run-local

View File

@ -15,8 +15,7 @@ func main() {
log := slog.Default()
var opts []grpc.DialOption
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...)
conn, err := grpc.NewClient(config.GetDollarServiceURL(), opts...)
if err != nil {
log.Error("creating gerpc conn", "error", err)
panic(err)

View File

@ -18,8 +18,8 @@ func GetBCDURL() string {
return getEnvValue("BCD")
}
func GetBDRURL() string {
return getEnvValue("BDR")
func GetBRDURL() string {
return getEnvValue("BRD")
}
func GetBHDURL() string {

View File

@ -10,5 +10,5 @@ services:
watch:
- action: rebuild
path: ./

View File

@ -13,15 +13,15 @@ import (
"github.com/playwright-community/playwright-go"
)
type bdr struct{}
type brd struct{}
func NewBDR() ports.APIPorts {
return &bdr{}
func NewBRD() ports.APIPorts {
return &brd{}
}
func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
func (bd brd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
log = log.With("scrapper", "bdr")
if _, err := page.Goto(config.GetBDRURL(), playwright.PageGotoOptions{
log = log.With("scrapper", "brd")
if _, err := page.Goto(config.GetBRDURL(), playwright.PageGotoOptions{
Timeout: &tout,
WaitUntil: playwright.WaitUntilStateLoad,
}); err != nil {

View File

@ -86,10 +86,6 @@ func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger
place := strings.ReplaceAll(n[0], "US$RD$", "")
place = strings.ToLower(place)
price := helpers.Normalize(val)
if err != nil {
log.Error("parsing value", "where", place, "err", err)
return nil, err
}
switch place {
case "compramos":
inst.Compra = price

View File

@ -14,7 +14,7 @@ func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
case "bcd":
parser = NewBCD()
case "brd":
parser = NewBDR()
parser = NewBRD()
case "bhd":
parser = NewBHD()
case "bnc":

255
k8s/cronjobs.yml.template Normal file
View File

@ -0,0 +1,255 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-inf-$REPONAME
spec:
schedule: "*/20 8-19 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-inf-$REPONAME
image: $IMAGE
env:
- name: WHO
value: inf
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
- name: GENERAL
value: $GENERAL
- name: NATSURI
value: $NATS_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bcd-$REPONAME
spec:
schedule: "3,33 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bcd-$REPONAME
image: $IMAGE
env:
- name: BCD
value: $BCD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bcd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bpd-$REPONAME
spec:
schedule: "5,35 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bpd-$REPONAME
image: $IMAGE
env:
- name: BPD
value: $BPD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bpd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bhd-$REPONAME
spec:
schedule: "8,38 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bhd-$REPONAME
image: $IMAGE
env:
- name: BHD
value: $BHD
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bhd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-brd-$REPONAME
spec:
schedule: "22,52 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-brd-$REPONAME
image: $IMAGE
env:
- name: BDR
value: $BDR
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: brd
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-apap-$REPONAME
spec:
schedule: "25,55 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-apap-$REPONAME
image: $IMAGE
env:
- name: APA
value: $APA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: apap
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-bnc-$REPONAME
spec:
schedule: "11,41 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-bnc-$REPONAME
image: $IMAGE
env:
- name: BNC
value: $BNC
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: bnc
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-vimenca-$REPONAME
spec:
schedule: "14,44 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-vimenca-$REPONAME
image: $IMAGE
env:
- name: VIMENCA
value: $VIMENCA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: vimenca
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: job-scotia-$REPONAME
spec:
schedule: "17,47 8-22 * * 1-6"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
backoffLimit: 2
parallelism: 1
template:
spec:
restartPolicy: OnFailure
containers:
- name: crawler-scotia-$REPONAME
image: $IMAGE
env:
- name: SCOTIA
value: $SCOTIA
- name: NATSURI
value: $NATS_SERVICE_URL
- name: WHO
value: scotia
- name: DOLLAR_SERVICE_URL
value: $DOLLAR_SERVICE_URL