maximo/us-dop-scrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

removing old code adopting new architecture

Browse Source
This commit is contained in:
maximo tejeda 2024-04-18 09:33:55 -04:00
parent 7d28c9649a
commit 151c8f7a7b
28 changed files with 263 additions and 843 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
apa/apa.go
View File

@ -1,97 +0,0 @@
package apa
import (
"context"
"fmt"
"log/slog"
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("APA")
)
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
tout := 120000.00
log = log.With("scrapper", "apap")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
WaitUntil: playwright.WaitUntilStateLoad,
}); err != nil {
log.Error("could not get info", "error", err)
return nil, err
}
button := page.Locator("#exchangesRates")
button.WaitFor()
button.Click()
compraLocator := page.Locator("#currency-buy-USD")
ventaLocator := page.Locator("#currency-sell-USD")
compraSTR, err := compraLocator.TextContent()
if err != nil {
log.Error("could not get compra str", "err", err)
return nil, err
}
ventaSTR, err := ventaLocator.TextContent()
if err != nil {
log.Error("could not get venta string", "err", err)
return nil, err
}
inst = &db.History{
Name: "asociacion popular de ahorros y prestamos",
Parser: "apap",
Parsed: time.Now().UTC(),
}
inst.Venta = helpers.Normalize(ventaSTR)
inst.Compra = helpers.Normalize(compraSTR)
if inst.Compra == 0 || inst.Venta == 0 {
return nil, fmt.Errorf("apa: institution not parsed: %v", inst)
}
log.Info("parsed", "value", inst)
return inst, nil
}
func ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
ua := helpers.NewMobileUA()
b := *browser
page, err := b.NewPage(playwright.BrowserNewPageOptions{
UserAgent: &ua,
// IsMobile: &t,
HasTouch: &t,
Viewport: &playwright.Size{
Width: 412,
Height: 915,
},
Screen: &playwright.Size{
Width: 412,
Height: 915,
},
})
if err != nil {
log.Error("creating page", "error", err)
os.Exit(1)
}
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
// here we execute db operations
if err != nil {
return err
}
err = db.Inspect(*inst)
return err
}

BIN
bin/crawler-linux-amd64-0.0.0_1
View File

Binary file not shown.

BIN
bin/crawler-linux-arm64-0.0.0_1
View File

Binary file not shown.

92
cmd/crawler/main.go
View File

@ -1,79 +1,37 @@
package main
import (
"context"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/apa"
"github.com/maximotejeda/us_dop_scrapper/bcd"
"github.com/maximotejeda/us_dop_scrapper/bdr"
"github.com/maximotejeda/us_dop_scrapper/bhd"
"github.com/maximotejeda/us_dop_scrapper/bnc"
"github.com/maximotejeda/us_dop_scrapper/bpd"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/inf"
"github.com/maximotejeda/us_dop_scrapper/scotia"
"github.com/maximotejeda/us_dop_scrapper/vimenca"
"github.com/playwright-community/playwright-go"
"github.com/maximotejeda/us_dop_scrapper/config"
"github.com/maximotejeda/us_dop_scrapper/internal/adapters/crawler"
dl "github.com/maximotejeda/us_dop_scrapper/internal/adapters/dolar"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/api"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func main() {
var err error
dbRoute := os.Getenv("DBURI")
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{}))
db := db.Dial(dbRoute, log)
// create a chrome and feed parsers
db.CreateTables()
chrome, firefox, webkit := helpers.CreateBrowser(log)
browserList := []*playwright.Browser{chrome, firefox, webkit}
longTick := time.NewTicker(time.Minute * 2)
defer longTick.Stop()
infoTick := time.NewTicker(time.Minute * 1)
defer infoTick.Stop()
ctx, cancel := context.WithCancel(context.Background())
errN := map[string]int{
"bcd": 0,
"bpd": 0,
"apap": 0,
"inf": 0,
}
defer cancel()
who := os.Getenv("WHO")
switch who {
case "bcd":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bcd", bcd.ExecParser)
case "bpd":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bpd", bpd.ExecParser)
case "apa":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "apa", apa.ExecParser)
case "brd":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "brd", bdr.ExecParser)
case "bhd":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bhd", bhd.ExecParser)
case "bnc":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bnc", bnc.ExecParser)
case "scotia":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "scotia", scotia.ExecParser)
case "vimenca":
err = helpers.ExecTask(ctx, db, browserList, log, errN, "vimenca", vimenca.ExecParser)
default:
err = helpers.ExecTask(ctx, db, browserList, log, errN, "inf", inf.ExecParser)
}
log := slog.Default()
var opts []grpc.DialOption
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...)
if err != nil {
log.Info("task executed with errors", "name", who, "error", err)
os.Exit(1)
return
log.Error("creating gerpc conn", "error", err)
panic(err)
}
log.Info("SUCCESS - task executed", "name", who)
os.Exit(0)
defer conn.Close()
dol, err := dl.NewAdapter(conn)
if err != nil {
log.Error("creating service adapter", "error", err)
panic(err)
}
crawler, err := crawler.Selector(config.GetWho(), dol)
if err != nil {
log.Error("selecting crawler adapter", "error", err)
panic(err)
}
app := api.NewApplication(crawler)
app.Run()
}

4
config/config.go
View File

@ -6,6 +6,10 @@ func GetWho() string {
return getEnvValue("WHO")
}
func GetDollarServiceURL() string {
return getEnvValue("DOLLAR_SERVICE_URL")
}
func getEnvValue(key string) string {
if os.Getenv(key) == "" {
panic("key not found " + key)

423
db-old/db.go
View File

@ -1,423 +0,0 @@
package db
import (
"database/sql"
_ "embed"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/maximotejeda/us_dop_scrapper/models"
"github.com/maximotejeda/us_dop_scrapper/pub"
_ "modernc.org/sqlite"
)
//go:embed schema.sql
var schema string
type DB struct {
*sql.DB
log *slog.Logger
}
type change struct {
Before models.Institucion `json:"before"`
After models.Institucion `json:"after"`
}
type Message struct {
Message string `json:"message"`
Data change `json:"data"`
Error error `json:"error"`
}
type Institution struct {
ID int
Name string
ShortName string
Created time.Time
}
// Dial
func Dial(path string, log *slog.Logger) *DB {
db, err := sql.Open("sqlite", path)
if err != nil {
fmt.Printf("opening database: %s", err.Error())
panic("opening database")
}
if err := db.Ping(); err != nil {
fmt.Printf("pinging database: %s", err.Error())
panic("pinging database")
}
return &DB{db, log}
}
// Schema
func (db *DB) CreateTables() {
_, err := db.Exec(schema)
if err != nil {
panic(err)
}
}
// Inspect
// Handle behavior of the changes
// Will report errors to a nats consumer
func (db *DB) Inspect(enter models.Institucion) error {
if db == nil {
return fmt.Errorf("nil or empty database")
}
pub, close := pub.Publisher()
defer close()
msg := Message{}
// Get last row added
inst, err := db.GetLatest(enter.Parser, enter.Name)
// if no rows are found because of first enter a name - parser ?
if errors.Is(sql.ErrNoRows, err) {
db.log.Info("adding new item to table: ", "parse", enter.Parser, "name", enter.Name)
msg.Message = "add new institution"
msg.Data.After = enter
data, err := json.Marshal(msg)
if err != nil {
db.log.Error("marshaling struct", "error", err)
}
id, err := db.ADDInstitution(enter.Name)
if err != nil {
return err
}
defer pub("dolar-crawler", data)
return db.AddNew(enter, id)
}
// check prices compra venta
if inst == nil {
db.log.Error("row is nil", "name", enter.Name, "parser", enter.Parser)
return fmt.Errorf("row is nil, not entering row")
}
if enter.Compra == inst.Compra && enter.Venta == inst.Venta {
return nil
} else {
// if one of them changes create a new row
db.log.Info("change registered, adding item", "parse", enter.Parser, "name", enter.Name, "compra enter", enter.Compra, "compra db", inst.Compra, "venta enter", enter.Venta, "venta db", inst.Venta)
msg.Message = "change registered"
msg.Data.After = enter
msg.Data.Before = *inst
data, err := json.Marshal(msg)
if err != nil {
db.log.Error("marshaling struct", "error", err)
}
ins, err := db.GETInstitution(enter.Name)
if err != nil {
return err
}
defer pub("dolar-crawler", data)
return db.AddNew(enter, int64(ins.ID))
}
}
// GetLatest
// returns the latest row in a specific parser and name
// we are using DateTime in DB and date.Datetime in go
func (db *DB) GetLatest(parser string, name string) (inst *models.Institucion, err error) {
var parsed string
inst = &models.Institucion{}
stmtt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE d.parser = ? AND i.name = ? ORDER BY d.parsed DESC LIMIT 1;")
if err != nil {
db.log.Error("preparing stmtt", "error", err.Error())
return nil, err
}
defer stmtt.Close()
if err := stmtt.QueryRow(parser, name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
db.log.Error("getting latest", "error", err.Error(), "parser", parser, "name", name)
return nil, err
}
inst.Parsed, err = time.Parse(time.DateTime, parsed)
if err != nil {
//db.log.Error("parsed", "error", err.Error())
return nil, err
}
return inst, nil
}
// AddNew
// Add a new row in the dolar table
// Will send to nats changes on prices
func (db *DB) AddNew(row models.Institucion, id int64) error {
stmt, err := db.Prepare("INSERT INTO dolars (name_id, compra, venta, parser, parsed) VALUES(?,?,?,?,?);")
if err != nil {
return err
}
defer stmt.Close()
parsed := row.Parsed.Format(time.DateTime)
_, err = stmt.Exec(&id, &row.Compra, &row.Venta, &row.Parser, &parsed)
if err != nil {
return err
}
return nil
}
func (db *DB) ADDInstitution(name string) (id int64, err error) {
stmt, err := db.Prepare("INSERT INTO institutions (name, short_name, created) VALUES(?,?,?);")
if err != nil {
return 0, err
}
defer stmt.Close()
parsed := time.Now().Format(time.DateTime)
short := shortner(name)
res, err := stmt.Exec(&name, short, &parsed)
if err != nil {
return 0, err
}
id, err = res.LastInsertId()
if err != nil {
return 0, err
}
return id, nil
}
func (db *DB) GETInstitution(name string) (inst *Institution, err error) {
institution := Institution{}
stmtt, err := db.Prepare("SELECT id, name, short_name FROM institutions WHERE name = ?")
if err != nil {
db.log.Error("preparing stmt", "error", err.Error())
return nil, err
}
defer stmtt.Close()
if err := stmtt.QueryRow(name).Scan(&institution.ID, &institution.Name, &institution.ShortName); err != nil {
db.log.Error("getting institution", "error", err.Error(), "short name", institution.ShortName, "name", name)
return nil, err
}
return inst, err
}
func (db *DB) GetAll() ([]string, error) {
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i;")
if err != nil {
db.log.Error("[db-GetAll]", "error", err)
return nil, err
}
rows, err := stmt.Query()
if err != nil {
db.log.Error("[db-GetAll-stmt]", "error", err)
return nil, err
}
defer rows.Close()
insts := []string{}
for rows.Next() {
inst := ""
if err = rows.Scan(&inst); err != nil {
return nil, err
}
if inst == "" {
continue
}
insts = append(insts, inst)
}
if err := rows.Err(); err != nil {
return insts, err
}
return insts, nil
}
func (db *DB) GetBancos() ([]string, error) {
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%ban%' OR i.name LIKE '%scoti%'")
if err != nil {
db.log.Error("[inst-GetAll]", "error", err)
return nil, err
}
rows, err := stmt.Query()
if err != nil {
db.log.Error("[inst-GetAll-stmt]", "error", err)
return nil, err
}
defer rows.Close()
insts := []string{}
for rows.Next() {
inst := ""
if err = rows.Scan(&inst); err != nil {
return nil, err
}
if inst == "" {
continue
}
insts = append(insts, inst)
}
if err := rows.Err(); err != nil {
return insts, err
}
return insts, nil
}
func (db *DB) GetCajas() ([]string, error) {
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%asociacion%'")
if err != nil {
db.log.Error("[inst-GetAll]", "error", err)
return nil, err
}
rows, err := stmt.Query()
if err != nil {
db.log.Error("[inst-GetAll-stmt]", "error", err)
return nil, err
}
defer rows.Close()
insts := []string{}
for rows.Next() {
inst := ""
if err = rows.Scan(&inst); err != nil {
return nil, err
}
if inst == "" {
continue
}
insts = append(insts, inst)
}
if err := rows.Err(); err != nil {
return insts, err
}
return insts, nil
}
func (db *DB) GetAgentes() ([]string, error) {
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name NOT LIKE '%ban%' AND i.name NOT LIKE '%scoti%' AND i.name NOT LIKE '%asociacion%'")
if err != nil {
db.log.Error("[inst-GetAll]", "error", err)
return nil, err
}
rows, err := stmt.Query()
if err != nil {
db.log.Error("[inst-GetAll-stmt]", "error", err)
return nil, err
}
defer rows.Close()
insts := []string{}
for rows.Next() {
inst := ""
if err = rows.Scan(&inst); err != nil {
return nil, err
}
if inst == "" {
continue
}
insts = append(insts, inst)
}
if err := rows.Err(); err != nil {
return insts, err
}
return insts, nil
}
func (db *DB) GetLastPrice(name string) (inst *models.Institucion, err error) {
var parsed string
inst = &models.Institucion{}
stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? ORDER BY parsed DESC LIMIT 1;")
if err != nil {
db.log.Error("preparing", "error", err.Error())
return nil, err
}
defer stmt.Close()
if err := stmt.QueryRow(name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
db.log.Error("getting last price", "error", err.Error(), "name", name)
return nil, err
}
inst.Parsed, err = time.Parse(time.DateTime, parsed)
if err != nil {
//db.log.Error("parsed", "error", err.Error())
return nil, err
}
return inst, nil
}
func (db *DB) GetChangeSince(name string, duration time.Duration) (insts []*models.Institucion, err error) {
date := time.Now().Add(-duration).Format(time.DateTime)
stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? AND parsed > ? ORDER BY parsed DESC;")
if err != nil {
db.log.Error("[GetChangeSince] preparing", "error", err.Error())
return nil, err
}
defer stmt.Close()
rows, err := stmt.Query(name, date)
if err != nil {
db.log.Error("[GetChangeSince] preparing", "error", err.Error())
return nil, err
}
defer rows.Close()
for rows.Next() {
inst := models.Institucion{}
parsed := ""
if err := rows.Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
db.log.Error("[GetChangeSince] scanning", "error", err)
return nil, err
}
inst.Parsed, err = time.Parse(time.DateTime, parsed)
if err != nil {
//db.log.Error("parsed", "error", err.Error())
continue
}
insts = append(insts, &inst)
}
return insts, nil
}
func shortner(name string) string {
if name == "" {
return ""
}
switch strings.ToLower(name) {
case "banco popular":
return "bpd"
case "banreservas":
return "brd"
case "banco central dominicano":
return "bcd"
case "banco hipotecario dominicano":
return "bhd"
case "asociacion popular de ahorros y prestamos":
return "apap"
case "asociacion cibao de ahorros y prestamos":
return "acap"
case "asociacion la nacional de ahorros y prestamos":
return "alnap"
case "asociacion peravia de ahorros y prestamos":
return "apeap"
case "banco santa cruz":
return "bsc"
case "imbert y balbuena":
return "imb"
case "banco activo dominicana":
return "bacd"
case "scotiabank cambio online":
return "scline"
case "banco lopez de haro":
return "blh"
}
nameList := strings.Split(name, " ")
switch len(nameList) {
case 1:
return nameList[0]
case 2:
return string(nameList[0][0]) + nameList[1][0:2]
case 3:
return string(nameList[0][0] + nameList[1][0] + nameList[2][0])
default:
return "n/a"
}
}

18
db-old/schema.sql
View File

@ -1,18 +0,0 @@
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS 'dolars' (
id INTEGER PRIMARY KEY,
name_id INTEGER NOT NULL,
compra REAL NOT NULL,
venta REAL NOT NULL,
parser TEXT NOT NULL,
parsed TEXT NOT NULL,
FOREIGN KEY(name_id) REFERENCES institutions(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS 'institutions' (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
short_name TEXT NOT NULL,
created TEXT NOT NULL
);

1
helpers/.#helpers.go
View File

@ -1 +0,0 @@
maximo@debian-pc.9800:1713363571

1
helpers/helpers.go
View File

@ -162,7 +162,6 @@ func CreateBrowser(log *slog.Logger) (chrome *playwright.Browser, firefox *playw
// ExecTask
func ExecTask(
ctx context.Context,
dbi *db.DB,
browser []*playwright.Browser,
log *slog.Logger,

1
inf/inf_test.go
View File

@ -1 +0,0 @@
package inf

16
internal/adapters/crawler/apap.go
View File

@ -13,19 +13,17 @@ import (
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("APA")
)
type Apap struct {
client ports.DollarPort
}
func NewApap() ports.APIPorts {
return &Apap{}
func NewApap(client ports.DollarPort) ports.APIPorts {
return &Apap{client: client}
}
func (a Apap) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
uri := os.Getenv("APA")
log = log.With("scrapper", "apap")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
@ -95,10 +93,14 @@ func (a Apap) ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
_, err = a.Scrape(ctx, page, log)
histList, err := a.Scrape(ctx, page, log)
// here we execute db operations
if err != nil {
return err
}
err = a.client.NewHistory(histList[0])
if err != nil {
return err
}
return err
}

36
bcd/bcd.go → internal/adapters/crawler/bcd.go
View File

@ -1,4 +1,4 @@
package bcd
package crawler
import (
"context"
@ -7,23 +7,26 @@ import (
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/models"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
type bcd struct {
models.Institucion
client ports.DollarPort
}
var (
uri = os.Getenv("BCD")
)
func NewBCD(client ports.DollarPort) ports.APIPorts {
return &bcd{
client: client,
}
}
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func (b bcd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
log = log.With("scrapper", "bcd")
tout := 90000.00
uri := os.Getenv("BCD")
if _, err = page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
WaitUntil: playwright.WaitUntilStateLoad,
@ -51,10 +54,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
return nil, err
}
inst = &db.History{
inst := &domain.History{
Parser: "bcd",
Name: "banco central dominicano",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
inst.Compra = helpers.Normalize(compra)
@ -63,12 +66,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
if inst.Compra == 0 || inst.Venta == 0 {
return nil, fmt.Errorf("bcd: institution not parsed compra or venta cant be 0")
}
return inst, nil
return []*domain.History{inst}, nil
}
func ExecParser(
func (bc bcd) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -94,11 +96,15 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
inst, err := bc.Scrape(ctx, page, log)
if err != nil {
return err
}
err = bc.client.NewHistory(inst[0])
if err != nil {
return err
}
err = db.Inspect(*inst)
return err
}

34
bdr/bdr.go → internal/adapters/crawler/bdr.go
View File

@ -1,4 +1,4 @@
package bdr
package crawler
import (
"context"
@ -7,18 +7,25 @@ import (
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("BDR")
)
type bdr struct {
client ports.DollarPort
}
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func NewBDR(client ports.DollarPort) ports.APIPorts {
return &bdr{
client: client,
}
}
func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
log = log.With("scrapper", "bdr")
uri := os.Getenv("BDR")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
WaitUntil: playwright.WaitUntilStateLoad,
@ -47,10 +54,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
log.Error("parsing compra", "err", err)
return nil, err
}
inst = &db.History{
inst := &domain.History{
Name: "banreservas",
Parser: "brd",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
compra := helpers.Normalize(compraSTR)
@ -62,13 +69,12 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
if inst.Compra == 0 || inst.Venta == 0 {
return nil, fmt.Errorf("brd: institution not parsed")
}
return inst, nil
insts = append(insts, inst)
return insts, nil
}
func ExecParser(
func (bd bdr) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -94,11 +100,11 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
insts, err := bd.Scrape(ctx, page, log)
if err != nil {
return err
}
err = db.Inspect(*inst)
err = bd.client.NewHistory(insts[0])
return err
}

33
bhd/bhd.go → internal/adapters/crawler/bhd.go
View File

@ -1,4 +1,4 @@
package bhd
package crawler
import (
"context"
@ -8,19 +8,27 @@ import (
"strings"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("BHD")
)
type bhd struct {
client ports.DollarPort
}
func NewBHD(client ports.DollarPort) ports.APIPorts {
return &bhd{
client: client,
}
}
// Scrape
// needs a mobile User Agent
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
uri := os.Getenv("BHD")
log = log.With("scrapper", "bhd")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
@ -51,10 +59,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
return nil, err
}
inst = &db.History{
inst := &domain.History{
Name: "banco hipotecario dominicano",
Parser: "bhd",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
for _, it := range fieldGroup {
@ -101,12 +109,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
return nil, fmt.Errorf("bhd: institution not parsed: %v", inst)
}
//log.Info(fmt.Sprintf("%v", inst))
return inst, nil
return []*domain.History{inst}, nil
}
func ExecParser(
func (bh bhd) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -132,10 +139,10 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
inst, err := bh.Scrape(ctx, page, log)
if err != nil {
return err
}
err = db.Inspect(*inst)
bh.client.NewHistory(inst[0])
return err
}

33
bnc/bnc.go → internal/adapters/crawler/bnc.go
View File

@ -1,4 +1,4 @@
package bnc
package crawler
import (
"context"
@ -7,17 +7,25 @@ import (
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("BNC")
)
type bnc struct {
client ports.DollarPort
}
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func NewBNC(client ports.DollarPort) ports.APIPorts {
return &bnc{
client: client,
}
}
func (bn bnc) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
uri := os.Getenv("BNC")
log = log.With("scrapper", "bnc")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
@ -44,10 +52,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
log.Error("could not get venta string", "err", err)
return nil, err
}
inst = &db.History{
inst := &domain.History{
Name: "banesco",
Parser: "bnc",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
inst.Venta = helpers.Normalize(ventaSTR)
@ -56,12 +64,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
if inst.Compra == 0 || inst.Venta == 0 {
return nil, fmt.Errorf("bnc: institution not parsed: %v", inst)
}
return inst, nil
return []*domain.History{inst}, nil
}
func ExecParser(
func (bn bnc) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -87,11 +94,11 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
inst, err := bn.Scrape(ctx, page, log)
// here we execute db operations
if err != nil {
return err
}
err = db.Inspect(*inst)
bn.client.NewHistory(inst[0])
return err
}

35
bpd/bpd.go → internal/adapters/crawler/bpd.go
View File

@ -1,4 +1,4 @@
package bpd
package crawler
import (
"context"
@ -8,21 +8,29 @@ import (
"strconv"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("BPD")
)
type bpd struct {
client ports.DollarPort
}
func NewBPD(client ports.DollarPort) ports.APIPorts {
return &bpd{
client: client,
}
}
// Scrape
// needs a mobile User Agent
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func (bp bpd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
//start := time.Now()
uri := os.Getenv("BPD")
log = log.With("scrapper", "bpd")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
@ -54,10 +62,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
log.Error("compra value", "err", err)
return nil, err
}
inst = &db.History{
inst := &domain.History{
Name: "banco popular",
Parser: "bpd",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
compra, err := strconv.ParseFloat(compraSTR, 64)
if err != nil {
@ -75,7 +83,7 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
if inst.Compra == 0 || inst.Venta == 0 {
return nil, fmt.Errorf("bpd: institution not parsed")
}
return inst, nil
return []*domain.History{inst}, nil
}
func HoverTasas(page playwright.Page) {
@ -83,9 +91,8 @@ func HoverTasas(page playwright.Page) {
tasasMenu.Hover()
}
func ExecParser(
func (bp bpd) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -111,10 +118,10 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
inst, err := bp.Scrape(ctx, page, log)
if err != nil {
return err
}
err = db.Inspect(*inst)
bp.client.NewHistory(inst[0])
return err
}

20
internal/adapters/crawler/crawler.go
View File

@ -6,11 +6,27 @@ import (
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
)
func Selector(who string) (ports.APIPorts, error) {
func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) {
var parser ports.APIPorts
switch who {
case "apap":
parser = NewApap()
parser = NewApap(client)
case "bcd":
parser = NewBCD(client)
case "bdr":
parser = NewBDR(client)
case "bhd":
parser = NewBHD(client)
case "bnc":
parser = NewBNC(client)
case "bpd":
parser = NewBPD(client)
case "inf":
parser = NewINF(client)
case "scotia":
parser = NewScotia(client)
case "vimenca":
parser = NewVimenca(client)
default:
return nil, fmt.Errorf("not recognize who: " + who)
}

33
inf/general.go → internal/adapters/crawler/inf.go
View File

@ -1,4 +1,4 @@
package inf
package crawler
import (
"context"
@ -8,17 +8,25 @@ import (
"strings"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("GENERAL")
)
type inf struct {
client ports.DollarPort
}
func NewINF(client ports.DollarPort) ports.APIPorts {
return &inf{
client: client,
}
}
// Scrape
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*db.History, err error) {
func (in inf) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*domain.History, err error) {
uri := os.Getenv("GENERAL")
log = log.With("scrapper", "general")
tout := float64(120000)
if _, err := page.Goto(uri, playwright.PageGotoOptions{
@ -42,9 +50,9 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instLi
return nil, err
}
scotia := false // in this page there are 2 scotia one the change online the other is tha bank
instList = []*db.History{}
instList = []*domain.History{}
for _, entry := range entries {
inst := &db.History{
inst := &domain.History{
Parser: "inf",
}
title, _ := entry.Locator("span.nombre").TextContent()
@ -67,7 +75,7 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instLi
inst.Compra = helpers.Normalize(compra)
inst.Venta = helpers.Normalize(venta)
inst.Parsed = time.Now().UTC()
inst.Parsed = time.Now().Unix()
// if one of the inst has 0 on the sell/buy dont process it
if inst.Compra == 0 || inst.Venta == 0 {
log.Warn("skipping", "nombre", inst.Name, "compra", inst.Compra, "venta", inst.Venta)
@ -92,9 +100,8 @@ func getValue(place playwright.Locator) string {
}
// ExecParser
func ExecParser(
func (in inf) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) error {
t := true
@ -120,13 +127,13 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
instList, err := Scrape(ctx, page, log)
instList, err := in.Scrape(ctx, page, log)
if err != nil {
return err
}
for _, inst := range instList {
log.Info("processing", "name", inst.Name)
err = db.Inspect(*inst)
err = in.client.NewHistory(inst)
if err != nil {
log.Error(fmt.Sprintf("inspecting %s", inst.Name), "error", err)
}

35
scotia/scotia.go → internal/adapters/crawler/scotia.go
View File

@ -1,4 +1,4 @@
package scotia
package crawler
import (
"context"
@ -7,17 +7,25 @@ import (
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("SCOTIA")
)
type scotia struct {
client ports.DollarPort
}
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*db.History, err error) {
func NewScotia(client ports.DollarPort) ports.APIPorts {
return &scotia{
client: client,
}
}
func (sct scotia) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
tout := 120000.00
uri := os.Getenv("SCOTIA")
log = log.With("scrapper", "scotia")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
Timeout: &tout,
@ -55,10 +63,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts
log.Error("could not get venta string", "err", err)
return nil, err
}
instOnsite := &db.History{
instOnsite := &domain.History{
Name: "scotiabank",
Parser: "scotia",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
instOnsite.Venta = helpers.Normalize(ventaOnsiteSTR)
@ -79,10 +87,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts
return nil, err
}
instOnline := &db.History{
instOnline := &domain.History{
Name: "scotiabank cambio online",
Parser: "scotia",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
instOnline.Venta = helpers.Normalize(ventaOnlineSTR)
instOnline.Compra = helpers.Normalize(compraOnlineSTR)
@ -95,9 +103,8 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts
return insts, nil
}
func ExecParser(
func (sct scotia) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -123,13 +130,13 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
insts, err := Scrape(ctx, page, log)
insts, err := sct.Scrape(ctx, page, log)
// here we execute db operations
if err != nil {
return err
}
for _, inst := range insts {
err = db.Inspect(*inst)
sct.client.NewHistory(inst)
}
return err
}

37
vimenca/vimenca.go → internal/adapters/crawler/vimenca.go
View File

@ -1,4 +1,4 @@
package vimenca
package crawler
import (
"context"
@ -6,16 +6,24 @@ import (
"os"
"time"
"github.com/maximotejeda/us_dop_db/db"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"github.com/playwright-community/playwright-go"
)
var (
uri = os.Getenv("VIMENCA")
)
type vimenca struct {
client ports.DollarPort
}
func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) {
func NewVimenca(client ports.DollarPort) ports.APIPorts {
return &vimenca{
client: client,
}
}
func (v vimenca) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
uri := os.Getenv("VIMENCA")
tout := 120000.00
log = log.With("scrapper", "vimenca")
if _, err := page.Goto(uri, playwright.PageGotoOptions{
@ -45,21 +53,20 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *
log.Error("could not get venta string", "err", err)
return nil, err
}
inst = &db.History{
inst := &domain.History{
Name: "banco vimenca",
Parser: "vimenca",
Parsed: time.Now().UTC(),
Parsed: time.Now().Unix(),
}
inst.Venta = helpers.Normalize(ventaSTR)
inst.Compra = helpers.Normalize(compraSTR)
log.Info("institution", "value", inst)
return inst, nil
return []*domain.History{inst}, nil
}
func ExecParser(
func (v vimenca) ExecParser(
ctx context.Context,
db *db.DB,
browser *playwright.Browser,
log *slog.Logger) (err error) {
t := true
@ -85,13 +92,15 @@ func ExecParser(
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
defer page.Close()
defer cancel()
inst, err := Scrape(ctx, page, log)
inst, err := v.Scrape(ctx, page, log)
// here we execute db operations
if err != nil {
return err
}
err = db.Inspect(*inst)
err = v.client.NewHistory(inst[0])
if err != nil {
return err
}
return err
}

1
internal/adapters/dolar/.#dolar.go
View File

@ -1 +0,0 @@
maximo@debian-pc.9800:1713363571

26
internal/adapters/dolar/dolar.go
View File

@ -6,21 +6,27 @@ import (
"github.com/maximotejeda/msvc-proto/golang/dolar"
"github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
type Adapter struct {
dolar dolar.DollarClient
conn *grpc.ClientConn
}
func NewAdapter(dolarServiceURL string) (*Adapter, error) {
var opts []grpc.DialOption
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
conn, err := grpc.Dial(dolarServiceURL, opts...)
if err != nil {
return nil, err
}
defer conn.Close()
func NewAdapter(conn *grpc.ClientConn) (*Adapter, error) {
client := dolar.NewDollarClient(conn)
return &Adapter{dolar: client}, nil
return &Adapter{dolar: client, conn: conn}, nil
}
func (a *Adapter) NewHistory(history *domain.History) error {
_, err := a.dolar.NewHistory(context.Background(),
&dolar.AddDolarRequest{
Institution: &dolar.History{
Name: history.Name,
Compra: float32(history.Compra),
Venta: float32(history.Venta),
Parser: history.Parser,
Parsed: history.Parsed,
}})
return err
}

1
internal/application/core/api/.#api.go
View File

@ -1 +0,0 @@
maximo@debian-pc.9800:1713363571

25
internal/application/core/api/api.go
View File

@ -1,10 +1,10 @@
package api
import (
"log/slog"
"github.com/maximotejeda/us_dop_scrapper/config"
"context"
"github.com/maximotejeda/us_dop_scrapper/helpers"
"github.com/maximotejeda/us_dop_scrapper/internal/ports"
"log/slog"
)
type Application struct {
@ -12,15 +12,28 @@ type Application struct {
api ports.APIPorts
}
func NewApplication() *Application {
func NewApplication(crawler ports.APIPorts) *Application {
log := slog.Default()
log = log.With("application", "root")
return &Application{
log: log,
api: crawler,
}
}
func (a Application) Run() {
who := config.GetWho()
ctx := context.Background()
ch, ff, wk := helpers.CreateBrowser(a.log)
err := a.api.ExecParser(ctx, ch, a.log)
if err != nil {
a.log.Info("failed on frist browser", "browser", "chrome", "error", err)
err := a.api.ExecParser(ctx, ff, a.log)
if err != nil {
a.log.Error("failed on second browser", "browser", "firefox", "error", err)
err := a.api.ExecParser(ctx, wk, a.log)
if err != nil {
a.log.Error("tried all browsers error", "brwser", "webkit", "error", err)
}
}
}
}

1
internal/application/core/domain/.#domain.go
View File

@ -1 +0,0 @@
maximo@debian-pc.9800:1713363571

12
internal/application/core/domain/domain.go
View File

@ -1,10 +1,10 @@
package domain
type History struct {
ID int64 `json:""`
Name string `json:""`
Compra float64 `json:""`
Venta float64 `json:""`
Parser string `json:""`
Parsed int64 `json:""`
ID int64 `json:"id"`
Name string `json:"name"`
Compra float64 `json:"compra"`
Venta float64 `json:"venta"`
Parser string `json:"parser"`
Parsed int64 `json:"parsed"`
}

51
wait/wait.go
View File

@ -1,51 +0,0 @@
package wait
import (
"fmt"
"time"
)
// from monday to friday from 12:00 utc to 22:00 utc
// saturday from 12:00 utc to 18:00 utc
// sunday wont work
func WaitAmount(actualTime time.Time) (infoDuration time.Duration, longDuration time.Duration) {
var nextDayTime time.Time
// day of the week start on sunday=0
dayNumber := actualTime.UTC().Weekday()
year, month, day, hour := actualTime.UTC().Year(), actualTime.UTC().Month(), actualTime.UTC().Day(), actualTime.UTC().Hour()
nextDayTemplate := fmt.Sprintf("%d-%02d-%02d 12:05:00", year, month, day)
parsedDate, err := time.Parse(time.DateTime, nextDayTemplate)
if err != nil {
panic(err)
}
// in case of waiting to next day or weekend
switch dayNumber {
case 0:
nextDayTime = parsedDate.Add(24 * time.Hour)
return time.Until(nextDayTime), time.Until(nextDayTime)
case 6:
if hour > 16 {
nextDayTime = parsedDate.Add(48 * time.Hour)
return time.Until(nextDayTime), time.Until(nextDayTime)
} else {
info := time.Until(actualTime.UTC().Add(26 * time.Minute))
long := time.Until(actualTime.UTC().Add(59 * time.Minute))
return info, long
}
default:
nextDayTime = parsedDate.Add(24 * time.Hour)
switch {
case hour >= 22:
// next day wait
return time.Until(nextDayTime), time.Until(nextDayTime)
case hour < 12:
return time.Until(parsedDate), time.Until(parsedDate)
default:
info := time.Until(actualTime.UTC().Add(26 * time.Minute))
long := time.Until(actualTime.UTC().Add(59 * time.Minute))
return info, long
}
}
}

40
wait/wait_test.go
View File

@ -1,40 +0,0 @@
package wait
import (
"testing"
"time"
)
func TestWaitAmount(t *testing.T) {
type cases struct {
name string
initDate time.Time
infoAmount float64
longAmount float64
}
scenario := []cases{
{
name: "success/3h",
initDate: time.Now().Add(-1 * time.Hour),
infoAmount: 24,
longAmount: 24,
},
{
name: "success/now",
initDate: time.Now(),
infoAmount: 24,
longAmount: 24,
},
}
for _, tt := range scenario {
t.Run(tt.name, func(t *testing.T) {
info, long := WaitAmount(tt.initDate)
if info.Hours() == 0 {
t.Errorf("wanted: %f got: %f, time: %v", tt.infoAmount, info.Hours(), tt.initDate.UTC())
}
if long.Hours() == 0 {
t.Errorf("wanted: %f got: %f, time: %v", tt.longAmount, long.Hours(), tt.initDate.UTC())
}
})
}
}