diff --git a/apa/apa.go b/apa/apa.go deleted file mode 100644 index 5d81687..0000000 --- a/apa/apa.go +++ /dev/null @@ -1,97 +0,0 @@ -package apa - -import ( - "context" - "fmt" - "log/slog" - "os" - "time" - - "github.com/maximotejeda/us_dop_db/db" - "github.com/maximotejeda/us_dop_scrapper/helpers" - "github.com/playwright-community/playwright-go" -) - -var ( - uri = os.Getenv("APA") -) - -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { - tout := 120000.00 - log = log.With("scrapper", "apap") - if _, err := page.Goto(uri, playwright.PageGotoOptions{ - Timeout: &tout, - WaitUntil: playwright.WaitUntilStateLoad, - }); err != nil { - log.Error("could not get info", "error", err) - return nil, err - } - button := page.Locator("#exchangesRates") - button.WaitFor() - button.Click() - - compraLocator := page.Locator("#currency-buy-USD") - ventaLocator := page.Locator("#currency-sell-USD") - - compraSTR, err := compraLocator.TextContent() - if err != nil { - log.Error("could not get compra str", "err", err) - return nil, err - } - ventaSTR, err := ventaLocator.TextContent() - if err != nil { - log.Error("could not get venta string", "err", err) - return nil, err - } - inst = &db.History{ - Name: "asociacion popular de ahorros y prestamos", - Parser: "apap", - Parsed: time.Now().UTC(), - } - - inst.Venta = helpers.Normalize(ventaSTR) - inst.Compra = helpers.Normalize(compraSTR) - - if inst.Compra == 0 || inst.Venta == 0 { - return nil, fmt.Errorf("apa: institution not parsed: %v", inst) - } - log.Info("parsed", "value", inst) - return inst, nil -} - -func ExecParser( - ctx context.Context, - db *db.DB, - browser *playwright.Browser, - log *slog.Logger) (err error) { - t := true - ua := helpers.NewMobileUA() - b := *browser - page, err := b.NewPage(playwright.BrowserNewPageOptions{ - UserAgent: &ua, - // IsMobile: &t, - HasTouch: &t, - Viewport: &playwright.Size{ - Width: 412, - Height: 915, - }, - Screen: &playwright.Size{ - Width: 412, - Height: 915, - }, - }) - if err != nil { - log.Error("creating page", "error", err) - os.Exit(1) - } - ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) - defer page.Close() - defer cancel() - inst, err := Scrape(ctx, page, log) - // here we execute db operations - if err != nil { - return err - } - err = db.Inspect(*inst) - return err -} diff --git a/bin/crawler-linux-amd64-0.0.0_1 b/bin/crawler-linux-amd64-0.0.0_1 index d8844db..4d4a320 100755 Binary files a/bin/crawler-linux-amd64-0.0.0_1 and b/bin/crawler-linux-amd64-0.0.0_1 differ diff --git a/bin/crawler-linux-arm64-0.0.0_1 b/bin/crawler-linux-arm64-0.0.0_1 index e472c1d..fe09e82 100755 Binary files a/bin/crawler-linux-arm64-0.0.0_1 and b/bin/crawler-linux-arm64-0.0.0_1 differ diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index 8e83f35..b628eb1 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -1,79 +1,37 @@ package main import ( - "context" "log/slog" - "os" - "os/signal" - "syscall" - "time" - "github.com/maximotejeda/us_dop_db/db" - "github.com/maximotejeda/us_dop_scrapper/apa" - "github.com/maximotejeda/us_dop_scrapper/bcd" - "github.com/maximotejeda/us_dop_scrapper/bdr" - "github.com/maximotejeda/us_dop_scrapper/bhd" - "github.com/maximotejeda/us_dop_scrapper/bnc" - "github.com/maximotejeda/us_dop_scrapper/bpd" - "github.com/maximotejeda/us_dop_scrapper/helpers" - "github.com/maximotejeda/us_dop_scrapper/inf" - "github.com/maximotejeda/us_dop_scrapper/scotia" - "github.com/maximotejeda/us_dop_scrapper/vimenca" - "github.com/playwright-community/playwright-go" + "github.com/maximotejeda/us_dop_scrapper/config" + "github.com/maximotejeda/us_dop_scrapper/internal/adapters/crawler" + dl "github.com/maximotejeda/us_dop_scrapper/internal/adapters/dolar" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) func main() { - var err error - dbRoute := os.Getenv("DBURI") - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{})) - db := db.Dial(dbRoute, log) - // create a chrome and feed parsers - db.CreateTables() - chrome, firefox, webkit := helpers.CreateBrowser(log) - browserList := []*playwright.Browser{chrome, firefox, webkit} - longTick := time.NewTicker(time.Minute * 2) - defer longTick.Stop() - infoTick := time.NewTicker(time.Minute * 1) - defer infoTick.Stop() - ctx, cancel := context.WithCancel(context.Background()) - errN := map[string]int{ - "bcd": 0, - "bpd": 0, - "apap": 0, - "inf": 0, - } - - defer cancel() - who := os.Getenv("WHO") - - switch who { - case "bcd": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "bcd", bcd.ExecParser) - case "bpd": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "bpd", bpd.ExecParser) - case "apa": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "apa", apa.ExecParser) - case "brd": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "brd", bdr.ExecParser) - case "bhd": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "bhd", bhd.ExecParser) - case "bnc": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "bnc", bnc.ExecParser) - case "scotia": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "scotia", scotia.ExecParser) - case "vimenca": - err = helpers.ExecTask(ctx, db, browserList, log, errN, "vimenca", vimenca.ExecParser) - default: - err = helpers.ExecTask(ctx, db, browserList, log, errN, "inf", inf.ExecParser) - } + log := slog.Default() + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.Dial(config.GetDollarServiceURL(), opts...) if err != nil { - log.Info("task executed with errors", "name", who, "error", err) - os.Exit(1) - return + log.Error("creating gerpc conn", "error", err) + panic(err) } - log.Info("SUCCESS - task executed", "name", who) - os.Exit(0) + defer conn.Close() + dol, err := dl.NewAdapter(conn) + if err != nil { + log.Error("creating service adapter", "error", err) + panic(err) + } + crawler, err := crawler.Selector(config.GetWho(), dol) + if err != nil { + log.Error("selecting crawler adapter", "error", err) + panic(err) + } + app := api.NewApplication(crawler) + app.Run() } diff --git a/config/config.go b/config/config.go index d429248..2ddec80 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,10 @@ func GetWho() string { return getEnvValue("WHO") } +func GetDollarServiceURL() string { + return getEnvValue("DOLLAR_SERVICE_URL") +} + func getEnvValue(key string) string { if os.Getenv(key) == "" { panic("key not found " + key) diff --git a/db-old/db.go b/db-old/db.go deleted file mode 100644 index 4e1e32c..0000000 --- a/db-old/db.go +++ /dev/null @@ -1,423 +0,0 @@ -package db - -import ( - "database/sql" - _ "embed" - "encoding/json" - "errors" - "fmt" - "log/slog" - "strings" - "time" - - "github.com/maximotejeda/us_dop_scrapper/models" - "github.com/maximotejeda/us_dop_scrapper/pub" - _ "modernc.org/sqlite" -) - -//go:embed schema.sql -var schema string - -type DB struct { - *sql.DB - log *slog.Logger -} - -type change struct { - Before models.Institucion `json:"before"` - After models.Institucion `json:"after"` -} - -type Message struct { - Message string `json:"message"` - Data change `json:"data"` - Error error `json:"error"` -} - -type Institution struct { - ID int - Name string - ShortName string - Created time.Time -} - -// Dial -func Dial(path string, log *slog.Logger) *DB { - db, err := sql.Open("sqlite", path) - if err != nil { - fmt.Printf("opening database: %s", err.Error()) - panic("opening database") - } - if err := db.Ping(); err != nil { - fmt.Printf("pinging database: %s", err.Error()) - panic("pinging database") - } - return &DB{db, log} -} - -// Schema -func (db *DB) CreateTables() { - _, err := db.Exec(schema) - if err != nil { - panic(err) - } -} - -// Inspect -// Handle behavior of the changes -// Will report errors to a nats consumer -func (db *DB) Inspect(enter models.Institucion) error { - if db == nil { - return fmt.Errorf("nil or empty database") - } - pub, close := pub.Publisher() - defer close() - msg := Message{} - // Get last row added - - inst, err := db.GetLatest(enter.Parser, enter.Name) - // if no rows are found because of first enter a name - parser ? - if errors.Is(sql.ErrNoRows, err) { - db.log.Info("adding new item to table: ", "parse", enter.Parser, "name", enter.Name) - msg.Message = "add new institution" - msg.Data.After = enter - - data, err := json.Marshal(msg) - if err != nil { - db.log.Error("marshaling struct", "error", err) - } - - id, err := db.ADDInstitution(enter.Name) - if err != nil { - return err - } - defer pub("dolar-crawler", data) - return db.AddNew(enter, id) - } - - // check prices compra venta - if inst == nil { - db.log.Error("row is nil", "name", enter.Name, "parser", enter.Parser) - return fmt.Errorf("row is nil, not entering row") - } - if enter.Compra == inst.Compra && enter.Venta == inst.Venta { - return nil - } else { - // if one of them changes create a new row - db.log.Info("change registered, adding item", "parse", enter.Parser, "name", enter.Name, "compra enter", enter.Compra, "compra db", inst.Compra, "venta enter", enter.Venta, "venta db", inst.Venta) - - msg.Message = "change registered" - msg.Data.After = enter - msg.Data.Before = *inst - - data, err := json.Marshal(msg) - if err != nil { - db.log.Error("marshaling struct", "error", err) - } - ins, err := db.GETInstitution(enter.Name) - if err != nil { - return err - } - - defer pub("dolar-crawler", data) - return db.AddNew(enter, int64(ins.ID)) - } -} - -// GetLatest -// returns the latest row in a specific parser and name -// we are using DateTime in DB and date.Datetime in go -func (db *DB) GetLatest(parser string, name string) (inst *models.Institucion, err error) { - var parsed string - inst = &models.Institucion{} - stmtt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE d.parser = ? AND i.name = ? ORDER BY d.parsed DESC LIMIT 1;") - if err != nil { - db.log.Error("preparing stmtt", "error", err.Error()) - return nil, err - } - defer stmtt.Close() - - if err := stmtt.QueryRow(parser, name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { - db.log.Error("getting latest", "error", err.Error(), "parser", parser, "name", name) - return nil, err - } - - inst.Parsed, err = time.Parse(time.DateTime, parsed) - if err != nil { - //db.log.Error("parsed", "error", err.Error()) - return nil, err - } - return inst, nil -} - -// AddNew -// Add a new row in the dolar table -// Will send to nats changes on prices -func (db *DB) AddNew(row models.Institucion, id int64) error { - stmt, err := db.Prepare("INSERT INTO dolars (name_id, compra, venta, parser, parsed) VALUES(?,?,?,?,?);") - if err != nil { - return err - } - defer stmt.Close() - parsed := row.Parsed.Format(time.DateTime) - _, err = stmt.Exec(&id, &row.Compra, &row.Venta, &row.Parser, &parsed) - if err != nil { - return err - } - return nil -} - -func (db *DB) ADDInstitution(name string) (id int64, err error) { - stmt, err := db.Prepare("INSERT INTO institutions (name, short_name, created) VALUES(?,?,?);") - if err != nil { - return 0, err - } - defer stmt.Close() - parsed := time.Now().Format(time.DateTime) - short := shortner(name) - res, err := stmt.Exec(&name, short, &parsed) - if err != nil { - return 0, err - } - id, err = res.LastInsertId() - if err != nil { - return 0, err - } - return id, nil - -} -func (db *DB) GETInstitution(name string) (inst *Institution, err error) { - institution := Institution{} - stmtt, err := db.Prepare("SELECT id, name, short_name FROM institutions WHERE name = ?") - if err != nil { - db.log.Error("preparing stmt", "error", err.Error()) - return nil, err - } - defer stmtt.Close() - if err := stmtt.QueryRow(name).Scan(&institution.ID, &institution.Name, &institution.ShortName); err != nil { - db.log.Error("getting institution", "error", err.Error(), "short name", institution.ShortName, "name", name) - return nil, err - } - return inst, err -} - -func (db *DB) GetAll() ([]string, error) { - stmt, err := db.Prepare("SELECT i.name FROM institutions AS i;") - if err != nil { - db.log.Error("[db-GetAll]", "error", err) - return nil, err - } - rows, err := stmt.Query() - if err != nil { - db.log.Error("[db-GetAll-stmt]", "error", err) - return nil, err - } - defer rows.Close() - insts := []string{} - for rows.Next() { - inst := "" - - if err = rows.Scan(&inst); err != nil { - return nil, err - } - if inst == "" { - continue - } - insts = append(insts, inst) - } - if err := rows.Err(); err != nil { - return insts, err - } - return insts, nil - -} -func (db *DB) GetBancos() ([]string, error) { - stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%ban%' OR i.name LIKE '%scoti%'") - if err != nil { - db.log.Error("[inst-GetAll]", "error", err) - return nil, err - } - rows, err := stmt.Query() - if err != nil { - db.log.Error("[inst-GetAll-stmt]", "error", err) - return nil, err - } - defer rows.Close() - insts := []string{} - for rows.Next() { - inst := "" - - if err = rows.Scan(&inst); err != nil { - return nil, err - } - if inst == "" { - continue - } - insts = append(insts, inst) - } - if err := rows.Err(); err != nil { - return insts, err - } - return insts, nil - -} -func (db *DB) GetCajas() ([]string, error) { - stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%asociacion%'") - if err != nil { - db.log.Error("[inst-GetAll]", "error", err) - return nil, err - } - rows, err := stmt.Query() - if err != nil { - db.log.Error("[inst-GetAll-stmt]", "error", err) - return nil, err - } - defer rows.Close() - insts := []string{} - for rows.Next() { - inst := "" - - if err = rows.Scan(&inst); err != nil { - return nil, err - } - if inst == "" { - continue - } - insts = append(insts, inst) - } - if err := rows.Err(); err != nil { - return insts, err - } - return insts, nil - -} - -func (db *DB) GetAgentes() ([]string, error) { - stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name NOT LIKE '%ban%' AND i.name NOT LIKE '%scoti%' AND i.name NOT LIKE '%asociacion%'") - if err != nil { - db.log.Error("[inst-GetAll]", "error", err) - return nil, err - } - rows, err := stmt.Query() - if err != nil { - db.log.Error("[inst-GetAll-stmt]", "error", err) - return nil, err - } - defer rows.Close() - insts := []string{} - for rows.Next() { - inst := "" - - if err = rows.Scan(&inst); err != nil { - return nil, err - } - if inst == "" { - continue - } - insts = append(insts, inst) - } - if err := rows.Err(); err != nil { - return insts, err - } - return insts, nil - -} - -func (db *DB) GetLastPrice(name string) (inst *models.Institucion, err error) { - var parsed string - inst = &models.Institucion{} - stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? ORDER BY parsed DESC LIMIT 1;") - if err != nil { - db.log.Error("preparing", "error", err.Error()) - return nil, err - } - defer stmt.Close() - - if err := stmt.QueryRow(name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { - db.log.Error("getting last price", "error", err.Error(), "name", name) - return nil, err - } - - inst.Parsed, err = time.Parse(time.DateTime, parsed) - if err != nil { - //db.log.Error("parsed", "error", err.Error()) - return nil, err - } - return inst, nil - -} -func (db *DB) GetChangeSince(name string, duration time.Duration) (insts []*models.Institucion, err error) { - date := time.Now().Add(-duration).Format(time.DateTime) - stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? AND parsed > ? ORDER BY parsed DESC;") - if err != nil { - db.log.Error("[GetChangeSince] preparing", "error", err.Error()) - return nil, err - } - defer stmt.Close() - rows, err := stmt.Query(name, date) - if err != nil { - db.log.Error("[GetChangeSince] preparing", "error", err.Error()) - return nil, err - } - defer rows.Close() - for rows.Next() { - inst := models.Institucion{} - parsed := "" - if err := rows.Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { - db.log.Error("[GetChangeSince] scanning", "error", err) - return nil, err - } - inst.Parsed, err = time.Parse(time.DateTime, parsed) - if err != nil { - //db.log.Error("parsed", "error", err.Error()) - continue - } - insts = append(insts, &inst) - } - return insts, nil -} - -func shortner(name string) string { - if name == "" { - return "" - } - switch strings.ToLower(name) { - case "banco popular": - return "bpd" - case "banreservas": - return "brd" - case "banco central dominicano": - return "bcd" - case "banco hipotecario dominicano": - return "bhd" - case "asociacion popular de ahorros y prestamos": - return "apap" - case "asociacion cibao de ahorros y prestamos": - return "acap" - case "asociacion la nacional de ahorros y prestamos": - return "alnap" - case "asociacion peravia de ahorros y prestamos": - return "apeap" - case "banco santa cruz": - return "bsc" - case "imbert y balbuena": - return "imb" - case "banco activo dominicana": - return "bacd" - case "scotiabank cambio online": - return "scline" - case "banco lopez de haro": - return "blh" - } - nameList := strings.Split(name, " ") - switch len(nameList) { - case 1: - return nameList[0] - case 2: - return string(nameList[0][0]) + nameList[1][0:2] - case 3: - return string(nameList[0][0] + nameList[1][0] + nameList[2][0]) - default: - return "n/a" - } -} diff --git a/db-old/schema.sql b/db-old/schema.sql deleted file mode 100644 index f2fba03..0000000 --- a/db-old/schema.sql +++ /dev/null @@ -1,18 +0,0 @@ -PRAGMA foreign_keys = ON; - -CREATE TABLE IF NOT EXISTS 'dolars' ( - id INTEGER PRIMARY KEY, - name_id INTEGER NOT NULL, - compra REAL NOT NULL, - venta REAL NOT NULL, - parser TEXT NOT NULL, - parsed TEXT NOT NULL, - FOREIGN KEY(name_id) REFERENCES institutions(id) ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS 'institutions' ( - id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - short_name TEXT NOT NULL, - created TEXT NOT NULL -); diff --git a/helpers/.#helpers.go b/helpers/.#helpers.go deleted file mode 120000 index feda370..0000000 --- a/helpers/.#helpers.go +++ /dev/null @@ -1 +0,0 @@ -maximo@debian-pc.9800:1713363571 \ No newline at end of file diff --git a/helpers/helpers.go b/helpers/helpers.go index 4e15c21..56a6202 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -162,7 +162,6 @@ func CreateBrowser(log *slog.Logger) (chrome *playwright.Browser, firefox *playw // ExecTask func ExecTask( ctx context.Context, - dbi *db.DB, browser []*playwright.Browser, log *slog.Logger, diff --git a/inf/inf_test.go b/inf/inf_test.go deleted file mode 100644 index 8e3d714..0000000 --- a/inf/inf_test.go +++ /dev/null @@ -1 +0,0 @@ -package inf diff --git a/internal/adapters/crawler/apap.go b/internal/adapters/crawler/apap.go index d2f2307..57b3e2b 100644 --- a/internal/adapters/crawler/apap.go +++ b/internal/adapters/crawler/apap.go @@ -13,19 +13,17 @@ import ( "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("APA") -) - type Apap struct { + client ports.DollarPort } -func NewApap() ports.APIPorts { - return &Apap{} +func NewApap(client ports.DollarPort) ports.APIPorts { + return &Apap{client: client} } func (a Apap) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 + uri := os.Getenv("APA") log = log.With("scrapper", "apap") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, @@ -95,10 +93,14 @@ func (a Apap) ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - _, err = a.Scrape(ctx, page, log) + histList, err := a.Scrape(ctx, page, log) // here we execute db operations if err != nil { return err } + err = a.client.NewHistory(histList[0]) + if err != nil { + return err + } return err } diff --git a/bcd/bcd.go b/internal/adapters/crawler/bcd.go similarity index 73% rename from bcd/bcd.go rename to internal/adapters/crawler/bcd.go index 903c457..bee81bc 100644 --- a/bcd/bcd.go +++ b/internal/adapters/crawler/bcd.go @@ -1,4 +1,4 @@ -package bcd +package crawler import ( "context" @@ -7,23 +7,26 @@ import ( "os" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" - "github.com/maximotejeda/us_dop_scrapper/models" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) type bcd struct { - models.Institucion + client ports.DollarPort } -var ( - uri = os.Getenv("BCD") -) +func NewBCD(client ports.DollarPort) ports.APIPorts { + return &bcd{ + client: client, + } +} -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func (b bcd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { log = log.With("scrapper", "bcd") tout := 90000.00 + uri := os.Getenv("BCD") if _, err = page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, WaitUntil: playwright.WaitUntilStateLoad, @@ -51,10 +54,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * return nil, err } - inst = &db.History{ + inst := &domain.History{ Parser: "bcd", Name: "banco central dominicano", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } inst.Compra = helpers.Normalize(compra) @@ -63,12 +66,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * if inst.Compra == 0 || inst.Venta == 0 { return nil, fmt.Errorf("bcd: institution not parsed compra or venta cant be 0") } - return inst, nil + return []*domain.History{inst}, nil } -func ExecParser( +func (bc bcd) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -94,11 +96,15 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + inst, err := bc.Scrape(ctx, page, log) + if err != nil { + return err + } + err = bc.client.NewHistory(inst[0]) + if err != nil { return err } - err = db.Inspect(*inst) return err } diff --git a/bdr/bdr.go b/internal/adapters/crawler/bdr.go similarity index 74% rename from bdr/bdr.go rename to internal/adapters/crawler/bdr.go index afdc21a..5ff49c2 100644 --- a/bdr/bdr.go +++ b/internal/adapters/crawler/bdr.go @@ -1,4 +1,4 @@ -package bdr +package crawler import ( "context" @@ -7,18 +7,25 @@ import ( "os" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("BDR") -) +type bdr struct { + client ports.DollarPort +} -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func NewBDR(client ports.DollarPort) ports.APIPorts { + return &bdr{ + client: client, + } +} +func (bd bdr) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 log = log.With("scrapper", "bdr") + uri := os.Getenv("BDR") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, WaitUntil: playwright.WaitUntilStateLoad, @@ -47,10 +54,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * log.Error("parsing compra", "err", err) return nil, err } - inst = &db.History{ + inst := &domain.History{ Name: "banreservas", Parser: "brd", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } compra := helpers.Normalize(compraSTR) @@ -62,13 +69,12 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * if inst.Compra == 0 || inst.Venta == 0 { return nil, fmt.Errorf("brd: institution not parsed") } - - return inst, nil + insts = append(insts, inst) + return insts, nil } -func ExecParser( +func (bd bdr) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -94,11 +100,11 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + insts, err := bd.Scrape(ctx, page, log) if err != nil { return err } - err = db.Inspect(*inst) + err = bd.client.NewHistory(insts[0]) return err } diff --git a/bhd/bhd.go b/internal/adapters/crawler/bhd.go similarity index 82% rename from bhd/bhd.go rename to internal/adapters/crawler/bhd.go index be7ae35..733ed45 100644 --- a/bhd/bhd.go +++ b/internal/adapters/crawler/bhd.go @@ -1,4 +1,4 @@ -package bhd +package crawler import ( "context" @@ -8,19 +8,27 @@ import ( "strings" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("BHD") -) +type bhd struct { + client ports.DollarPort +} + +func NewBHD(client ports.DollarPort) ports.APIPorts { + return &bhd{ + client: client, + } +} // Scrape // needs a mobile User Agent -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 + uri := os.Getenv("BHD") log = log.With("scrapper", "bhd") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, @@ -51,10 +59,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * return nil, err } - inst = &db.History{ + inst := &domain.History{ Name: "banco hipotecario dominicano", Parser: "bhd", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } for _, it := range fieldGroup { @@ -101,12 +109,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * return nil, fmt.Errorf("bhd: institution not parsed: %v", inst) } //log.Info(fmt.Sprintf("%v", inst)) - return inst, nil + return []*domain.History{inst}, nil } -func ExecParser( +func (bh bhd) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -132,10 +139,10 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + inst, err := bh.Scrape(ctx, page, log) if err != nil { return err } - err = db.Inspect(*inst) + bh.client.NewHistory(inst[0]) return err } diff --git a/bnc/bnc.go b/internal/adapters/crawler/bnc.go similarity index 74% rename from bnc/bnc.go rename to internal/adapters/crawler/bnc.go index d9735cb..4aca442 100644 --- a/bnc/bnc.go +++ b/internal/adapters/crawler/bnc.go @@ -1,4 +1,4 @@ -package bnc +package crawler import ( "context" @@ -7,17 +7,25 @@ import ( "os" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("BNC") -) +type bnc struct { + client ports.DollarPort +} -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func NewBNC(client ports.DollarPort) ports.APIPorts { + return &bnc{ + client: client, + } +} + +func (bn bnc) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 + uri := os.Getenv("BNC") log = log.With("scrapper", "bnc") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, @@ -44,10 +52,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * log.Error("could not get venta string", "err", err) return nil, err } - inst = &db.History{ + inst := &domain.History{ Name: "banesco", Parser: "bnc", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } inst.Venta = helpers.Normalize(ventaSTR) @@ -56,12 +64,11 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * if inst.Compra == 0 || inst.Venta == 0 { return nil, fmt.Errorf("bnc: institution not parsed: %v", inst) } - return inst, nil + return []*domain.History{inst}, nil } -func ExecParser( +func (bn bnc) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -87,11 +94,11 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + inst, err := bn.Scrape(ctx, page, log) // here we execute db operations if err != nil { return err } - err = db.Inspect(*inst) + bn.client.NewHistory(inst[0]) return err } diff --git a/bpd/bpd.go b/internal/adapters/crawler/bpd.go similarity index 79% rename from bpd/bpd.go rename to internal/adapters/crawler/bpd.go index bf68654..d188e48 100644 --- a/bpd/bpd.go +++ b/internal/adapters/crawler/bpd.go @@ -1,4 +1,4 @@ -package bpd +package crawler import ( "context" @@ -8,21 +8,29 @@ import ( "strconv" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("BPD") -) +type bpd struct { + client ports.DollarPort +} + +func NewBPD(client ports.DollarPort) ports.APIPorts { + return &bpd{ + client: client, + } +} // Scrape // needs a mobile User Agent -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func (bp bpd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 - //start := time.Now() + + uri := os.Getenv("BPD") log = log.With("scrapper", "bpd") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, @@ -54,10 +62,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * log.Error("compra value", "err", err) return nil, err } - inst = &db.History{ + inst := &domain.History{ Name: "banco popular", Parser: "bpd", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } compra, err := strconv.ParseFloat(compraSTR, 64) if err != nil { @@ -75,7 +83,7 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * if inst.Compra == 0 || inst.Venta == 0 { return nil, fmt.Errorf("bpd: institution not parsed") } - return inst, nil + return []*domain.History{inst}, nil } func HoverTasas(page playwright.Page) { @@ -83,9 +91,8 @@ func HoverTasas(page playwright.Page) { tasasMenu.Hover() } -func ExecParser( +func (bp bpd) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -111,10 +118,10 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + inst, err := bp.Scrape(ctx, page, log) if err != nil { return err } - err = db.Inspect(*inst) + bp.client.NewHistory(inst[0]) return err } diff --git a/internal/adapters/crawler/crawler.go b/internal/adapters/crawler/crawler.go index 06030e8..2f11783 100644 --- a/internal/adapters/crawler/crawler.go +++ b/internal/adapters/crawler/crawler.go @@ -6,11 +6,27 @@ import ( "github.com/maximotejeda/us_dop_scrapper/internal/ports" ) -func Selector(who string) (ports.APIPorts, error) { +func Selector(who string, client ports.DollarPort) (ports.APIPorts, error) { var parser ports.APIPorts switch who { case "apap": - parser = NewApap() + parser = NewApap(client) + case "bcd": + parser = NewBCD(client) + case "bdr": + parser = NewBDR(client) + case "bhd": + parser = NewBHD(client) + case "bnc": + parser = NewBNC(client) + case "bpd": + parser = NewBPD(client) + case "inf": + parser = NewINF(client) + case "scotia": + parser = NewScotia(client) + case "vimenca": + parser = NewVimenca(client) default: return nil, fmt.Errorf("not recognize who: " + who) } diff --git a/inf/general.go b/internal/adapters/crawler/inf.go similarity index 81% rename from inf/general.go rename to internal/adapters/crawler/inf.go index 3fea88f..403b38c 100644 --- a/inf/general.go +++ b/internal/adapters/crawler/inf.go @@ -1,4 +1,4 @@ -package inf +package crawler import ( "context" @@ -8,17 +8,25 @@ import ( "strings" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("GENERAL") -) +type inf struct { + client ports.DollarPort +} + +func NewINF(client ports.DollarPort) ports.APIPorts { + return &inf{ + client: client, + } +} // Scrape -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*db.History, err error) { +func (in inf) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*domain.History, err error) { + uri := os.Getenv("GENERAL") log = log.With("scrapper", "general") tout := float64(120000) if _, err := page.Goto(uri, playwright.PageGotoOptions{ @@ -42,9 +50,9 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instLi return nil, err } scotia := false // in this page there are 2 scotia one the change online the other is tha bank - instList = []*db.History{} + instList = []*domain.History{} for _, entry := range entries { - inst := &db.History{ + inst := &domain.History{ Parser: "inf", } title, _ := entry.Locator("span.nombre").TextContent() @@ -67,7 +75,7 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instLi inst.Compra = helpers.Normalize(compra) inst.Venta = helpers.Normalize(venta) - inst.Parsed = time.Now().UTC() + inst.Parsed = time.Now().Unix() // if one of the inst has 0 on the sell/buy dont process it if inst.Compra == 0 || inst.Venta == 0 { log.Warn("skipping", "nombre", inst.Name, "compra", inst.Compra, "venta", inst.Venta) @@ -92,9 +100,8 @@ func getValue(place playwright.Locator) string { } // ExecParser -func ExecParser( +func (in inf) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) error { t := true @@ -120,13 +127,13 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - instList, err := Scrape(ctx, page, log) + instList, err := in.Scrape(ctx, page, log) if err != nil { return err } for _, inst := range instList { log.Info("processing", "name", inst.Name) - err = db.Inspect(*inst) + err = in.client.NewHistory(inst) if err != nil { log.Error(fmt.Sprintf("inspecting %s", inst.Name), "error", err) } diff --git a/scotia/scotia.go b/internal/adapters/crawler/scotia.go similarity index 81% rename from scotia/scotia.go rename to internal/adapters/crawler/scotia.go index 70be4ee..193d0d2 100644 --- a/scotia/scotia.go +++ b/internal/adapters/crawler/scotia.go @@ -1,4 +1,4 @@ -package scotia +package crawler import ( "context" @@ -7,17 +7,25 @@ import ( "os" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("SCOTIA") -) +type scotia struct { + client ports.DollarPort +} -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*db.History, err error) { +func NewScotia(client ports.DollarPort) ports.APIPorts { + return &scotia{ + client: client, + } +} + +func (sct scotia) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { tout := 120000.00 + uri := os.Getenv("SCOTIA") log = log.With("scrapper", "scotia") if _, err := page.Goto(uri, playwright.PageGotoOptions{ Timeout: &tout, @@ -55,10 +63,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts log.Error("could not get venta string", "err", err) return nil, err } - instOnsite := &db.History{ + instOnsite := &domain.History{ Name: "scotiabank", Parser: "scotia", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } instOnsite.Venta = helpers.Normalize(ventaOnsiteSTR) @@ -79,10 +87,10 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts return nil, err } - instOnline := &db.History{ + instOnline := &domain.History{ Name: "scotiabank cambio online", Parser: "scotia", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } instOnline.Venta = helpers.Normalize(ventaOnlineSTR) instOnline.Compra = helpers.Normalize(compraOnlineSTR) @@ -95,9 +103,8 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts return insts, nil } -func ExecParser( +func (sct scotia) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -123,13 +130,13 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - insts, err := Scrape(ctx, page, log) + insts, err := sct.Scrape(ctx, page, log) // here we execute db operations if err != nil { return err } for _, inst := range insts { - err = db.Inspect(*inst) + sct.client.NewHistory(inst) } return err } diff --git a/vimenca/vimenca.go b/internal/adapters/crawler/vimenca.go similarity index 71% rename from vimenca/vimenca.go rename to internal/adapters/crawler/vimenca.go index 4fc7f3e..5c3c6f3 100644 --- a/vimenca/vimenca.go +++ b/internal/adapters/crawler/vimenca.go @@ -1,4 +1,4 @@ -package vimenca +package crawler import ( "context" @@ -6,16 +6,24 @@ import ( "os" "time" - "github.com/maximotejeda/us_dop_db/db" "github.com/maximotejeda/us_dop_scrapper/helpers" + "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" + "github.com/maximotejeda/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) -var ( - uri = os.Getenv("VIMENCA") -) +type vimenca struct { + client ports.DollarPort +} -func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst *db.History, err error) { +func NewVimenca(client ports.DollarPort) ports.APIPorts { + return &vimenca{ + client: client, + } +} + +func (v vimenca) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) { + uri := os.Getenv("VIMENCA") tout := 120000.00 log = log.With("scrapper", "vimenca") if _, err := page.Goto(uri, playwright.PageGotoOptions{ @@ -45,21 +53,20 @@ func Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (inst * log.Error("could not get venta string", "err", err) return nil, err } - inst = &db.History{ + inst := &domain.History{ Name: "banco vimenca", Parser: "vimenca", - Parsed: time.Now().UTC(), + Parsed: time.Now().Unix(), } inst.Venta = helpers.Normalize(ventaSTR) inst.Compra = helpers.Normalize(compraSTR) log.Info("institution", "value", inst) - return inst, nil + return []*domain.History{inst}, nil } -func ExecParser( +func (v vimenca) ExecParser( ctx context.Context, - db *db.DB, browser *playwright.Browser, log *slog.Logger) (err error) { t := true @@ -85,13 +92,15 @@ func ExecParser( ctx, cancel := context.WithTimeout(ctx, 6*time.Minute) defer page.Close() defer cancel() - inst, err := Scrape(ctx, page, log) + inst, err := v.Scrape(ctx, page, log) // here we execute db operations if err != nil { return err } - - err = db.Inspect(*inst) + err = v.client.NewHistory(inst[0]) + if err != nil { + return err + } return err } diff --git a/internal/adapters/dolar/.#dolar.go b/internal/adapters/dolar/.#dolar.go deleted file mode 120000 index feda370..0000000 --- a/internal/adapters/dolar/.#dolar.go +++ /dev/null @@ -1 +0,0 @@ -maximo@debian-pc.9800:1713363571 \ No newline at end of file diff --git a/internal/adapters/dolar/dolar.go b/internal/adapters/dolar/dolar.go index 13db334..3080ed3 100644 --- a/internal/adapters/dolar/dolar.go +++ b/internal/adapters/dolar/dolar.go @@ -6,21 +6,27 @@ import ( "github.com/maximotejeda/msvc-proto/golang/dolar" "github.com/maximotejeda/us_dop_scrapper/internal/application/core/domain" "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) type Adapter struct { dolar dolar.DollarClient + conn *grpc.ClientConn } -func NewAdapter(dolarServiceURL string) (*Adapter, error) { - var opts []grpc.DialOption - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - conn, err := grpc.Dial(dolarServiceURL, opts...) - if err != nil { - return nil, err - } - defer conn.Close() +func NewAdapter(conn *grpc.ClientConn) (*Adapter, error) { client := dolar.NewDollarClient(conn) - return &Adapter{dolar: client}, nil + return &Adapter{dolar: client, conn: conn}, nil +} + +func (a *Adapter) NewHistory(history *domain.History) error { + _, err := a.dolar.NewHistory(context.Background(), + &dolar.AddDolarRequest{ + Institution: &dolar.History{ + Name: history.Name, + Compra: float32(history.Compra), + Venta: float32(history.Venta), + Parser: history.Parser, + Parsed: history.Parsed, + }}) + return err } diff --git a/internal/application/core/api/.#api.go b/internal/application/core/api/.#api.go deleted file mode 120000 index feda370..0000000 --- a/internal/application/core/api/.#api.go +++ /dev/null @@ -1 +0,0 @@ -maximo@debian-pc.9800:1713363571 \ No newline at end of file diff --git a/internal/application/core/api/api.go b/internal/application/core/api/api.go index 0c6af10..ef5004a 100644 --- a/internal/application/core/api/api.go +++ b/internal/application/core/api/api.go @@ -1,10 +1,10 @@ package api import ( - "log/slog" - - "github.com/maximotejeda/us_dop_scrapper/config" + "context" + "github.com/maximotejeda/us_dop_scrapper/helpers" "github.com/maximotejeda/us_dop_scrapper/internal/ports" + "log/slog" ) type Application struct { @@ -12,15 +12,28 @@ type Application struct { api ports.APIPorts } -func NewApplication() *Application { +func NewApplication(crawler ports.APIPorts) *Application { log := slog.Default() log = log.With("application", "root") return &Application{ log: log, + api: crawler, } } func (a Application) Run() { - who := config.GetWho() - + ctx := context.Background() + ch, ff, wk := helpers.CreateBrowser(a.log) + err := a.api.ExecParser(ctx, ch, a.log) + if err != nil { + a.log.Info("failed on frist browser", "browser", "chrome", "error", err) + err := a.api.ExecParser(ctx, ff, a.log) + if err != nil { + a.log.Error("failed on second browser", "browser", "firefox", "error", err) + err := a.api.ExecParser(ctx, wk, a.log) + if err != nil { + a.log.Error("tried all browsers error", "brwser", "webkit", "error", err) + } + } + } } diff --git a/internal/application/core/domain/.#domain.go b/internal/application/core/domain/.#domain.go deleted file mode 120000 index feda370..0000000 --- a/internal/application/core/domain/.#domain.go +++ /dev/null @@ -1 +0,0 @@ -maximo@debian-pc.9800:1713363571 \ No newline at end of file diff --git a/internal/application/core/domain/domain.go b/internal/application/core/domain/domain.go index a34dc46..f0db12d 100644 --- a/internal/application/core/domain/domain.go +++ b/internal/application/core/domain/domain.go @@ -1,10 +1,10 @@ package domain type History struct { - ID int64 `json:""` - Name string `json:""` - Compra float64 `json:""` - Venta float64 `json:""` - Parser string `json:""` - Parsed int64 `json:""` + ID int64 `json:"id"` + Name string `json:"name"` + Compra float64 `json:"compra"` + Venta float64 `json:"venta"` + Parser string `json:"parser"` + Parsed int64 `json:"parsed"` } diff --git a/wait/wait.go b/wait/wait.go deleted file mode 100644 index 8b52cec..0000000 --- a/wait/wait.go +++ /dev/null @@ -1,51 +0,0 @@ -package wait - -import ( - "fmt" - "time" -) - -// from monday to friday from 12:00 utc to 22:00 utc -// saturday from 12:00 utc to 18:00 utc -// sunday wont work -func WaitAmount(actualTime time.Time) (infoDuration time.Duration, longDuration time.Duration) { - var nextDayTime time.Time - - // day of the week start on sunday=0 - dayNumber := actualTime.UTC().Weekday() - year, month, day, hour := actualTime.UTC().Year(), actualTime.UTC().Month(), actualTime.UTC().Day(), actualTime.UTC().Hour() - nextDayTemplate := fmt.Sprintf("%d-%02d-%02d 12:05:00", year, month, day) - parsedDate, err := time.Parse(time.DateTime, nextDayTemplate) - if err != nil { - panic(err) - } - // in case of waiting to next day or weekend - switch dayNumber { - case 0: - nextDayTime = parsedDate.Add(24 * time.Hour) - return time.Until(nextDayTime), time.Until(nextDayTime) - case 6: - if hour > 16 { - nextDayTime = parsedDate.Add(48 * time.Hour) - return time.Until(nextDayTime), time.Until(nextDayTime) - } else { - info := time.Until(actualTime.UTC().Add(26 * time.Minute)) - long := time.Until(actualTime.UTC().Add(59 * time.Minute)) - return info, long - - } - default: - nextDayTime = parsedDate.Add(24 * time.Hour) - switch { - case hour >= 22: - // next day wait - return time.Until(nextDayTime), time.Until(nextDayTime) - case hour < 12: - return time.Until(parsedDate), time.Until(parsedDate) - default: - info := time.Until(actualTime.UTC().Add(26 * time.Minute)) - long := time.Until(actualTime.UTC().Add(59 * time.Minute)) - return info, long - } - } -} diff --git a/wait/wait_test.go b/wait/wait_test.go deleted file mode 100644 index a9b6619..0000000 --- a/wait/wait_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package wait - -import ( - "testing" - "time" -) - -func TestWaitAmount(t *testing.T) { - type cases struct { - name string - initDate time.Time - infoAmount float64 - longAmount float64 - } - scenario := []cases{ - { - name: "success/3h", - initDate: time.Now().Add(-1 * time.Hour), - infoAmount: 24, - longAmount: 24, - }, - { - name: "success/now", - initDate: time.Now(), - infoAmount: 24, - longAmount: 24, - }, - } - for _, tt := range scenario { - t.Run(tt.name, func(t *testing.T) { - info, long := WaitAmount(tt.initDate) - if info.Hours() == 0 { - t.Errorf("wanted: %f got: %f, time: %v", tt.infoAmount, info.Hours(), tt.initDate.UTC()) - } - if long.Hours() == 0 { - t.Errorf("wanted: %f got: %f, time: %v", tt.longAmount, long.Hours(), tt.initDate.UTC()) - } - }) - } -}