maximo tejeda
98dce2d47c
Some checks failed
dev test / test (push) Successful in 1m20s
dev test / vulnCheck (push) Successful in 32s
dev test / Ci-Lint (push) Successful in 28s
${{ github.actor }} executed Build Push Prod / build (push) Failing after 37s
${{ github.actor }} executed Build Push Prod / deploy (push) Has been skipped
105 lines
2.6 KiB
Go
105 lines
2.6 KiB
Go
package crawler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/config"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/helpers"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/internal/application/core/domain"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/internal/ports"
|
|
"github.com/playwright-community/playwright-go"
|
|
)
|
|
|
|
type bhd struct{}
|
|
|
|
func NewBHD() ports.APIPorts {
|
|
return &bhd{}
|
|
}
|
|
|
|
// Scrape
|
|
// needs a mobile User Agent
|
|
func (bh bhd) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (insts []*domain.History, err error) {
|
|
tout := 120000.00
|
|
log = log.With("scrapper", "bhd")
|
|
if _, err := page.Goto(config.GetBHDURL(), playwright.PageGotoOptions{
|
|
Timeout: &tout,
|
|
WaitUntil: playwright.WaitUntilStateLoad,
|
|
}); err != nil {
|
|
log.Error("could not get info", "error", err)
|
|
return nil, err
|
|
}
|
|
|
|
err = page.WaitForLoadState()
|
|
if err != nil {
|
|
log.Error("waiting for page state", "err", err)
|
|
return nil, err
|
|
}
|
|
|
|
err = page.Locator("html body").WaitFor()
|
|
if err != nil {
|
|
log.Error("waiting for locating body", "err", err)
|
|
return nil, err
|
|
}
|
|
|
|
fieldGroupLocator := page.Locator("div.field_group")
|
|
_ = fieldGroupLocator.WaitFor()
|
|
fieldGroup, err := fieldGroupLocator.All()
|
|
//log.Info("divs", "number", len(fieldGroup))
|
|
if err != nil {
|
|
log.Error("locating field group", "err", err)
|
|
return nil, err
|
|
}
|
|
|
|
inst := &domain.History{
|
|
Name: "banco hipotecario dominicano",
|
|
Parser: "bhd",
|
|
Parsed: time.Now().Unix(),
|
|
}
|
|
|
|
for _, it := range fieldGroup {
|
|
n, _ := it.AllTextContents()
|
|
if n[0] == "CompramosUS$RD$" || n[0] == "VendemosUS$RD$" {
|
|
// we work from here in the second div as first is amount of dollars
|
|
div := it.Locator("div")
|
|
/*input := div.Locator("div > div input")
|
|
val, _ := input.InputValue()
|
|
if val == "1"{
|
|
input := div.Locator("div > div input")
|
|
}*/
|
|
places, _ := div.Locator("div").All()
|
|
|
|
for _, x := range places {
|
|
//txt, _ := x.TextContent()
|
|
input := x.Locator("div input")
|
|
if n, _ := input.Count(); n <= 0 {
|
|
//log.Info("no input")
|
|
continue
|
|
}
|
|
val, _ := input.InputValue()
|
|
if val == "" || val == "1" {
|
|
continue
|
|
}
|
|
place := strings.ReplaceAll(n[0], "US$RD$", "")
|
|
place = strings.ToLower(place)
|
|
price := helpers.Normalize(val)
|
|
switch place {
|
|
case "compramos":
|
|
inst.Compra = price
|
|
case "vendemos":
|
|
inst.Venta = price
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
if inst.Compra == 0 || inst.Venta == 0 {
|
|
return nil, fmt.Errorf("bhd: institution not parsed: %v", inst)
|
|
}
|
|
//log.Info(fmt.Sprintf("%v", inst))
|
|
return []*domain.History{inst}, nil
|
|
}
|