package crawler import ( "context" "log/slog" "strings" "time" "git.maximotejeda.com/maximo/us_dop_scrapper/config" "git.maximotejeda.com/maximo/us_dop_scrapper/helpers" "git.maximotejeda.com/maximo/us_dop_scrapper/internal/application/core/domain" "git.maximotejeda.com/maximo/us_dop_scrapper/internal/ports" "github.com/playwright-community/playwright-go" ) type inf struct{} func NewINF() ports.APIPorts { return &inf{} } // Scrape func (in inf) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*domain.History, err error) { log = log.With("scrapper", "general") tout := float64(120000) if _, err := page.Goto(config.GetINFURL(), playwright.PageGotoOptions{ Timeout: &tout, WaitUntil: playwright.WaitUntilStateLoad, }); err != nil { log.Error("could not get info", "error", err) return nil, err } entriesLocator := page.Locator("table#Dolar > tbody > tr") entriesLocator.WaitFor(playwright.LocatorWaitForOptions{ Timeout: &tout, State: playwright.WaitForSelectorStateVisible, }) entries, err := entriesLocator.All() if err != nil { log.Error("could not get info", "error", err) return nil, err } instList = []*domain.History{} for _, entry := range entries { inst := &domain.History{ Parser: "inf", } title, _ := entry.Locator("span.nombre").TextContent() name := "" if title != "" { name = helpers.RemoveAccent(strings.ToLower(title)) } else { continue } inst.Name = name compraLocator, ventaLocator := entry.Locator("td:nth-child(2)"), entry.Locator("td:nth-child(3)") compra := getValue(compraLocator) venta := getValue(ventaLocator) inst.Compra = helpers.Normalize(compra) inst.Venta = helpers.Normalize(venta) inst.Parsed = time.Now().Unix() // if one of the inst has 0 on the sell/buy dont process it if inst.Compra == 0 || inst.Venta == 0 { log.Warn("skipping", "nombre", inst.Name, "compra", inst.Compra, "venta", inst.Venta) continue } switch { case strings.Contains(inst.Name, "banreservas"), strings.Contains(inst.Name,"banco popular"), strings.Contains(inst.Name,"scotia"), strings.Contains(inst.Name,"hipotecario"), strings.Contains(inst.Name,"asociacion popular"), strings.Contains(inst.Name,"vimenca"): continue } instList = append(instList, inst) } return instList, nil } func getValue(place playwright.Locator) string { text, _ := place.AllInnerTexts() value := "" if len(text) <= 0 { return "" } nextList := strings.Split(text[0], " ") if len(nextList) > 0 { value = strings.Replace(nextList[0], "=", "", 1) } return value }