95 lines
2.5 KiB
Go
95 lines
2.5 KiB
Go
package crawler
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/config"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/helpers"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/internal/application/core/domain"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/internal/ports"
|
|
"github.com/playwright-community/playwright-go"
|
|
)
|
|
|
|
type inf struct{}
|
|
|
|
func NewINF() ports.APIPorts {
|
|
return &inf{}
|
|
}
|
|
|
|
// Scrape
|
|
func (in inf) Scrape(ctx context.Context, page playwright.Page, log *slog.Logger) (instList []*domain.History, err error) {
|
|
log = log.With("scrapper", "general")
|
|
tout := float64(120000)
|
|
if _, err := page.Goto(config.GetINFURL(), playwright.PageGotoOptions{
|
|
Timeout: &tout,
|
|
WaitUntil: playwright.WaitUntilStateLoad,
|
|
}); err != nil {
|
|
log.Error("could not get info", "error", err)
|
|
return nil, err
|
|
}
|
|
|
|
entriesLocator := page.Locator("table#Dolar > tbody > tr")
|
|
entriesLocator.WaitFor(playwright.LocatorWaitForOptions{
|
|
Timeout: &tout,
|
|
State: playwright.WaitForSelectorStateVisible,
|
|
})
|
|
|
|
entries, err := entriesLocator.All()
|
|
|
|
if err != nil {
|
|
log.Error("could not get info", "error", err)
|
|
return nil, err
|
|
}
|
|
|
|
instList = []*domain.History{}
|
|
for _, entry := range entries {
|
|
inst := &domain.History{
|
|
Parser: "inf",
|
|
}
|
|
title, _ := entry.Locator("span.nombre").TextContent()
|
|
name := ""
|
|
if title != "" {
|
|
name = helpers.RemoveAccent(strings.ToLower(title))
|
|
} else {
|
|
continue
|
|
}
|
|
inst.Name = name
|
|
|
|
compraLocator, ventaLocator := entry.Locator("td:nth-child(2)"), entry.Locator("td:nth-child(3)")
|
|
compra := getValue(compraLocator)
|
|
venta := getValue(ventaLocator)
|
|
|
|
inst.Compra = helpers.Normalize(compra)
|
|
inst.Venta = helpers.Normalize(venta)
|
|
|
|
inst.Parsed = time.Now().Unix()
|
|
// if one of the inst has 0 on the sell/buy dont process it
|
|
if inst.Compra == 0 || inst.Venta == 0 {
|
|
log.Warn("skipping", "nombre", inst.Name, "compra", inst.Compra, "venta", inst.Venta)
|
|
continue
|
|
}
|
|
switch {
|
|
case strings.Contains(inst.Name, "banreservas"), strings.Contains(inst.Name,"banco popular"), strings.Contains(inst.Name,"scotia"), strings.Contains(inst.Name,"hipotecario"), strings.Contains(inst.Name,"asociacion popular"), strings.Contains(inst.Name,"vimenca"):
|
|
continue
|
|
}
|
|
instList = append(instList, inst)
|
|
}
|
|
|
|
return instList, nil
|
|
}
|
|
func getValue(place playwright.Locator) string {
|
|
text, _ := place.AllInnerTexts()
|
|
value := ""
|
|
if len(text) <= 0 {
|
|
return ""
|
|
}
|
|
nextList := strings.Split(text[0], " ")
|
|
if len(nextList) > 0 {
|
|
value = strings.Replace(nextList[0], "=", "", 1)
|
|
}
|
|
return value
|
|
}
|