76 lines
1.7 KiB
Go
76 lines
1.7 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"time"
|
|
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/helpers"
|
|
"git.maximotejeda.com/maximo/us_dop_scrapper/internal/ports"
|
|
"github.com/playwright-community/playwright-go"
|
|
)
|
|
|
|
type Application struct {
|
|
log *slog.Logger
|
|
api ports.APIPorts
|
|
client ports.DollarPort
|
|
}
|
|
|
|
func NewApplication(crawler ports.APIPorts, client ports.DollarPort) *Application {
|
|
log := slog.Default()
|
|
log = log.With("application", "root")
|
|
return &Application{
|
|
log: log,
|
|
api: crawler,
|
|
client: client,
|
|
}
|
|
}
|
|
|
|
func (a Application) Run() {
|
|
ctx := context.Background()
|
|
ch, ff, wk := helpers.CreateBrowser(a.log)
|
|
err := a.ExecParser(ctx, ch, a.log)
|
|
if err != nil {
|
|
a.log.Info("failed on frist browser", "browser", "chrome", "error", err)
|
|
err := a.ExecParser(ctx, ff, a.log)
|
|
if err != nil {
|
|
a.log.Error("failed on second browser", "browser", "firefox", "error", err)
|
|
err := a.ExecParser(ctx, wk, a.log)
|
|
if err != nil {
|
|
a.log.Error("tried all browsers error", "brwser", "webkit", "error", err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (a Application) ExecParser(
|
|
ctx context.Context,
|
|
browser *playwright.BrowserContext,
|
|
log *slog.Logger) (err error) {
|
|
b := *browser
|
|
page, err := b.NewPage()
|
|
if err != nil {
|
|
log.Error("creating page", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 6*time.Minute)
|
|
defer page.Close()
|
|
defer cancel()
|
|
histList, err := a.api.Scrape(ctx, page, log)
|
|
// here we execute db operations
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, hist := range histList {
|
|
err = a.client.NewHistory(hist)
|
|
if err != nil {
|
|
a.log.Error("creating new hist", "history", hist, "error", err)
|
|
}
|
|
a.log.Info("parsed Success", "parser", hist.Parser, "item", hist)
|
|
}
|
|
return err
|
|
}
|