80 lines
2.4 KiB
Go
80 lines
2.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/maximotejeda/us_dop_db/db"
|
|
"github.com/maximotejeda/us_dop_scrapper/apa"
|
|
"github.com/maximotejeda/us_dop_scrapper/bcd"
|
|
"github.com/maximotejeda/us_dop_scrapper/bdr"
|
|
"github.com/maximotejeda/us_dop_scrapper/bhd"
|
|
"github.com/maximotejeda/us_dop_scrapper/bnc"
|
|
"github.com/maximotejeda/us_dop_scrapper/bpd"
|
|
"github.com/maximotejeda/us_dop_scrapper/helpers"
|
|
"github.com/maximotejeda/us_dop_scrapper/inf"
|
|
"github.com/maximotejeda/us_dop_scrapper/scotia"
|
|
"github.com/maximotejeda/us_dop_scrapper/vimenca"
|
|
"github.com/playwright-community/playwright-go"
|
|
)
|
|
|
|
func main() {
|
|
var err error
|
|
dbRoute := os.Getenv("DBURI")
|
|
sig := make(chan os.Signal, 1)
|
|
signal.Notify(sig, os.Interrupt, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
|
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{}))
|
|
db := db.Dial(dbRoute, log)
|
|
// create a chrome and feed parsers
|
|
db.CreateTables()
|
|
chrome, firefox, webkit := helpers.CreateBrowser(log)
|
|
browserList := []*playwright.Browser{chrome, firefox, webkit}
|
|
longTick := time.NewTicker(time.Minute * 2)
|
|
defer longTick.Stop()
|
|
infoTick := time.NewTicker(time.Minute * 1)
|
|
defer infoTick.Stop()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
errN := map[string]int{
|
|
"bcd": 0,
|
|
"bpd": 0,
|
|
"apap": 0,
|
|
"inf": 0,
|
|
}
|
|
|
|
defer cancel()
|
|
who := os.Getenv("WHO")
|
|
|
|
switch who {
|
|
case "bcd":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bcd", bcd.ExecParser)
|
|
case "bpd":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bpd", bpd.ExecParser)
|
|
case "apa":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "apa", apa.ExecParser)
|
|
case "brd":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "brd", bdr.ExecParser)
|
|
case "bhd":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bhd", bhd.ExecParser)
|
|
case "bnc":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "bnc", bnc.ExecParser)
|
|
case "scotia":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "scotia", scotia.ExecParser)
|
|
case "vimenca":
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "vimenca", vimenca.ExecParser)
|
|
default:
|
|
err = helpers.ExecTask(ctx, db, browserList, log, errN, "inf", inf.ExecParser)
|
|
}
|
|
|
|
if err != nil {
|
|
log.Info("task executed with errors", "name", who, "error", err)
|
|
os.Exit(1)
|
|
return
|
|
}
|
|
log.Info("SUCCESS - task executed", "name", who)
|
|
os.Exit(0)
|
|
}
|