package db import ( "database/sql" _ "embed" "encoding/json" "errors" "fmt" "log/slog" "strings" "time" "github.com/maximotejeda/us_dop_scrapper/models" "github.com/maximotejeda/us_dop_scrapper/pub" _ "modernc.org/sqlite" ) //go:embed schema.sql var schema string type DB struct { *sql.DB log *slog.Logger } type change struct { Before models.Institucion `json:"before"` After models.Institucion `json:"after"` } type Message struct { Message string `json:"message"` Data change `json:"data"` Error error `json:"error"` } type Institution struct { ID int Name string ShortName string Created time.Time } // Dial func Dial(path string, log *slog.Logger) *DB { db, err := sql.Open("sqlite", path) if err != nil { fmt.Printf("opening database: %s", err.Error()) panic("opening database") } if err := db.Ping(); err != nil { fmt.Printf("pinging database: %s", err.Error()) panic("pinging database") } return &DB{db, log} } // Schema func (db *DB) CreateTables() { _, err := db.Exec(schema) if err != nil { panic(err) } } // Inspect // Handle behavior of the changes // Will report errors to a nats consumer func (db *DB) Inspect(enter models.Institucion) error { if db == nil { return fmt.Errorf("nil or empty database") } pub, close := pub.Publisher() defer close() msg := Message{} // Get last row added inst, err := db.GetLatest(enter.Parser, enter.Name) // if no rows are found because of first enter a name - parser ? if errors.Is(sql.ErrNoRows, err) { db.log.Info("adding new item to table: ", "parse", enter.Parser, "name", enter.Name) msg.Message = "add new institution" msg.Data.After = enter data, err := json.Marshal(msg) if err != nil { db.log.Error("marshaling struct", "error", err) } id, err := db.ADDInstitution(enter.Name) if err != nil { return err } defer pub("dolar-crawler", data) return db.AddNew(enter, id) } // check prices compra venta if inst == nil { db.log.Error("row is nil", "name", enter.Name, "parser", enter.Parser) return fmt.Errorf("row is nil, not entering row") } if enter.Compra == inst.Compra && enter.Venta == inst.Venta { return nil } else { // if one of them changes create a new row db.log.Info("change registered, adding item", "parse", enter.Parser, "name", enter.Name, "compra enter", enter.Compra, "compra db", inst.Compra, "venta enter", enter.Venta, "venta db", inst.Venta) msg.Message = "change registered" msg.Data.After = enter msg.Data.Before = *inst data, err := json.Marshal(msg) if err != nil { db.log.Error("marshaling struct", "error", err) } ins, err := db.GETInstitution(enter.Name) if err != nil { return err } defer pub("dolar-crawler", data) return db.AddNew(enter, int64(ins.ID)) } } // GetLatest // returns the latest row in a specific parser and name // we are using DateTime in DB and date.Datetime in go func (db *DB) GetLatest(parser string, name string) (inst *models.Institucion, err error) { var parsed string inst = &models.Institucion{} stmtt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE d.parser = ? AND i.name = ? ORDER BY d.parsed DESC LIMIT 1;") if err != nil { db.log.Error("preparing stmtt", "error", err.Error()) return nil, err } defer stmtt.Close() if err := stmtt.QueryRow(parser, name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { db.log.Error("getting latest", "error", err.Error(), "parser", parser, "name", name) return nil, err } inst.Parsed, err = time.Parse(time.DateTime, parsed) if err != nil { //db.log.Error("parsed", "error", err.Error()) return nil, err } return inst, nil } // AddNew // Add a new row in the dolar table // Will send to nats changes on prices func (db *DB) AddNew(row models.Institucion, id int64) error { stmt, err := db.Prepare("INSERT INTO dolars (name_id, compra, venta, parser, parsed) VALUES(?,?,?,?,?);") if err != nil { return err } defer stmt.Close() parsed := row.Parsed.Format(time.DateTime) _, err = stmt.Exec(&id, &row.Compra, &row.Venta, &row.Parser, &parsed) if err != nil { return err } return nil } func (db *DB) ADDInstitution(name string) (id int64, err error) { stmt, err := db.Prepare("INSERT INTO institutions (name, short_name, created) VALUES(?,?,?);") if err != nil { return 0, err } defer stmt.Close() parsed := time.Now().Format(time.DateTime) short := shortner(name) res, err := stmt.Exec(&name, short, &parsed) if err != nil { return 0, err } id, err = res.LastInsertId() if err != nil { return 0, err } return id, nil } func (db *DB) GETInstitution(name string) (inst *Institution, err error) { institution := Institution{} stmtt, err := db.Prepare("SELECT id, name, short_name FROM institutions WHERE name = ?") if err != nil { db.log.Error("preparing stmt", "error", err.Error()) return nil, err } defer stmtt.Close() if err := stmtt.QueryRow(name).Scan(&institution.ID, &institution.Name, &institution.ShortName); err != nil { db.log.Error("getting institution", "error", err.Error(), "short name", institution.ShortName, "name", name) return nil, err } return inst, err } func (db *DB) GetAll() ([]string, error) { stmt, err := db.Prepare("SELECT i.name FROM institutions AS i;") if err != nil { db.log.Error("[db-GetAll]", "error", err) return nil, err } rows, err := stmt.Query() if err != nil { db.log.Error("[db-GetAll-stmt]", "error", err) return nil, err } defer rows.Close() insts := []string{} for rows.Next() { inst := "" if err = rows.Scan(&inst); err != nil { return nil, err } if inst == "" { continue } insts = append(insts, inst) } if err := rows.Err(); err != nil { return insts, err } return insts, nil } func (db *DB) GetBancos() ([]string, error) { stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%ban%' OR i.name LIKE '%scoti%'") if err != nil { db.log.Error("[inst-GetAll]", "error", err) return nil, err } rows, err := stmt.Query() if err != nil { db.log.Error("[inst-GetAll-stmt]", "error", err) return nil, err } defer rows.Close() insts := []string{} for rows.Next() { inst := "" if err = rows.Scan(&inst); err != nil { return nil, err } if inst == "" { continue } insts = append(insts, inst) } if err := rows.Err(); err != nil { return insts, err } return insts, nil } func (db *DB) GetCajas() ([]string, error) { stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%asociacion%'") if err != nil { db.log.Error("[inst-GetAll]", "error", err) return nil, err } rows, err := stmt.Query() if err != nil { db.log.Error("[inst-GetAll-stmt]", "error", err) return nil, err } defer rows.Close() insts := []string{} for rows.Next() { inst := "" if err = rows.Scan(&inst); err != nil { return nil, err } if inst == "" { continue } insts = append(insts, inst) } if err := rows.Err(); err != nil { return insts, err } return insts, nil } func (db *DB) GetAgentes() ([]string, error) { stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name NOT LIKE '%ban%' AND i.name NOT LIKE '%scoti%' AND i.name NOT LIKE '%asociacion%'") if err != nil { db.log.Error("[inst-GetAll]", "error", err) return nil, err } rows, err := stmt.Query() if err != nil { db.log.Error("[inst-GetAll-stmt]", "error", err) return nil, err } defer rows.Close() insts := []string{} for rows.Next() { inst := "" if err = rows.Scan(&inst); err != nil { return nil, err } if inst == "" { continue } insts = append(insts, inst) } if err := rows.Err(); err != nil { return insts, err } return insts, nil } func (db *DB) GetLastPrice(name string) (inst *models.Institucion, err error) { var parsed string inst = &models.Institucion{} stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? ORDER BY parsed DESC LIMIT 1;") if err != nil { db.log.Error("preparing", "error", err.Error()) return nil, err } defer stmt.Close() if err := stmt.QueryRow(name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { db.log.Error("getting last price", "error", err.Error(), "name", name) return nil, err } inst.Parsed, err = time.Parse(time.DateTime, parsed) if err != nil { //db.log.Error("parsed", "error", err.Error()) return nil, err } return inst, nil } func (db *DB) GetChangeSince(name string, duration time.Duration) (insts []*models.Institucion, err error) { date := time.Now().Add(-duration).Format(time.DateTime) stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? AND parsed > ? ORDER BY parsed DESC;") if err != nil { db.log.Error("[GetChangeSince] preparing", "error", err.Error()) return nil, err } defer stmt.Close() rows, err := stmt.Query(name, date) if err != nil { db.log.Error("[GetChangeSince] preparing", "error", err.Error()) return nil, err } defer rows.Close() for rows.Next() { inst := models.Institucion{} parsed := "" if err := rows.Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil { db.log.Error("[GetChangeSince] scanning", "error", err) return nil, err } inst.Parsed, err = time.Parse(time.DateTime, parsed) if err != nil { //db.log.Error("parsed", "error", err.Error()) continue } insts = append(insts, &inst) } return insts, nil } func shortner(name string) string { if name == "" { return "" } switch strings.ToLower(name) { case "banco popular": return "bpd" case "banreservas": return "brd" case "banco central dominicano": return "bcd" case "banco hipotecario dominicano": return "bhd" case "asociacion popular de ahorros y prestamos": return "apap" case "asociacion cibao de ahorros y prestamos": return "acap" case "asociacion la nacional de ahorros y prestamos": return "alnap" case "asociacion peravia de ahorros y prestamos": return "apeap" case "banco santa cruz": return "bsc" case "imbert y balbuena": return "imb" case "banco activo dominicana": return "bacd" case "scotiabank cambio online": return "scline" case "banco lopez de haro": return "blh" } nameList := strings.Split(name, " ") switch len(nameList) { case 1: return nameList[0] case 2: return string(nameList[0][0]) + nameList[1][0:2] case 3: return string(nameList[0][0] + nameList[1][0] + nameList[2][0]) default: return "n/a" } }