424 lines
10 KiB
Go
424 lines
10 KiB
Go
package db
|
|
|
|
import (
|
|
"database/sql"
|
|
_ "embed"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/maximotejeda/us_dop_scrapper/models"
|
|
"github.com/maximotejeda/us_dop_scrapper/pub"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
//go:embed schema.sql
|
|
var schema string
|
|
|
|
type DB struct {
|
|
*sql.DB
|
|
log *slog.Logger
|
|
}
|
|
|
|
type change struct {
|
|
Before models.Institucion `json:"before"`
|
|
After models.Institucion `json:"after"`
|
|
}
|
|
|
|
type Message struct {
|
|
Message string `json:"message"`
|
|
Data change `json:"data"`
|
|
Error error `json:"error"`
|
|
}
|
|
|
|
type Institution struct {
|
|
ID int
|
|
Name string
|
|
ShortName string
|
|
Created time.Time
|
|
}
|
|
|
|
// Dial
|
|
func Dial(path string, log *slog.Logger) *DB {
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
fmt.Printf("opening database: %s", err.Error())
|
|
panic("opening database")
|
|
}
|
|
if err := db.Ping(); err != nil {
|
|
fmt.Printf("pinging database: %s", err.Error())
|
|
panic("pinging database")
|
|
}
|
|
return &DB{db, log}
|
|
}
|
|
|
|
// Schema
|
|
func (db *DB) CreateTables() {
|
|
_, err := db.Exec(schema)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// Inspect
|
|
// Handle behavior of the changes
|
|
// Will report errors to a nats consumer
|
|
func (db *DB) Inspect(enter models.Institucion) error {
|
|
if db == nil {
|
|
return fmt.Errorf("nil or empty database")
|
|
}
|
|
pub, close := pub.Publisher()
|
|
defer close()
|
|
msg := Message{}
|
|
// Get last row added
|
|
|
|
inst, err := db.GetLatest(enter.Parser, enter.Name)
|
|
// if no rows are found because of first enter a name - parser ?
|
|
if errors.Is(sql.ErrNoRows, err) {
|
|
db.log.Info("adding new item to table: ", "parse", enter.Parser, "name", enter.Name)
|
|
msg.Message = "add new institution"
|
|
msg.Data.After = enter
|
|
|
|
data, err := json.Marshal(msg)
|
|
if err != nil {
|
|
db.log.Error("marshaling struct", "error", err)
|
|
}
|
|
|
|
id, err := db.ADDInstitution(enter.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pub("dolar-crawler", data)
|
|
return db.AddNew(enter, id)
|
|
}
|
|
|
|
// check prices compra venta
|
|
if inst == nil {
|
|
db.log.Error("row is nil", "name", enter.Name, "parser", enter.Parser)
|
|
return fmt.Errorf("row is nil, not entering row")
|
|
}
|
|
if enter.Compra == inst.Compra && enter.Venta == inst.Venta {
|
|
return nil
|
|
} else {
|
|
// if one of them changes create a new row
|
|
db.log.Info("change registered, adding item", "parse", enter.Parser, "name", enter.Name, "compra enter", enter.Compra, "compra db", inst.Compra, "venta enter", enter.Venta, "venta db", inst.Venta)
|
|
|
|
msg.Message = "change registered"
|
|
msg.Data.After = enter
|
|
msg.Data.Before = *inst
|
|
|
|
data, err := json.Marshal(msg)
|
|
if err != nil {
|
|
db.log.Error("marshaling struct", "error", err)
|
|
}
|
|
ins, err := db.GETInstitution(enter.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer pub("dolar-crawler", data)
|
|
return db.AddNew(enter, int64(ins.ID))
|
|
}
|
|
}
|
|
|
|
// GetLatest
|
|
// returns the latest row in a specific parser and name
|
|
// we are using DateTime in DB and date.Datetime in go
|
|
func (db *DB) GetLatest(parser string, name string) (inst *models.Institucion, err error) {
|
|
var parsed string
|
|
inst = &models.Institucion{}
|
|
stmtt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE d.parser = ? AND i.name = ? ORDER BY d.parsed DESC LIMIT 1;")
|
|
if err != nil {
|
|
db.log.Error("preparing stmtt", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
defer stmtt.Close()
|
|
|
|
if err := stmtt.QueryRow(parser, name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
|
|
db.log.Error("getting latest", "error", err.Error(), "parser", parser, "name", name)
|
|
return nil, err
|
|
}
|
|
|
|
inst.Parsed, err = time.Parse(time.DateTime, parsed)
|
|
if err != nil {
|
|
//db.log.Error("parsed", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
return inst, nil
|
|
}
|
|
|
|
// AddNew
|
|
// Add a new row in the dolar table
|
|
// Will send to nats changes on prices
|
|
func (db *DB) AddNew(row models.Institucion, id int64) error {
|
|
stmt, err := db.Prepare("INSERT INTO dolars (name_id, compra, venta, parser, parsed) VALUES(?,?,?,?,?);")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer stmt.Close()
|
|
parsed := row.Parsed.Format(time.DateTime)
|
|
_, err = stmt.Exec(&id, &row.Compra, &row.Venta, &row.Parser, &parsed)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (db *DB) ADDInstitution(name string) (id int64, err error) {
|
|
stmt, err := db.Prepare("INSERT INTO institutions (name, short_name, created) VALUES(?,?,?);")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer stmt.Close()
|
|
parsed := time.Now().Format(time.DateTime)
|
|
short := shortner(name)
|
|
res, err := stmt.Exec(&name, short, &parsed)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
id, err = res.LastInsertId()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return id, nil
|
|
|
|
}
|
|
func (db *DB) GETInstitution(name string) (inst *Institution, err error) {
|
|
institution := Institution{}
|
|
stmtt, err := db.Prepare("SELECT id, name, short_name FROM institutions WHERE name = ?")
|
|
if err != nil {
|
|
db.log.Error("preparing stmt", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
defer stmtt.Close()
|
|
if err := stmtt.QueryRow(name).Scan(&institution.ID, &institution.Name, &institution.ShortName); err != nil {
|
|
db.log.Error("getting institution", "error", err.Error(), "short name", institution.ShortName, "name", name)
|
|
return nil, err
|
|
}
|
|
return inst, err
|
|
}
|
|
|
|
func (db *DB) GetAll() ([]string, error) {
|
|
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i;")
|
|
if err != nil {
|
|
db.log.Error("[db-GetAll]", "error", err)
|
|
return nil, err
|
|
}
|
|
rows, err := stmt.Query()
|
|
if err != nil {
|
|
db.log.Error("[db-GetAll-stmt]", "error", err)
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
insts := []string{}
|
|
for rows.Next() {
|
|
inst := ""
|
|
|
|
if err = rows.Scan(&inst); err != nil {
|
|
return nil, err
|
|
}
|
|
if inst == "" {
|
|
continue
|
|
}
|
|
insts = append(insts, inst)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return insts, err
|
|
}
|
|
return insts, nil
|
|
|
|
}
|
|
func (db *DB) GetBancos() ([]string, error) {
|
|
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%ban%' OR i.name LIKE '%scoti%'")
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll]", "error", err)
|
|
return nil, err
|
|
}
|
|
rows, err := stmt.Query()
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll-stmt]", "error", err)
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
insts := []string{}
|
|
for rows.Next() {
|
|
inst := ""
|
|
|
|
if err = rows.Scan(&inst); err != nil {
|
|
return nil, err
|
|
}
|
|
if inst == "" {
|
|
continue
|
|
}
|
|
insts = append(insts, inst)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return insts, err
|
|
}
|
|
return insts, nil
|
|
|
|
}
|
|
func (db *DB) GetCajas() ([]string, error) {
|
|
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name LIKE '%asociacion%'")
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll]", "error", err)
|
|
return nil, err
|
|
}
|
|
rows, err := stmt.Query()
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll-stmt]", "error", err)
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
insts := []string{}
|
|
for rows.Next() {
|
|
inst := ""
|
|
|
|
if err = rows.Scan(&inst); err != nil {
|
|
return nil, err
|
|
}
|
|
if inst == "" {
|
|
continue
|
|
}
|
|
insts = append(insts, inst)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return insts, err
|
|
}
|
|
return insts, nil
|
|
|
|
}
|
|
|
|
func (db *DB) GetAgentes() ([]string, error) {
|
|
stmt, err := db.Prepare("SELECT i.name FROM institutions AS i WHERE i.name NOT LIKE '%ban%' AND i.name NOT LIKE '%scoti%' AND i.name NOT LIKE '%asociacion%'")
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll]", "error", err)
|
|
return nil, err
|
|
}
|
|
rows, err := stmt.Query()
|
|
if err != nil {
|
|
db.log.Error("[inst-GetAll-stmt]", "error", err)
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
insts := []string{}
|
|
for rows.Next() {
|
|
inst := ""
|
|
|
|
if err = rows.Scan(&inst); err != nil {
|
|
return nil, err
|
|
}
|
|
if inst == "" {
|
|
continue
|
|
}
|
|
insts = append(insts, inst)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return insts, err
|
|
}
|
|
return insts, nil
|
|
|
|
}
|
|
|
|
func (db *DB) GetLastPrice(name string) (inst *models.Institucion, err error) {
|
|
var parsed string
|
|
inst = &models.Institucion{}
|
|
stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? ORDER BY parsed DESC LIMIT 1;")
|
|
if err != nil {
|
|
db.log.Error("preparing", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
defer stmt.Close()
|
|
|
|
if err := stmt.QueryRow(name).Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
|
|
db.log.Error("getting last price", "error", err.Error(), "name", name)
|
|
return nil, err
|
|
}
|
|
|
|
inst.Parsed, err = time.Parse(time.DateTime, parsed)
|
|
if err != nil {
|
|
//db.log.Error("parsed", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
return inst, nil
|
|
|
|
}
|
|
func (db *DB) GetChangeSince(name string, duration time.Duration) (insts []*models.Institucion, err error) {
|
|
date := time.Now().Add(-duration).Format(time.DateTime)
|
|
stmt, err := db.Prepare("SELECT i.name, d.parser, d.compra, d.venta, d.parsed FROM dolars AS d JOIN institutions as i ON d.name_id = i.id WHERE name = ? AND parsed > ? ORDER BY parsed DESC;")
|
|
if err != nil {
|
|
db.log.Error("[GetChangeSince] preparing", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
defer stmt.Close()
|
|
rows, err := stmt.Query(name, date)
|
|
if err != nil {
|
|
db.log.Error("[GetChangeSince] preparing", "error", err.Error())
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
for rows.Next() {
|
|
inst := models.Institucion{}
|
|
parsed := ""
|
|
if err := rows.Scan(&inst.Name, &inst.Parser, &inst.Compra, &inst.Venta, &parsed); err != nil {
|
|
db.log.Error("[GetChangeSince] scanning", "error", err)
|
|
return nil, err
|
|
}
|
|
inst.Parsed, err = time.Parse(time.DateTime, parsed)
|
|
if err != nil {
|
|
//db.log.Error("parsed", "error", err.Error())
|
|
continue
|
|
}
|
|
insts = append(insts, &inst)
|
|
}
|
|
return insts, nil
|
|
}
|
|
|
|
func shortner(name string) string {
|
|
if name == "" {
|
|
return ""
|
|
}
|
|
switch strings.ToLower(name) {
|
|
case "banco popular":
|
|
return "bpd"
|
|
case "banreservas":
|
|
return "brd"
|
|
case "banco central dominicano":
|
|
return "bcd"
|
|
case "banco hipotecario dominicano":
|
|
return "bhd"
|
|
case "asociacion popular de ahorros y prestamos":
|
|
return "apap"
|
|
case "asociacion cibao de ahorros y prestamos":
|
|
return "acap"
|
|
case "asociacion la nacional de ahorros y prestamos":
|
|
return "alnap"
|
|
case "asociacion peravia de ahorros y prestamos":
|
|
return "apeap"
|
|
case "banco santa cruz":
|
|
return "bsc"
|
|
case "imbert y balbuena":
|
|
return "imb"
|
|
case "banco activo dominicana":
|
|
return "bacd"
|
|
case "scotiabank cambio online":
|
|
return "scline"
|
|
case "banco lopez de haro":
|
|
return "blh"
|
|
}
|
|
nameList := strings.Split(name, " ")
|
|
switch len(nameList) {
|
|
case 1:
|
|
return nameList[0]
|
|
case 2:
|
|
return string(nameList[0][0]) + nameList[1][0:2]
|
|
case 3:
|
|
return string(nameList[0][0] + nameList[1][0] + nameList[2][0])
|
|
default:
|
|
return "n/a"
|
|
}
|
|
}
|