jadudm/grosbeak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Queueing, prepping to fetch

Browse Source 
A lot more to go, but this is the core.

Need to think about how to test the queue handlers.
This commit is contained in:
Matt Jadud
2025-11-30 21:29:30 -05:00
parent 06cdc68be7
commit f53639af2f
14 changed files with 258 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/domain64/database.go
View File

@@ -3,6 +3,7 @@ package domain64
import (
"context"
"database/sql"
_ "embed"
"fmt"
"log"
"slices"
@@ -12,21 +13,33 @@ import (
"github.com/jpillora/go-tld"
)
//go:embed sqlc/schema.sql
var ddl string
type Domain64Map struct {
mu sync.Mutex
m map[string]int
DB *sql.DB
Flushed bool
Queries *sqlc.Queries
}
func NewDomain64Map(db *sql.DB) (*Domain64Map, error) {
func NewDomain64Map() (*Domain64Map, error) {
d64m := &Domain64Map{}
d64m.DB = db
d64m.Flushed = true
// Init the tables if a DB pointer is passed in.
d64m.DB = nil
d64m.Queries = nil
return d64m, nil
}
func (d64m *Domain64Map) Setup(db *sql.DB) {
log.Printf("creating domain64 tables\n")
if _, err := db.ExecContext(context.Background(), ddl); err != nil {
log.Printf("setup err: %s", err.Error())
panic(err)
}
d64m.DB = db
d64m.Queries = sqlc.New(db)
}
func (d64m *Domain64Map) URLToRFQDN(url *tld.URL) string {
s := ""
if url.TLD != "" {
@@ -140,21 +153,21 @@ func (d64m *Domain64Map) URLToDomain64(url *tld.URL) (*Domain64, error) {
}
d64 := &Domain64{}
queries := sqlc.New(d64m.DB)
// These manipulate both the DB and the Domain64 struct
err := _get_or_insert_tld(queries, d64, url)
err := _get_or_insert_tld(d64m.Queries, d64, url)
if err != nil {
return nil, err
}
err = _get_or_insert_domain(queries, d64, url)
err = _get_or_insert_domain(d64m.Queries, d64, url)
if err != nil {
return nil, err
}
err = _get_or_insert_subdomain(queries, d64, url)
err = _get_or_insert_subdomain(d64m.Queries, d64, url)
if err != nil {
return nil, err
}
err = _get_or_insert_path(queries, d64, url)
err = _get_or_insert_path(d64m.Queries, d64, url)
if err != nil {
return nil, err
}

19
internal/domain64/database_test.go
View File

@@ -3,16 +3,12 @@ package domain64
import (
"context"
"database/sql"
_ "embed"
"testing"
"github.com/jpillora/go-tld"
_ "modernc.org/sqlite"
)
//go:embed sqlc/schema.sql
var ddl string
func setup() *sql.DB {
ctx := context.Background()
@@ -32,7 +28,8 @@ func setup() *sql.DB {
func TestNewDomain64Map(t *testing.T) {
db := setup()
M, err := NewDomain64Map(db)
M, err := NewDomain64Map()
M.Setup(db)
if err != nil {
// TODO
t.Error(err)
@@ -43,7 +40,8 @@ func TestNewDomain64Map(t *testing.T) {
}
func TestURLToRFQDN(t *testing.T) {
M, _ := NewDomain64Map(nil)
M, _ := NewDomain64Map()
// M.Setup(db)
simple, _ := tld.Parse("https://jadud.com/")
rfqdn := M.URLToRFQDN(simple)
if rfqdn != "com.jadud/" {
@@ -53,7 +51,8 @@ func TestURLToRFQDN(t *testing.T) {
func TestURLToDomain64(t *testing.T) {
db := setup()
M, _ := NewDomain64Map(db)
M, _ := NewDomain64Map()
M.Setup(db)
simple, _ := tld.Parse("https://jadud.com/")
d64, _ := M.URLToDomain64(simple)
if d64.TLD != 1 {
@@ -67,7 +66,8 @@ func TestURLToDomain64(t *testing.T) {
func TestURLToDomain64_02(t *testing.T) {
db := setup()
M, _ := NewDomain64Map(db)
M, _ := NewDomain64Map()
M.Setup(db)
simple1, _ := tld.Parse("https://jadud.com/")
simple2, _ := tld.Parse("https://another.com/")
d64_1, _ := M.URLToDomain64(simple1)
@@ -93,7 +93,8 @@ func TestURLToDomain64_02(t *testing.T) {
func TestURLToDomain64_03(t *testing.T) {
db := setup()
M, _ := NewDomain64Map(db)
M, _ := NewDomain64Map()
M.Setup(db)
var tests = []struct {
url string
tld int64

39
internal/engine/entre.go Normal file
View File

@@ -0,0 +1,39 @@
package engine
import (
"context"
"fmt"
"log"
"time"
"git.jadud.com/jadudm/grosbeak/internal/liteq"
base "git.jadud.com/jadudm/grosbeak/internal/types"
)
func Entre(queue *liteq.JobQueue, ej *base.EntreJob) error {
n := time.Now()
var ignore_tag string
switch ej.UpdateFrequency {
case base.UPDATE_DAILY:
ignore_tag = fmt.Sprintf("%s|y%d-yd%d", ej.URL, n.Year(), n.YearDay())
case base.UPDATE_WEEKLY:
ignore_tag = fmt.Sprintf("%s|y%d-w%d", ej.URL, n.Year(), n.YearDay()/7)
case base.UPDATE_MONTHLY:
ignore_tag = fmt.Sprintf("%s|y%d-m%d", ej.URL, n.Year(), n.Month())
default:
ignore_tag = fmt.Sprintf("%s|y%d-yd%d", ej.URL, n.Year(), n.YearDay())
}
err := queue.QueueJob(context.Background(), liteq.QueueJobParams{
Queue: "fetch",
// This only works for things in the `queued` state
DedupingKey: liteq.IgnoreDuplicate(ignore_tag),
Job: ej.AsJson(),
})
if err != nil {
log.Printf("entre err %s: %s\n", ej.URL, err.Error())
}
return err
}

34
internal/engine/fetch.go Normal file
View File

@@ -0,0 +1,34 @@
package engine
import (
"context"
"log"
"git.jadud.com/jadudm/grosbeak/internal/domain64"
"git.jadud.com/jadudm/grosbeak/internal/liteq"
"git.jadud.com/jadudm/grosbeak/internal/types"
"github.com/jpillora/go-tld"
"github.com/tidwall/gjson"
)
func Fetch(d64m *domain64.Domain64Map) types.QueueWorker {
_f := func(ctx context.Context, job *liteq.Job) error {
url := gjson.Get(job.Job, "url").String()
turl, err := tld.Parse(url)
if err != nil {
// If we can't parse it, shut the job down as completed.
return nil
}
d64, err := d64m.URLToDomain64(turl)
if err != nil {
log.Printf("urltodomain64 err %s: %s\n", url, err.Error())
}
log.Printf("fetching %s 0x%016x\n", url, d64.ToInt64())
return nil
}
return _f
}

0
internal/liteq/internal/jobs_test.go → internal/liteq/internal_test/jobs_test.go
View File

9
internal/types/base.go Normal file
View File

@@ -0,0 +1,9 @@
package types
import (
"context"
"git.jadud.com/jadudm/grosbeak/internal/liteq"
)
type QueueWorker func(ctx context.Context, job *liteq.Job) error

9
internal/types/constants.go Normal file
View File

@@ -0,0 +1,9 @@
package types
type UpdateFrequency string
const (
UPDATE_DAILY UpdateFrequency = "DAILY"
UPDATE_WEEKLY UpdateFrequency = "WEEKLY"
UPDATE_MONTHLY UpdateFrequency = "MONTHLY"
)

22
internal/types/jobs.go Normal file
View File

@@ -0,0 +1,22 @@
package types
import "encoding/json"
type Job interface {
AsJson() string
}
// Used by all of the jobs.
func _as_json(j Job) string {
as_json, _ := json.Marshal(j)
return string(as_json)
}
type EntreJob struct {
URL string `json:"url"`
UpdateFrequency UpdateFrequency `json:"update_frequency"`
}
func (ej *EntreJob) AsJson() string {
return _as_json(ej)
}