diff --git a/cmd/api/main.go b/cmd/api/main.go index 909f194..0e3c730 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -1,20 +1,84 @@ package main import ( + "context" + "database/sql" "fmt" + "log" + "math/rand" + "os" + "sync" + "time" - "git.jadud.com/grosbeak/internal/domain64" + _ "modernc.org/sqlite" + + liteq "git.jadud.com/grosbeak/internal/liteq" ) -func main() { - d64 := domain64.Domain64{ - TLD: 1, - Domain: 1, - Subdomain: 1, - Path: 1, - } - result := d64.ToInt64() - fmt.Printf("%064b\n", result) - fmt.Printf("%016x\n", result) - fmt.Println(result) +func Fetch(ctx context.Context, job *liteq.Job) error { + n := rand.Intn(50) + + time.Sleep(time.Duration(n) * time.Millisecond) + log.Println("Fetching", job.Job) + return nil +} + +func runWorkers(queue *liteq.JobQueue) { + go queue.Consume(context.Background(), liteq.ConsumeParams{ + Queue: "fetch", + PoolSize: 3, + VisibilityTimeout: 200000, + Worker: Fetch, + }) +} + +func Entre(queue *liteq.JobQueue, chUrl <-chan string) { + ctx := context.Background() + for { + url := <-chUrl + log.Println("Entre", url) + // Don't duplicate jobs on the same day of the year. + n := time.Now() + queue.QueueJob(ctx, liteq.QueueJobParams{ + Queue: "fetch", + // This only works for things in the `queued` state + DedupingKey: liteq.IgnoreDuplicate(fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay())), + Job: url, + }) + } +} + +func main() { + // Don't let `main()` exit + wg := &sync.WaitGroup{} + wg.Add(1) + + // FIXME: This path needs to come from the env. + liteqDB, err := sql.Open("sqlite", "liteq.db") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + liteq.Setup(liteqDB) + queue := liteq.New(liteqDB) + // The queue processes as long as this context is not cancelled. + + log.Println("Setting up workers...") + go runWorkers(queue) + + log.Println("Building network...") + // Create the network for the search engine. + chUrl := make(chan string) + + go Entre(queue, chUrl) + for { + chUrl <- "https://jadud.com/" + time.Sleep(2 * time.Second) + chUrl <- "https://berea.us/" + time.Sleep(2 * time.Second) + } + + // Don't exit. + log.Println("Waiting...") + wg.Wait() } diff --git a/go.mod b/go.mod index d37ed0d..4416c2b 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,16 @@ require ( ) require ( + github.com/alitto/pond v1.9.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.36.0 // indirect - modernc.org/libc v1.66.10 // indirect + golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.38.0 // indirect + modernc.org/libc v1.67.1 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index d2c73c3..0ae0f2c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= +github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= @@ -6,8 +8,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jpillora/go-tld v1.2.1 h1:kDKOkmXLlskqjcvNs7w5XHLep7c8WM7Xd4HQjxllVMk= github.com/jpillora/go-tld v1.2.1/go.mod h1:plzIl7xr5UWKGy7R+giuv+L/nOjrPjsoWxy/ST9OBUk= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/internal/liteq b/internal/liteq new file mode 160000 index 0000000..5ba4ae4 --- /dev/null +++ b/internal/liteq @@ -0,0 +1 @@ +Subproject commit 5ba4ae43a399b9e7e1f702040141116b091ad00c