Adding liteq as a library

This is easier.

cmd/api/main.go

@@ -1,20 +1,84 @@
 package main
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"sync"
+	"time"
 
-	"git.jadud.com/grosbeak/internal/domain64"
+	_ "modernc.org/sqlite"
+
+	liteq "git.jadud.com/grosbeak/internal/liteq"
 )
 
-func main() {
+func Fetch(ctx context.Context, job *liteq.Job) error {
-	d64 := domain64.Domain64{
+	n := rand.Intn(50)
-		TLD:       1,
+
-		Domain:    1,
+	time.Sleep(time.Duration(n) * time.Millisecond)
-		Subdomain: 1,
+	log.Println("Fetching", job.Job)
-		Path:      1,
+	return nil
-	}
+}
-	result := d64.ToInt64()
+
-	fmt.Printf("%064b\n", result)
+func runWorkers(queue *liteq.JobQueue) {
-	fmt.Printf("%016x\n", result)
+	go queue.Consume(context.Background(), liteq.ConsumeParams{
-	fmt.Println(result)
+		Queue:             "fetch",
+		PoolSize:          3,
+		VisibilityTimeout: 200000,
+		Worker:            Fetch,
+	})
+}
+
+func Entre(queue *liteq.JobQueue, chUrl <-chan string) {
+	ctx := context.Background()
+	for {
+		url := <-chUrl
+		log.Println("Entre", url)
+		// Don't duplicate jobs on the same day of the year.
+		n := time.Now()
+		queue.QueueJob(ctx, liteq.QueueJobParams{
+			Queue: "fetch",
+			// This only works for things in the `queued` state
+			DedupingKey: liteq.IgnoreDuplicate(fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay())),
+			Job:         url,
+		})
+	}
+}
+
+func main() {
+	// Don't let `main()` exit
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	// FIXME: This path needs to come from the env.
+	liteqDB, err := sql.Open("sqlite", "liteq.db")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	liteq.Setup(liteqDB)
+	queue := liteq.New(liteqDB)
+	// The queue processes as long as this context is not cancelled.
+
+	log.Println("Setting up workers...")
+	go runWorkers(queue)
+
+	log.Println("Building network...")
+	// Create the network for the search engine.
+	chUrl := make(chan string)
+
+	go Entre(queue, chUrl)
+	for {
+		chUrl <- "https://jadud.com/"
+		time.Sleep(2 * time.Second)
+		chUrl <- "https://berea.us/"
+		time.Sleep(2 * time.Second)
+	}
+
+	// Don't exit.
+	log.Println("Waiting...")
+	wg.Wait()
 }

go.mod

@@ -10,15 +10,16 @@ require (
 )
 
 require (
+	github.com/alitto/pond v1.9.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
+	golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/sys v0.36.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
-	modernc.org/libc v1.66.10 // indirect
+	modernc.org/libc v1.67.1 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs=
+github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
@@ -6,8 +8,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/jpillora/go-tld v1.2.1 h1:kDKOkmXLlskqjcvNs7w5XHLep7c8WM7Xd4HQjxllVMk=
 github.com/jpillora/go-tld v1.2.1/go.mod h1:plzIl7xr5UWKGy7R+giuv+L/nOjrPjsoWxy/ST9OBUk=
+github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
+github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=