Integrated, working.

This integrates the liteq, and it prevents duplicates in a way that
matches my use-case.

I might try and push things back out to a separate module, but for now,
this will do.

cmd/api/main.go

@@ -12,9 +12,11 @@ import (
 
 	_ "modernc.org/sqlite"
 
-	liteq "git.jadud.com/grosbeak/internal/liteq"
+	liteq "git.jadud.com/jadudm/grosbeak/internal/liteq"
 )
 
+type queueWorker func(ctx context.Context, job *liteq.Job) error
+
 func Fetch(ctx context.Context, job *liteq.Job) error {
 	n := rand.Intn(50)
 
@@ -23,38 +25,48 @@ func Fetch(ctx context.Context, job *liteq.Job) error {
 	return nil
 }
 
-func runWorkers(queue *liteq.JobQueue) {
-	go queue.Consume(context.Background(), liteq.ConsumeParams{
-		Queue:             "fetch",
-		PoolSize:          3,
-		VisibilityTimeout: 200000,
-		Worker:            Fetch,
-	})
+func runQ(queue *liteq.JobQueue, queueName string, worker queueWorker) {
+	for {
+		err := queue.Consume(context.Background(), liteq.ConsumeParams{
+			Queue:             queueName,
+			PoolSize:          3,
+			VisibilityTimeout: 20,
+			Worker:            worker,
+		})
+		if err != nil {
+			log.Printf("runQ/%s: %w", queueName, err.Error())
+			time.Sleep(2 * time.Second)
+		}
+		time.Sleep(1 * time.Second)
+	}
 }
 
 func Entre(queue *liteq.JobQueue, chUrl <-chan string) {
 	ctx := context.Background()
 	for {
 		url := <-chUrl
-		log.Println("Entre", url)
-		// Don't duplicate jobs on the same day of the year.
 		n := time.Now()
-		queue.QueueJob(ctx, liteq.QueueJobParams{
+		ignore_tag := fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay())
+		log.Println("entre", url, ignore_tag)
+		// Don't duplicate jobs on the same day of the year.
+		err := queue.QueueJob(ctx, liteq.QueueJobParams{
 			Queue: "fetch",
 			// This only works for things in the `queued` state
-			DedupingKey: liteq.IgnoreDuplicate(fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay())),
+			DedupingKey: liteq.IgnoreDuplicate(ignore_tag),
 			Job:         url,
 		})
+		if err != nil {
+			log.Println("entre err", err.Error())
+		}
 	}
 }
 
-func main() {
-	// Don't let `main()` exit
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
+func setupLiteQ() *liteq.JobQueue {
 
 	// FIXME: This path needs to come from the env.
 	liteqDB, err := sql.Open("sqlite", "liteq.db")
+	liteqDB.SetMaxOpenConns(1)
+
 	if err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -63,22 +75,36 @@ func main() {
 	queue := liteq.New(liteqDB)
 	// The queue processes as long as this context is not cancelled.
 
-	log.Println("Setting up workers...")
-	go runWorkers(queue)
+	log.Println("Setting up worker queues...")
+	queues := []struct {
+		queueName string
+		worker    queueWorker
+	}{
+		{"fetch", Fetch},
+	}
+	for _, q := range queues {
+		go runQ(queue, q.queueName, q.worker)
+	}
+	return queue
+}
+
+func main() {
+	// Don't let `main()` exit
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	queue := setupLiteQ()
 
-	log.Println("Building network...")
 	// Create the network for the search engine.
 	chUrl := make(chan string)
 
 	go Entre(queue, chUrl)
-	for {
+	for range 5 {
 		chUrl <- "https://jadud.com/"
-		time.Sleep(2 * time.Second)
 		chUrl <- "https://berea.us/"
-		time.Sleep(2 * time.Second)
 	}
 
 	// Don't exit.
-	log.Println("Waiting...")
+	log.Println("Waiting for godot...")
 	wg.Wait()
 }