Integrated, working.

This integrates the liteq, and it prevents duplicates in a way that
matches my use-case.

I might try and push things back out to a separate module, but for now,
this will do.

internal/liteq/bench/main.go

@@ -7,7 +7,7 @@ import (
 	"math/rand"
 	"time"
 
-	liteq "git.jadud.com/grosbeak/internal/liteq"
+	liteq "git.jadud.com/jadudm/grosbeak/internal/liteq"
 	_ "modernc.org/sqlite"
 )
 

internal/liteq/db/queries.sql

@@ -20,10 +20,9 @@ VALUES
         unixepoch(),
         ?,
         ?
-    ) ON CONFLICT (deduping_key, job_status)
+    ) ON CONFLICT (deduping_key)
 WHERE
-    deduping_key != ''
-    AND (job_status = 'queued' OR job_status = 'fetched' OR job_status = 'completed') DO NOTHING;
+    deduping_key != '' DO NOTHING;
 
 -- name: doQueueJobReplaceDupe :exec
 INSERT INTO

internal/liteq/db/schema.sql

@@ -21,10 +21,9 @@ WHERE
     job_status = 'queued'
     OR job_status = 'fetched';
 
-CREATE UNIQUE INDEX IF NOT EXISTS dedupe_ignore ON jobs (deduping_key, job_status)
+CREATE UNIQUE INDEX IF NOT EXISTS dedupe_ignore ON jobs (deduping_key)
 WHERE
-    deduping_key != ''
-    AND (job_status = 'queued' OR job_status = 'fetched' OR job_status = 'completed');
+    deduping_key != '';
 
 CREATE UNIQUE INDEX IF NOT EXISTS dedupe_replace ON jobs (deduping_key, job_status)
 WHERE

internal/liteq/internal/jobs_test.go

@@ -10,7 +10,7 @@ import (
 	"testing"
 	"time"
 
-	"git.jadud.com/grosbeak/internal/liteq/internal"
+	"git.jadud.com/jadudm/grosbeak/internal/liteq/internal"
 	"github.com/matryer/is"
 	_ "modernc.org/sqlite"
 )

internal/liteq/internal/methods.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log"
 	"time"
 
 	"database/sql/driver"
@@ -110,6 +111,7 @@ func (q *Queries) Consume(ctx context.Context, params ConsumeParams) error {
 	for {
 		// If the context gets canceled for example, stop consuming
 		if ctx.Err() != nil {
+			log.Println("context cancelled in Consume()")
 			return nil
 		}
 
@@ -120,6 +122,7 @@ func (q *Queries) Consume(ctx context.Context, params ConsumeParams) error {
 			})
 
 			if err != nil {
+				log.Println("error resetting jobs", err.Error())
 				return fmt.Errorf("error resetting jobs: %w", err)
 			}
 		}
@@ -143,6 +146,7 @@ func (q *Queries) Consume(ctx context.Context, params ConsumeParams) error {
 			workers.Submit(func() {
 				err := params.Worker(ctx, job)
 				if err != nil {
+					log.Println("worker failled", err.Error())
 					q.FailJob(ctx, FailJobParams{
 						ID:     job.ID,
 						Errors: ErrorList(append(job.Errors, err.Error())),

internal/liteq/internal/queries.sql.go

@@ -205,10 +205,9 @@ VALUES
         unixepoch(),
         ?,
         ?
-    ) ON CONFLICT (deduping_key, job_status)
+    ) ON CONFLICT (deduping_key)
 WHERE
-    deduping_key != ''
-    AND (job_status = 'queued' OR job_status = 'fetched' OR job_status = 'completed') DO NOTHING
+    deduping_key != '' DO NOTHING
 `
 
 type doQueueJobIgnoreDupeParams struct {

internal/liteq/internal/schema.go

@@ -25,10 +25,9 @@ WHERE
     job_status = 'queued'
     OR job_status = 'fetched';
 
-CREATE UNIQUE INDEX IF NOT EXISTS dedupe_ignore ON jobs (deduping_key, job_status)
+CREATE UNIQUE INDEX IF NOT EXISTS dedupe_ignore ON jobs (deduping_key)
 WHERE
-    deduping_key != ''
-    AND (job_status = 'queued' OR job_status = 'fetched' OR job_status = 'completed');
+    deduping_key != '';
 
 CREATE UNIQUE INDEX IF NOT EXISTS dedupe_replace ON jobs (deduping_key, job_status)
 WHERE

internal/liteq/liteq.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"database/sql"
 
-	internal "git.jadud.com/grosbeak/internal/liteq/internal"
+	internal "git.jadud.com/jadudm/grosbeak/internal/liteq/internal"
 )
 
 // Creates the db file with the tables and indexes