package main import ( "context" "database/sql" "fmt" "log" "math/rand" "os" "sync" "time" _ "modernc.org/sqlite" liteq "git.jadud.com/jadudm/grosbeak/internal/liteq" ) type queueWorker func(ctx context.Context, job *liteq.Job) error func Fetch(ctx context.Context, job *liteq.Job) error { n := rand.Intn(50) time.Sleep(time.Duration(n) * time.Millisecond) log.Println("Fetching", job.Job) return nil } func runQ(queue *liteq.JobQueue, queueName string, worker queueWorker) { for { err := queue.Consume(context.Background(), liteq.ConsumeParams{ Queue: queueName, PoolSize: 3, VisibilityTimeout: 20, Worker: worker, }) if err != nil { log.Printf("runQ/%s: %w", queueName, err.Error()) time.Sleep(2 * time.Second) } time.Sleep(1 * time.Second) } } func Entre(queue *liteq.JobQueue, chUrl <-chan string) { ctx := context.Background() for { url := <-chUrl n := time.Now() ignore_tag := fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay()) log.Println("entre", url, ignore_tag) // Don't duplicate jobs on the same day of the year. err := queue.QueueJob(ctx, liteq.QueueJobParams{ Queue: "fetch", // This only works for things in the `queued` state DedupingKey: liteq.IgnoreDuplicate(ignore_tag), Job: url, }) if err != nil { log.Println("entre err", err.Error()) } } } func setupLiteQ() *liteq.JobQueue { // FIXME: This path needs to come from the env. liteqDB, err := sql.Open("sqlite", "liteq.db") liteqDB.SetMaxOpenConns(1) if err != nil { fmt.Println(err) os.Exit(1) } liteq.Setup(liteqDB) queue := liteq.New(liteqDB) // The queue processes as long as this context is not cancelled. log.Println("Setting up worker queues...") queues := []struct { queueName string worker queueWorker }{ {"fetch", Fetch}, } for _, q := range queues { go runQ(queue, q.queueName, q.worker) } return queue } func main() { // Don't let `main()` exit wg := &sync.WaitGroup{} wg.Add(1) queue := setupLiteQ() // Create the network for the search engine. chUrl := make(chan string) go Entre(queue, chUrl) for range 5 { chUrl <- "https://jadud.com/" chUrl <- "https://berea.us/" } // Don't exit. log.Println("Waiting for godot...") wg.Wait() }