06cdc68be7
This integrates the liteq, and it prevents duplicates in a way that matches my use-case. I might try and push things back out to a separate module, but for now, this will do.
111 lines
2.2 KiB
Go
111 lines
2.2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"log"
|
|
"math/rand"
|
|
"os"
|
|
"sync"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
|
|
liteq "git.jadud.com/jadudm/grosbeak/internal/liteq"
|
|
)
|
|
|
|
type queueWorker func(ctx context.Context, job *liteq.Job) error
|
|
|
|
func Fetch(ctx context.Context, job *liteq.Job) error {
|
|
n := rand.Intn(50)
|
|
|
|
time.Sleep(time.Duration(n) * time.Millisecond)
|
|
log.Println("Fetching", job.Job)
|
|
return nil
|
|
}
|
|
|
|
func runQ(queue *liteq.JobQueue, queueName string, worker queueWorker) {
|
|
for {
|
|
err := queue.Consume(context.Background(), liteq.ConsumeParams{
|
|
Queue: queueName,
|
|
PoolSize: 3,
|
|
VisibilityTimeout: 20,
|
|
Worker: worker,
|
|
})
|
|
if err != nil {
|
|
log.Printf("runQ/%s: %w", queueName, err.Error())
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
time.Sleep(1 * time.Second)
|
|
}
|
|
}
|
|
|
|
func Entre(queue *liteq.JobQueue, chUrl <-chan string) {
|
|
ctx := context.Background()
|
|
for {
|
|
url := <-chUrl
|
|
n := time.Now()
|
|
ignore_tag := fmt.Sprintf("%s:%d:%d", url, n.Year(), n.YearDay())
|
|
log.Println("entre", url, ignore_tag)
|
|
// Don't duplicate jobs on the same day of the year.
|
|
err := queue.QueueJob(ctx, liteq.QueueJobParams{
|
|
Queue: "fetch",
|
|
// This only works for things in the `queued` state
|
|
DedupingKey: liteq.IgnoreDuplicate(ignore_tag),
|
|
Job: url,
|
|
})
|
|
if err != nil {
|
|
log.Println("entre err", err.Error())
|
|
}
|
|
}
|
|
}
|
|
|
|
func setupLiteQ() *liteq.JobQueue {
|
|
|
|
// FIXME: This path needs to come from the env.
|
|
liteqDB, err := sql.Open("sqlite", "liteq.db")
|
|
liteqDB.SetMaxOpenConns(1)
|
|
|
|
if err != nil {
|
|
fmt.Println(err)
|
|
os.Exit(1)
|
|
}
|
|
liteq.Setup(liteqDB)
|
|
queue := liteq.New(liteqDB)
|
|
// The queue processes as long as this context is not cancelled.
|
|
|
|
log.Println("Setting up worker queues...")
|
|
queues := []struct {
|
|
queueName string
|
|
worker queueWorker
|
|
}{
|
|
{"fetch", Fetch},
|
|
}
|
|
for _, q := range queues {
|
|
go runQ(queue, q.queueName, q.worker)
|
|
}
|
|
return queue
|
|
}
|
|
|
|
func main() {
|
|
// Don't let `main()` exit
|
|
wg := &sync.WaitGroup{}
|
|
wg.Add(1)
|
|
|
|
queue := setupLiteQ()
|
|
|
|
// Create the network for the search engine.
|
|
chUrl := make(chan string)
|
|
|
|
go Entre(queue, chUrl)
|
|
for range 5 {
|
|
chUrl <- "https://jadud.com/"
|
|
chUrl <- "https://berea.us/"
|
|
}
|
|
|
|
// Don't exit.
|
|
log.Println("Waiting for godot...")
|
|
wg.Wait()
|
|
}
|