Queueing, prepping to fetch

A lot more to go, but this is the core.

Need to think about how to test the queue handlers.

internal/engine/entre.go

@@ -0,0 +1,39 @@
+package engine
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"git.jadud.com/jadudm/grosbeak/internal/liteq"
+	base "git.jadud.com/jadudm/grosbeak/internal/types"
+)
+
+func Entre(queue *liteq.JobQueue, ej *base.EntreJob) error {
+	n := time.Now()
+
+	var ignore_tag string
+
+	switch ej.UpdateFrequency {
+	case base.UPDATE_DAILY:
+		ignore_tag = fmt.Sprintf("%s|y%d-yd%d", ej.URL, n.Year(), n.YearDay())
+	case base.UPDATE_WEEKLY:
+		ignore_tag = fmt.Sprintf("%s|y%d-w%d", ej.URL, n.Year(), n.YearDay()/7)
+	case base.UPDATE_MONTHLY:
+		ignore_tag = fmt.Sprintf("%s|y%d-m%d", ej.URL, n.Year(), n.Month())
+	default:
+		ignore_tag = fmt.Sprintf("%s|y%d-yd%d", ej.URL, n.Year(), n.YearDay())
+	}
+
+	err := queue.QueueJob(context.Background(), liteq.QueueJobParams{
+		Queue: "fetch",
+		// This only works for things in the `queued` state
+		DedupingKey: liteq.IgnoreDuplicate(ignore_tag),
+		Job:         ej.AsJson(),
+	})
+	if err != nil {
+		log.Printf("entre err %s: %s\n", ej.URL, err.Error())
+	}
+	return err
+}

internal/engine/fetch.go

@@ -0,0 +1,34 @@
+package engine
+
+import (
+	"context"
+	"log"
+
+	"git.jadud.com/jadudm/grosbeak/internal/domain64"
+	"git.jadud.com/jadudm/grosbeak/internal/liteq"
+	"git.jadud.com/jadudm/grosbeak/internal/types"
+	"github.com/jpillora/go-tld"
+	"github.com/tidwall/gjson"
+)
+
+func Fetch(d64m *domain64.Domain64Map) types.QueueWorker {
+	_f := func(ctx context.Context, job *liteq.Job) error {
+		url := gjson.Get(job.Job, "url").String()
+
+		turl, err := tld.Parse(url)
+		if err != nil {
+			// If we can't parse it, shut the job down as completed.
+			return nil
+		}
+
+		d64, err := d64m.URLToDomain64(turl)
+		if err != nil {
+			log.Printf("urltodomain64 err %s: %s\n", url, err.Error())
+		}
+
+		log.Printf("fetching %s 0x%016x\n", url, d64.ToInt64())
+		return nil
+	}
+
+	return _f
+}