f53639af2f
A lot more to go, but this is the core. Need to think about how to test the queue handlers.
35 lines
752 B
Go
35 lines
752 B
Go
package engine
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
|
|
"git.jadud.com/jadudm/grosbeak/internal/domain64"
|
|
"git.jadud.com/jadudm/grosbeak/internal/liteq"
|
|
"git.jadud.com/jadudm/grosbeak/internal/types"
|
|
"github.com/jpillora/go-tld"
|
|
"github.com/tidwall/gjson"
|
|
)
|
|
|
|
func Fetch(d64m *domain64.Domain64Map) types.QueueWorker {
|
|
_f := func(ctx context.Context, job *liteq.Job) error {
|
|
url := gjson.Get(job.Job, "url").String()
|
|
|
|
turl, err := tld.Parse(url)
|
|
if err != nil {
|
|
// If we can't parse it, shut the job down as completed.
|
|
return nil
|
|
}
|
|
|
|
d64, err := d64m.URLToDomain64(turl)
|
|
if err != nil {
|
|
log.Printf("urltodomain64 err %s: %s\n", url, err.Error())
|
|
}
|
|
|
|
log.Printf("fetching %s 0x%016x\n", url, d64.ToInt64())
|
|
return nil
|
|
}
|
|
|
|
return _f
|
|
}
|