Interim while bringing sqlc in

Makefile

@@ -0,0 +1,5 @@
+test: generate
+	cd internal/domain64 ; make test
+
+generate:
+	cd internal/domain64 ; make generate

README.md

@@ -22,3 +22,13 @@ The use-case is (essentially) single-user.
 
 ### /fetch/<b64:URL>
 
+## prereqs
+
+* go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
+
+
+## packages used
+
+* https://github.com/jpillora/go-tld
+* https://pkg.go.dev/modernc.org/sqlite
+* https://github.com/sqlc-dev/sqlc

cmd/api/main.go

@@ -13,7 +13,7 @@ func main() {
 		Subdomain: 1,
 		Path:      1,
 	}
-	result := d64.AsInt64()
+	result := d64.ToInt64()
 	fmt.Printf("%064b\n", result)
 	fmt.Printf("%016x\n", result)
 	fmt.Println(result)

go.mod

@@ -1,3 +1,24 @@
 module git.jadud.com/grosbeak
 
-go 1.23.2
+go 1.24.0
+
+toolchain go1.24.10
+
+require (
+	github.com/jpillora/go-tld v1.2.1
+	modernc.org/sqlite v1.40.1
+)
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/sys v0.36.0 // indirect
+	modernc.org/libc v1.66.10 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

go.sum

@@ -0,0 +1,58 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jpillora/go-tld v1.2.1 h1:kDKOkmXLlskqjcvNs7w5XHLep7c8WM7Xd4HQjxllVMk=
+github.com/jpillora/go-tld v1.2.1/go.mod h1:plzIl7xr5UWKGy7R+giuv+L/nOjrPjsoWxy/ST9OBUk=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
+golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
+golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
+modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4=
+modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A=
+modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q=
+modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
+modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A=
+modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY=
+modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/domain64/Makefile

@@ -0,0 +1,8 @@
+clean:
+	rm -rf db
+	
+generate: clean
+	sqlc generate
+
+test:
+	go test *.go

internal/domain64/database.go

@@ -0,0 +1,40 @@
+package domain64
+
+import (
+	"database/sql"
+	"sync"
+
+	"github.com/jpillora/go-tld"
+)
+
+type Domain64Map struct {
+	mu      sync.Mutex
+	m       map[string]int
+	DB      *sql.DB
+	Flushed bool
+}
+
+// Just use the D64 values.
+// TLD :: 0-255
+// TLD.Domain :: FF:FFFFFF or some range of ints
+// TLD.DOMAIN.SUBDOMAIN :: FF:FFFFFF:FF (bigger ints)
+// now it is just a map
+// https://stackoverflow.com/questions/40568759/sqlite-query-integer-field-based-on-a-bit
+
+func NewDomain64Map(db *sql.DB) (*Domain64Map, error) {
+	d64m := &Domain64Map{}
+	d64m.DB = db
+	d64m.Flushed = true
+	// Init the tables if a DB pointer is passed in.
+	return d64m, nil
+}
+
+func (d64m *Domain64Map) Insert(url string) {
+	_, err := tld.Parse(url)
+	if err != nil {
+		// TODO
+		panic(err)
+	}
+	// If we have this TLD, return the value for it.
+
+}

internal/domain64/database_test.go

@@ -0,0 +1,41 @@
+package domain64
+
+import (
+	"context"
+	"database/sql"
+	_ "embed"
+	"testing"
+
+	_ "modernc.org/sqlite"
+)
+
+//go:embed schema.sql
+var ddl string
+
+func setup() *sql.DB {
+	ctx := context.Background()
+
+	db, err := sql.Open("sqlite", ":memory:")
+	if err != nil {
+		// TODO
+		panic(err)
+	}
+
+	// create tables
+	if _, err := db.ExecContext(ctx, ddl); err != nil {
+		panic(err)
+	}
+	return db
+}
+
+func TestNewDomain64Map(t *testing.T) {
+	db := setup()
+	M, err := NewDomain64Map(db)
+	if err != nil {
+		// TODO
+		t.Error(err)
+	}
+	if M.DB == nil {
+		t.Error("DB should not be nil")
+	}
+}

internal/domain64/db.go

@@ -0,0 +1,31 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.30.0
+
+package domain64
+
+import (
+	"context"
+	"database/sql"
+)
+
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+func New(db DBTX) *Queries {
+	return &Queries{db: db}
+}
+
+type Queries struct {
+	db DBTX
+}
+
+func (q *Queries) WithTx(tx *sql.Tx) *Queries {
+	return &Queries{
+		db: tx,
+	}
+}

internal/domain64/domain64.go

@@ -20,13 +20,31 @@ packet-beta
 40-63: "FFFFFF | Path"
 ```
 */
-func (d64 *Domain64) AsInt64() int64 {
+const SHIFT_TLD = (64 - 8)
+const SHIFT_DOMAIN = (64 - (8 + 24))
+const SHIFT_SUBDOMAIN = (64 - (8 + 24 + 8))
+const SHIFT_PATH = (64 - (8 + 24 + 8 + 24))
+
+const MASK_TLD = 0xFF0000000000000
+const MASK_DOMAIN = 0x00FFFFFF00000000
+const MASK_SUBDOMAIN = 0x00000000FF000000
+const MASK_PATH = 0x0000000000FFFFFF
+
+func (d64 Domain64) ToInt64() int64 {
 	var result int64 = 0
-	result = result | (int64(d64.TLD) << (64 - 8))
+	result = result | (int64(d64.TLD) << SHIFT_TLD)
-	result = result | (int64(d64.Domain) << (64 - (8 + 24)))
+	result = result | (int64(d64.Domain) << SHIFT_DOMAIN)
-	result = result | (int64(d64.Subdomain) << (64 - (8 + 24 + 8)))
+	result = result | (int64(d64.Subdomain) << SHIFT_SUBDOMAIN)
-	result = result | (int64(d64.Path) << (64 - (8 + 24 + 8 + 24)))
+	result = result | (int64(d64.Path) << SHIFT_PATH)
 	return result
 }
 
-// https://gobyexample.com/testing-and-benchmarking
+func IntToDomain64(i int64) Domain64 {
+	d64 := Domain64{}
+
+	d64.TLD = uint8((i & MASK_TLD) >> SHIFT_TLD)
+	d64.Domain = uint16((i & MASK_DOMAIN) >> SHIFT_DOMAIN)
+	d64.Subdomain = uint8((i & MASK_SUBDOMAIN) >> SHIFT_SUBDOMAIN)
+	d64.Path = uint16(i & MASK_PATH)
+	return d64
+}

internal/domain64/domain64_test.go

@@ -0,0 +1,45 @@
+package domain64
+
+import (
+	"testing"
+)
+
+// https://gobyexample.com/testing-and-benchmarking
+
+// For testing the conversion between a Domain64 struct
+// and the int64 representation of that domain.
+var tests = []struct {
+	d64   Domain64
+	asInt int64
+}{
+	{
+		Domain64{
+			TLD:       1,
+			Domain:    1,
+			Subdomain: 1,
+			Path:      1,
+		}, 72057598349672449},
+	{
+		Domain64{
+			TLD:       2,
+			Domain:    1,
+			Subdomain: 1,
+			Path:      0,
+		}, 144115192387600384},
+}
+
+func TestDomain64ToInt(t *testing.T) {
+	for _, tt := range tests {
+		if tt.d64.ToInt64() != tt.asInt {
+			t.Errorf("%q != %d Domain64 did not convert", tt.d64, tt.asInt)
+		}
+	}
+}
+
+func TestIntToDomain64(t *testing.T) {
+	for _, tt := range tests {
+		if IntToDomain64(tt.asInt) != tt.d64 {
+			t.Errorf("%d != %q int64 did not convert", tt.asInt, tt.d64)
+		}
+	}
+}

internal/domain64/query.sql

@@ -0,0 +1,12 @@
+-- name: InsertDomain64 :exec
+INSERT INTO d64 
+  (url, d64, tld_id, domain_id, subdomain_id, path_id) 
+  VALUES 
+  (?, ?, ?, ?, ?, ?);
+
+-- name: GetTLD :one
+SELECT id from domain64
+  WHERE tld_id = ?
+
+-- name: CountTLD :one
+SELECT COUNT(*) FROM domain64 WHERE tld_id = ?;

internal/domain64/schema.sql

@@ -0,0 +1,9 @@
+CREATE TABLE domain64 (
+  id INT PRIMARY KEY,
+  url TEXT,
+  d64 BIGINT,
+  tld_id INT,
+  domain_id INT,
+  subdomain_id INT,
+  path_id INT
+)

internal/domain64/sqlc.yaml

@@ -0,0 +1,9 @@
+version: "2"
+sql:
+  - engine: "sqlite"
+    queries: "query.sql"
+    schema: "schema.sql"
+    gen:
+      go:
+        package: "db"
+        out: "db"