diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..99b6168 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +test: generate + cd internal/domain64 ; make test + +generate: + cd internal/domain64 ; make generate \ No newline at end of file diff --git a/README.md b/README.md index 5aa89c8..b91d7f8 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,13 @@ The use-case is (essentially) single-user. ### /fetch/ +## prereqs + +* go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest + + +## packages used + +* https://github.com/jpillora/go-tld +* https://pkg.go.dev/modernc.org/sqlite +* https://github.com/sqlc-dev/sqlc diff --git a/cmd/api/main.go b/cmd/api/main.go index 00f1e43..909f194 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -13,7 +13,7 @@ func main() { Subdomain: 1, Path: 1, } - result := d64.AsInt64() + result := d64.ToInt64() fmt.Printf("%064b\n", result) fmt.Printf("%016x\n", result) fmt.Println(result) diff --git a/go.mod b/go.mod index 55c7f19..d37ed0d 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,24 @@ module git.jadud.com/grosbeak -go 1.23.2 +go 1.24.0 + +toolchain go1.24.10 + +require ( + github.com/jpillora/go-tld v1.2.1 + modernc.org/sqlite v1.40.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/sys v0.36.0 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d2c73c3 --- /dev/null +++ b/go.sum @@ -0,0 +1,58 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jpillora/go-tld v1.2.1 h1:kDKOkmXLlskqjcvNs7w5XHLep7c8WM7Xd4HQjxllVMk= +github.com/jpillora/go-tld v1.2.1/go.mod h1:plzIl7xr5UWKGy7R+giuv+L/nOjrPjsoWxy/ST9OBUk= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.40.1 h1:VfuXcxcUWWKRBuP8+BR9L7VnmusMgBNNnBYGEe9w/iY= +modernc.org/sqlite v1.40.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/domain64/Makefile b/internal/domain64/Makefile new file mode 100644 index 0000000..5b431fc --- /dev/null +++ b/internal/domain64/Makefile @@ -0,0 +1,8 @@ +clean: + rm -rf db + +generate: clean + sqlc generate + +test: + go test *.go \ No newline at end of file diff --git a/internal/domain64/database.go b/internal/domain64/database.go new file mode 100644 index 0000000..a317615 --- /dev/null +++ b/internal/domain64/database.go @@ -0,0 +1,40 @@ +package domain64 + +import ( + "database/sql" + "sync" + + "github.com/jpillora/go-tld" +) + +type Domain64Map struct { + mu sync.Mutex + m map[string]int + DB *sql.DB + Flushed bool +} + +// Just use the D64 values. +// TLD :: 0-255 +// TLD.Domain :: FF:FFFFFF or some range of ints +// TLD.DOMAIN.SUBDOMAIN :: FF:FFFFFF:FF (bigger ints) +// now it is just a map +// https://stackoverflow.com/questions/40568759/sqlite-query-integer-field-based-on-a-bit + +func NewDomain64Map(db *sql.DB) (*Domain64Map, error) { + d64m := &Domain64Map{} + d64m.DB = db + d64m.Flushed = true + // Init the tables if a DB pointer is passed in. + return d64m, nil +} + +func (d64m *Domain64Map) Insert(url string) { + _, err := tld.Parse(url) + if err != nil { + // TODO + panic(err) + } + // If we have this TLD, return the value for it. + +} diff --git a/internal/domain64/database_test.go b/internal/domain64/database_test.go new file mode 100644 index 0000000..2a9a39d --- /dev/null +++ b/internal/domain64/database_test.go @@ -0,0 +1,41 @@ +package domain64 + +import ( + "context" + "database/sql" + _ "embed" + "testing" + + _ "modernc.org/sqlite" +) + +//go:embed schema.sql +var ddl string + +func setup() *sql.DB { + ctx := context.Background() + + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + // TODO + panic(err) + } + + // create tables + if _, err := db.ExecContext(ctx, ddl); err != nil { + panic(err) + } + return db +} + +func TestNewDomain64Map(t *testing.T) { + db := setup() + M, err := NewDomain64Map(db) + if err != nil { + // TODO + t.Error(err) + } + if M.DB == nil { + t.Error("DB should not be nil") + } +} diff --git a/internal/domain64/db.go b/internal/domain64/db.go new file mode 100644 index 0000000..84ac1cb --- /dev/null +++ b/internal/domain64/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package domain64 + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/domain64/domain64.go b/internal/domain64/domain64.go index 2d2b24f..448acc4 100644 --- a/internal/domain64/domain64.go +++ b/internal/domain64/domain64.go @@ -20,13 +20,31 @@ packet-beta 40-63: "FFFFFF | Path" ``` */ -func (d64 *Domain64) AsInt64() int64 { +const SHIFT_TLD = (64 - 8) +const SHIFT_DOMAIN = (64 - (8 + 24)) +const SHIFT_SUBDOMAIN = (64 - (8 + 24 + 8)) +const SHIFT_PATH = (64 - (8 + 24 + 8 + 24)) + +const MASK_TLD = 0xFF0000000000000 +const MASK_DOMAIN = 0x00FFFFFF00000000 +const MASK_SUBDOMAIN = 0x00000000FF000000 +const MASK_PATH = 0x0000000000FFFFFF + +func (d64 Domain64) ToInt64() int64 { var result int64 = 0 - result = result | (int64(d64.TLD) << (64 - 8)) - result = result | (int64(d64.Domain) << (64 - (8 + 24))) - result = result | (int64(d64.Subdomain) << (64 - (8 + 24 + 8))) - result = result | (int64(d64.Path) << (64 - (8 + 24 + 8 + 24))) + result = result | (int64(d64.TLD) << SHIFT_TLD) + result = result | (int64(d64.Domain) << SHIFT_DOMAIN) + result = result | (int64(d64.Subdomain) << SHIFT_SUBDOMAIN) + result = result | (int64(d64.Path) << SHIFT_PATH) return result } -// https://gobyexample.com/testing-and-benchmarking +func IntToDomain64(i int64) Domain64 { + d64 := Domain64{} + + d64.TLD = uint8((i & MASK_TLD) >> SHIFT_TLD) + d64.Domain = uint16((i & MASK_DOMAIN) >> SHIFT_DOMAIN) + d64.Subdomain = uint8((i & MASK_SUBDOMAIN) >> SHIFT_SUBDOMAIN) + d64.Path = uint16(i & MASK_PATH) + return d64 +} diff --git a/internal/domain64/domain64_test.go b/internal/domain64/domain64_test.go new file mode 100644 index 0000000..13ca4bd --- /dev/null +++ b/internal/domain64/domain64_test.go @@ -0,0 +1,45 @@ +package domain64 + +import ( + "testing" +) + +// https://gobyexample.com/testing-and-benchmarking + +// For testing the conversion between a Domain64 struct +// and the int64 representation of that domain. +var tests = []struct { + d64 Domain64 + asInt int64 +}{ + { + Domain64{ + TLD: 1, + Domain: 1, + Subdomain: 1, + Path: 1, + }, 72057598349672449}, + { + Domain64{ + TLD: 2, + Domain: 1, + Subdomain: 1, + Path: 0, + }, 144115192387600384}, +} + +func TestDomain64ToInt(t *testing.T) { + for _, tt := range tests { + if tt.d64.ToInt64() != tt.asInt { + t.Errorf("%q != %d Domain64 did not convert", tt.d64, tt.asInt) + } + } +} + +func TestIntToDomain64(t *testing.T) { + for _, tt := range tests { + if IntToDomain64(tt.asInt) != tt.d64 { + t.Errorf("%d != %q int64 did not convert", tt.asInt, tt.d64) + } + } +} diff --git a/internal/domain64/query.sql b/internal/domain64/query.sql new file mode 100644 index 0000000..1175a43 --- /dev/null +++ b/internal/domain64/query.sql @@ -0,0 +1,12 @@ +-- name: InsertDomain64 :exec +INSERT INTO d64 + (url, d64, tld_id, domain_id, subdomain_id, path_id) + VALUES + (?, ?, ?, ?, ?, ?); + +-- name: GetTLD :one +SELECT id from domain64 + WHERE tld_id = ? + +-- name: CountTLD :one +SELECT COUNT(*) FROM domain64 WHERE tld_id = ?; \ No newline at end of file diff --git a/internal/domain64/schema.sql b/internal/domain64/schema.sql new file mode 100644 index 0000000..386093c --- /dev/null +++ b/internal/domain64/schema.sql @@ -0,0 +1,9 @@ +CREATE TABLE domain64 ( + id INT PRIMARY KEY, + url TEXT, + d64 BIGINT, + tld_id INT, + domain_id INT, + subdomain_id INT, + path_id INT +) diff --git a/internal/domain64/sqlc.yaml b/internal/domain64/sqlc.yaml new file mode 100644 index 0000000..1a8db1e --- /dev/null +++ b/internal/domain64/sqlc.yaml @@ -0,0 +1,9 @@ +version: "2" +sql: + - engine: "sqlite" + queries: "query.sql" + schema: "schema.sql" + gen: + go: + package: "db" + out: "db" \ No newline at end of file