From 10b3083647e90968dff2d1c5d73280515235ac20 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 2 Nov 2024 20:29:32 +0000 Subject: [PATCH 01/12] WIP Add db dumper tool --- devtools/store/main.go | 64 ++++++++++++++++++++++++++++++++++++++++ go.mod | 4 +-- go.sum | 4 +-- internal/storage/gdbm.go | 4 +++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 devtools/store/main.go diff --git a/devtools/store/main.go b/devtools/store/main.go new file mode 100644 index 0000000..df27bb1 --- /dev/null +++ b/devtools/store/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path" + + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/storage" + "git.defalsify.org/vise.git/logging" +) + +var ( + logg = logging.NewVanilla() + scriptDir = path.Join("services", "registration") +) + +func init() { + initializers.LoadEnvVariables() +} + +func main() { + config.LoadConfig() + + var dbDir string + var sessionId string + var database string + var engineDebug bool + flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") + flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.BoolVar(&engineDebug, "d", false, "use engine debug output") + flag.Parse() + + ctx := context.Background() + //ctx = context.WithValue(ctx, "SessionId", sessionId) + ctx = context.WithValue(ctx, "Database", database) + + resourceDir := scriptDir + menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + + store, err := menuStorageService.GetUserdataDb(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + d, err := store.Dump(ctx, []byte(sessionId)) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + for true { + k, v := d.Next(ctx) + if k == nil { + break + } + fmt.Printf("%x %s\n", k, v) + } +} diff --git a/go.mod b/go.mod index 391c1a5..e30d42f 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,14 @@ go 1.23.0 toolchain go1.23.2 require ( - git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b + git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045 github.com/alecthomas/assert/v2 v2.2.2 github.com/grassrootseconomics/eth-custodial v1.3.0-beta github.com/peteole/testdata-loader v0.3.0 gopkg.in/leonelquinteros/gotext.v1 v1.3.1 ) -require github.com/grassrootseconomics/ussd-data-service v0.0.0-20241003123429-4904b4438a3a // indirect +require github.com/grassrootseconomics/ussd-data-service v0.0.0-20241003123429-4904b4438a3a require ( github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0ba38c1..21dbfb2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b h1:dxBplsIlzJHV+5EH+gzB+w08Blt7IJbb2jeRe1OEjLU= -git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045 h1:uFFVEymnsJGVcuI2r//oNAqDD8l0HOkkEvqpOJHdJiY= +git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= diff --git a/internal/storage/gdbm.go b/internal/storage/gdbm.go index 49de570..adb1d0f 100644 --- a/internal/storage/gdbm.go +++ b/internal/storage/gdbm.go @@ -114,3 +114,7 @@ func(tdb *ThreadGdbmDb) Close() error { tdb.db = nil return err } + +func(tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) { + return tdb.db.Dump(ctx, key) +} From 1a782c1db9b9d0339435f6e47d8c20e73ff23b7a Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 1 Dec 2024 17:32:06 +0000 Subject: [PATCH 02/12] Add debug package --- debug/db.go | 47 ++++++++++++++++++++++++++++++++++++++++++ debug/db_no.go | 14 +++++++++++++ debug/db_yes.go | 42 +++++++++++++++++++++++++++++++++++++ devtools/store/main.go | 3 ++- 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 debug/db.go create mode 100644 debug/db_no.go create mode 100644 debug/db_yes.go diff --git a/debug/db.go b/debug/db.go new file mode 100644 index 0000000..9efad00 --- /dev/null +++ b/debug/db.go @@ -0,0 +1,47 @@ +package debug + +import ( + "fmt" + "encoding/binary" + + "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/common" +) + +type KeyInfo struct { + SessionId string + Typ uint8 + SubTyp common.DataTyp + Label string + Description string +} + +func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { + o := KeyInfo{} + b := []byte(sessionId) + if len(k) <= len(b) { + return o, fmt.Errorf("storage key missing") + } + + o.SessionId = sessionId + + k = k[len(b):] + o.Typ = k[0] + k = k[1:] + + if o.Typ == storage.DATATYPE_USERSUB { + if len(k) == 0 { + return o, fmt.Errorf("missing subtype key") + } + v := binary.BigEndian.Uint16(k[:2]) + o.SubTyp = common.DataTyp(v) + o.Label = typToString(o.SubTyp) + k = k[2:] + } + + if len(k) != 0 { + return o, fmt.Errorf("excess key information") + } + + return o, nil +} diff --git a/debug/db_no.go b/debug/db_no.go new file mode 100644 index 0000000..a01c78f --- /dev/null +++ b/debug/db_no.go @@ -0,0 +1,14 @@ +// +build !debugdb + +package debug + +import ( + "fmt" + + "git.grassecon.net/urdt/ussd/common" +) + + +func typToString(v common.DataTyp) string { + return fmt.Sprintf("(%d)", v) +} diff --git a/debug/db_yes.go b/debug/db_yes.go new file mode 100644 index 0000000..fa8186e --- /dev/null +++ b/debug/db_yes.go @@ -0,0 +1,42 @@ +// +build debugdb + +package debug + +import ( + "git.defalsify.org/vise.git/db" + + "git.grassecon.net/urdt/ussd/common" +) + +var ( + dbTypStr map[common.DataTyp]string = map[common.DataTyp]string { + db.DATATYPE_USERDATA: "userdata", + common.DATA_ACCOUNT: "account", + common.DATA_ACCOUNT_CREATED: "account created", + common.DATA_TRACKING_ID: "tracking id", + common.DATA_PUBLIC_KEY: "public key", + common.DATA_CUSTODIAL_ID: "custodial id", + common.DATA_ACCOUNT_PIN: "account pin", + common.DATA_ACCOUNT_STATUS: "account status", + common.DATA_FIRST_NAME: "first name", + common.DATA_FAMILY_NAME: "family name", + common.DATA_YOB: "year of birth", + common.DATA_LOCATION: "location", + common.DATA_GENDER: "gender", + common.DATA_OFFERINGS: "offerings", + common.DATA_RECIPIENT: "recipient", + common.DATA_AMOUNT: "amount", + common.DATA_TEMPORARY_VALUE: "temporary value", + common.DATA_ACTIVE_SYM: "active sym", + common.DATA_ACTIVE_BAL: "active bal", + common.DATA_BLOCKED_NUMBER: "blocked number", + common.DATA_PUBLIC_KEY_REVERSE: "public key reverse", + common.DATA_ACTIVE_DECIMAL: "active decimal", + common.DATA_ACTIVE_ADDRESS: "active address", + common.DATA_TRANSACTIONS: "transactions", + } +) + +func typToString(v common.DataTyp) string { + return dbTypStr[v] +} diff --git a/devtools/store/main.go b/devtools/store/main.go index df27bb1..35587ea 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -22,6 +22,7 @@ func init() { initializers.LoadEnvVariables() } + func main() { config.LoadConfig() @@ -59,6 +60,6 @@ func main() { if k == nil { break } - fmt.Printf("%x %s\n", k, v) + fmt.Printf("%x %s %x\n", k, v) } } From 35cf3a1cd1d0586aadbb0dc68c43bab09a966bae Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 1 Dec 2024 18:41:28 +0000 Subject: [PATCH 03/12] Add debug label test, capability debug flag --- debug/cap.go | 5 +++++ debug/db_yes.go | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 debug/cap.go diff --git a/debug/cap.go b/debug/cap.go new file mode 100644 index 0000000..458bb48 --- /dev/null +++ b/debug/cap.go @@ -0,0 +1,5 @@ +package debug + +var ( + DebugCap uint32 +) diff --git a/debug/db_yes.go b/debug/db_yes.go index fa8186e..c323366 100644 --- a/debug/db_yes.go +++ b/debug/db_yes.go @@ -37,6 +37,10 @@ var ( } ) +func init() { + DebugCap |= 1 +} + func typToString(v common.DataTyp) string { return dbTypStr[v] } From 6d4f3109f8b312dc060f4c8c9d2b6f4791427b1a Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 1 Dec 2024 23:12:58 +0000 Subject: [PATCH 04/12] Consolidate subtyp and typ debug --- debug/db.go | 17 +++++++++++++++- debug/db_debug.go | 39 ++++++++++++++++++++++++++++++++++++ debug/db_no.go | 14 ------------- debug/db_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++ debug/db_yes.go | 46 ------------------------------------------- 5 files changed, 105 insertions(+), 61 deletions(-) create mode 100644 debug/db_debug.go delete mode 100644 debug/db_no.go create mode 100644 debug/db_test.go delete mode 100644 debug/db_yes.go diff --git a/debug/db.go b/debug/db.go index 9efad00..ccc8630 100644 --- a/debug/db.go +++ b/debug/db.go @@ -8,6 +8,10 @@ import ( "git.grassecon.net/urdt/ussd/common" ) +var ( + dbTypStr map[common.DataTyp]string = make(map[common.DataTyp]string) +) + type KeyInfo struct { SessionId string Typ uint8 @@ -19,6 +23,7 @@ type KeyInfo struct { func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { o := KeyInfo{} b := []byte(sessionId) + if len(k) <= len(b) { return o, fmt.Errorf("storage key missing") } @@ -35,8 +40,10 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { } v := binary.BigEndian.Uint16(k[:2]) o.SubTyp = common.DataTyp(v) - o.Label = typToString(o.SubTyp) + o.Label = subTypToString(o.SubTyp) k = k[2:] + } else { + o.Label = typToString(o.Typ) } if len(k) != 0 { @@ -45,3 +52,11 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { return o, nil } + +func subTypToString(v common.DataTyp) string { + return dbTypStr[v + storage.DATATYPE_USERSUB + 1] +} + +func typToString(v uint8) string { + return dbTypStr[common.DataTyp(uint16(v))] +} diff --git a/debug/db_debug.go b/debug/db_debug.go new file mode 100644 index 0000000..e4a3949 --- /dev/null +++ b/debug/db_debug.go @@ -0,0 +1,39 @@ +// +build debugdb + +package debug + +import ( + "git.defalsify.org/vise.git/db" + + "git.grassecon.net/urdt/ussd/common" + "git.grassecon.net/urdt/ussd/internal/storage" +) + +func init() { + DebugCap |= 1 + dbTypStr[db.DATATYPE_STATE] = "internal_state" + dbTypStr[db.DATATYPE_USERDATA] = "userdata" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT] = "account" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_CREATED] = "account_created" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRACKING_ID] = "tracking id" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY] = "public key" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_CUSTODIAL_ID] = "custodial id" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_PIN] = "account pin" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_STATUS] = "account status" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FIRST_NAME] = "first name" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FAMILY_NAME] = "family name" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_YOB] = "year of birth" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_LOCATION] = "location" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_GENDER] = "gender" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_OFFERINGS] = "offerings" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_RECIPIENT] = "recipient" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_AMOUNT] = "amount" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_SYM] = "active sym" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_BAL] = "active bal" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_ADDRESS] = "active address" + dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRANSACTIONS] = "transactions" +} diff --git a/debug/db_no.go b/debug/db_no.go deleted file mode 100644 index a01c78f..0000000 --- a/debug/db_no.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !debugdb - -package debug - -import ( - "fmt" - - "git.grassecon.net/urdt/ussd/common" -) - - -func typToString(v common.DataTyp) string { - return fmt.Sprintf("(%d)", v) -} diff --git a/debug/db_test.go b/debug/db_test.go new file mode 100644 index 0000000..2dae1e8 --- /dev/null +++ b/debug/db_test.go @@ -0,0 +1,50 @@ +package debug + +import ( + "testing" +) + +func TestDebugDbSubKeyInfo(t *testing.T) { + s := "foo" + b := []byte(s) + b = append(b, []byte{0x40, 0x00, 0x02}...) + r, err := ToKeyInfo(b, s) + if err != nil { + t.Fatal(err) + } + if r.SessionId != s { + t.Fatalf("expected %s, got %s", s, r.SessionId) + } + if r.Typ != 64 { + t.Fatalf("expected 64, got %d", r.Typ) + } + if r.SubTyp != 2 { + t.Fatalf("expected 2, got %d", r.SubTyp) + } + if DebugCap & 1 > 0 { + if r.Label != "tracking id" { + t.Fatalf("expected 'tracking id', got '%s'", r.Label) + } + } +} + +func TestDebugDbKeyInfo(t *testing.T) { + s := "bar" + b := []byte(s) + b = append(b, []byte{0x20}...) + r, err := ToKeyInfo(b, s) + if err != nil { + t.Fatal(err) + } + if r.SessionId != s { + t.Fatalf("expected %s, got %s", s, r.SessionId) + } + if r.Typ != 32 { + t.Fatalf("expected 64, got %d", r.Typ) + } + if DebugCap & 1 > 0 { + if r.Label != "userdata" { + t.Fatalf("expected 'userdata', got '%s'", r.Label) + } + } +} diff --git a/debug/db_yes.go b/debug/db_yes.go deleted file mode 100644 index c323366..0000000 --- a/debug/db_yes.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build debugdb - -package debug - -import ( - "git.defalsify.org/vise.git/db" - - "git.grassecon.net/urdt/ussd/common" -) - -var ( - dbTypStr map[common.DataTyp]string = map[common.DataTyp]string { - db.DATATYPE_USERDATA: "userdata", - common.DATA_ACCOUNT: "account", - common.DATA_ACCOUNT_CREATED: "account created", - common.DATA_TRACKING_ID: "tracking id", - common.DATA_PUBLIC_KEY: "public key", - common.DATA_CUSTODIAL_ID: "custodial id", - common.DATA_ACCOUNT_PIN: "account pin", - common.DATA_ACCOUNT_STATUS: "account status", - common.DATA_FIRST_NAME: "first name", - common.DATA_FAMILY_NAME: "family name", - common.DATA_YOB: "year of birth", - common.DATA_LOCATION: "location", - common.DATA_GENDER: "gender", - common.DATA_OFFERINGS: "offerings", - common.DATA_RECIPIENT: "recipient", - common.DATA_AMOUNT: "amount", - common.DATA_TEMPORARY_VALUE: "temporary value", - common.DATA_ACTIVE_SYM: "active sym", - common.DATA_ACTIVE_BAL: "active bal", - common.DATA_BLOCKED_NUMBER: "blocked number", - common.DATA_PUBLIC_KEY_REVERSE: "public key reverse", - common.DATA_ACTIVE_DECIMAL: "active decimal", - common.DATA_ACTIVE_ADDRESS: "active address", - common.DATA_TRANSACTIONS: "transactions", - } -) - -func init() { - DebugCap |= 1 -} - -func typToString(v common.DataTyp) string { - return dbTypStr[v] -} From c9deca11805f82c023d7026e8b37978ab319566c Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 2 Dec 2024 14:36:33 +0000 Subject: [PATCH 05/12] change userdata prefix on subprefix debug --- debug/db_debug.go | 47 +++++++++++++++++++++--------------------- devtools/store/main.go | 6 ++++++ 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/debug/db_debug.go b/debug/db_debug.go index e4a3949..7c17eed 100644 --- a/debug/db_debug.go +++ b/debug/db_debug.go @@ -12,28 +12,27 @@ import ( func init() { DebugCap |= 1 dbTypStr[db.DATATYPE_STATE] = "internal_state" - dbTypStr[db.DATATYPE_USERDATA] = "userdata" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT] = "account" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_CREATED] = "account_created" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRACKING_ID] = "tracking id" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY] = "public key" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_CUSTODIAL_ID] = "custodial id" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_PIN] = "account pin" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_STATUS] = "account status" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FIRST_NAME] = "first name" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FAMILY_NAME] = "family name" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_YOB] = "year of birth" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_LOCATION] = "location" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_GENDER] = "gender" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_OFFERINGS] = "offerings" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_RECIPIENT] = "recipient" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_AMOUNT] = "amount" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_SYM] = "active sym" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_BAL] = "active bal" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_ADDRESS] = "active address" - dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRANSACTIONS] = "transactions" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account_created" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address" + dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRANSACTIONS] = "transactions" } diff --git a/devtools/store/main.go b/devtools/store/main.go index 35587ea..d6abec9 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -62,4 +62,10 @@ func main() { } fmt.Printf("%x %s %x\n", k, v) } + + err = store.Close() + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } } From 3416fdf50c2ff82fadf9b4cc5afcf38b149de9c6 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 7 Dec 2024 23:06:03 +0000 Subject: [PATCH 06/12] Add keyinfo restore --- common/user_store.go | 4 +-- debug/db.go | 22 ++++++++++++---- debug/db_debug.go | 60 +++++++++++++++++++++++++------------------ debug/db_test.go | 44 ++++++++++++++++++++++++++------ devtools/gen/main.go | 61 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 40 deletions(-) create mode 100644 devtools/gen/main.go diff --git a/common/user_store.go b/common/user_store.go index 29796e2..6c770d8 100644 --- a/common/user_store.go +++ b/common/user_store.go @@ -20,7 +20,7 @@ type UserDataStore struct { func (store *UserDataStore) ReadEntry(ctx context.Context, sessionId string, typ DataTyp) ([]byte, error) { store.SetPrefix(db.DATATYPE_USERDATA) store.SetSession(sessionId) - k := PackKey(typ, []byte(sessionId)) + k := ToBytes(typ) return store.Get(ctx, k) } @@ -29,6 +29,6 @@ func (store *UserDataStore) ReadEntry(ctx context.Context, sessionId string, typ func (store *UserDataStore) WriteEntry(ctx context.Context, sessionId string, typ DataTyp, value []byte) error { store.SetPrefix(db.DATATYPE_USERDATA) store.SetSession(sessionId) - k := PackKey(typ, []byte(sessionId)) + k := ToBytes(typ) return store.Put(ctx, k, value) } diff --git a/debug/db.go b/debug/db.go index ccc8630..eb62a41 100644 --- a/debug/db.go +++ b/debug/db.go @@ -4,8 +4,8 @@ import ( "fmt" "encoding/binary" - "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/common" + "git.defalsify.org/vise.git/db" ) var ( @@ -30,11 +30,12 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { o.SessionId = sessionId - k = k[len(b):] - o.Typ = k[0] + o.Typ = uint8(k[0]) k = k[1:] + o.SessionId = string(k[:len(b)]) + k = k[len(b):] - if o.Typ == storage.DATATYPE_USERSUB { + if o.Typ == db.DATATYPE_USERDATA { if len(k) == 0 { return o, fmt.Errorf("missing subtype key") } @@ -53,8 +54,19 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { return o, nil } +func FromKey(k []byte) (KeyInfo, error) { + o := KeyInfo{} + + if len(k) < 4 { + return o, fmt.Errorf("insufficient key length") + } + + sessionIdBytes := k[1:len(k)-2] + return ToKeyInfo(k, string(sessionIdBytes)) +} + func subTypToString(v common.DataTyp) string { - return dbTypStr[v + storage.DATATYPE_USERSUB + 1] + return dbTypStr[v + db.DATATYPE_USERDATA + 1] } func typToString(v uint8) string { diff --git a/debug/db_debug.go b/debug/db_debug.go index 7c17eed..ed2dd66 100644 --- a/debug/db_debug.go +++ b/debug/db_debug.go @@ -6,33 +6,43 @@ import ( "git.defalsify.org/vise.git/db" "git.grassecon.net/urdt/ussd/common" - "git.grassecon.net/urdt/ussd/internal/storage" ) func init() { DebugCap |= 1 - dbTypStr[db.DATATYPE_STATE] = "internal_state" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account_created" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address" - dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRANSACTIONS] = "transactions" + dbTypStr[db.DATATYPE_STATE] = "internal state" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account created" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_SYMBOLS] = "voucher symbols" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_BALANCES] = "voucher balances" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_DECIMALS] = "voucher decimals" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_ADDRESSES] = "voucher addresses" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_SENDERS] = "tx senders" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_RECIPIENTS] = "tx recipients" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_VALUES] = "tx values" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_ADDRESSES] = "tx addresses" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_HASHES] = "tx hashes" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_DATES] = "tx dates" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_SYMBOLS] = "tx symbols" + dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_DECIMALS] = "tx decimals" } diff --git a/debug/db_test.go b/debug/db_test.go index 2dae1e8..25f781d 100644 --- a/debug/db_test.go +++ b/debug/db_test.go @@ -2,12 +2,16 @@ package debug import ( "testing" + + "git.grassecon.net/urdt/ussd/common" + "git.defalsify.org/vise.git/db" ) func TestDebugDbSubKeyInfo(t *testing.T) { s := "foo" - b := []byte(s) - b = append(b, []byte{0x40, 0x00, 0x02}...) + b := []byte{0x20} + b = append(b, []byte(s)...) + b = append(b, []byte{0x00, 0x02}...) r, err := ToKeyInfo(b, s) if err != nil { t.Fatal(err) @@ -15,7 +19,7 @@ func TestDebugDbSubKeyInfo(t *testing.T) { if r.SessionId != s { t.Fatalf("expected %s, got %s", s, r.SessionId) } - if r.Typ != 64 { + if r.Typ != 32 { t.Fatalf("expected 64, got %d", r.Typ) } if r.SubTyp != 2 { @@ -30,8 +34,32 @@ func TestDebugDbSubKeyInfo(t *testing.T) { func TestDebugDbKeyInfo(t *testing.T) { s := "bar" - b := []byte(s) - b = append(b, []byte{0x20}...) + b := []byte{0x10} + b = append(b, []byte(s)...) + r, err := ToKeyInfo(b, s) + if err != nil { + t.Fatal(err) + } + if r.SessionId != s { + t.Fatalf("expected %s, got %s", s, r.SessionId) + } + if r.Typ != 16 { + t.Fatalf("expected 16, got %d", r.Typ) + } + if DebugCap & 1 > 0 { + if r.Label != "internal state" { + t.Fatalf("expected 'internal_state', got '%s'", r.Label) + } + } +} + +func TestDebugDbKeyInfoRestore(t *testing.T) { + s := "bar" + b := []byte{db.DATATYPE_USERDATA} + b = append(b, []byte(s)...) + k := common.ToBytes(common.DATA_ACTIVE_SYM) + b = append(b, k...) + r, err := ToKeyInfo(b, s) if err != nil { t.Fatal(err) @@ -40,11 +68,11 @@ func TestDebugDbKeyInfo(t *testing.T) { t.Fatalf("expected %s, got %s", s, r.SessionId) } if r.Typ != 32 { - t.Fatalf("expected 64, got %d", r.Typ) + t.Fatalf("expected 32, got %d", r.Typ) } if DebugCap & 1 > 0 { - if r.Label != "userdata" { - t.Fatalf("expected 'userdata', got '%s'", r.Label) + if r.Label != "active sym" { + t.Fatalf("expected 'active sym', got '%s'", r.Label) } } } diff --git a/devtools/gen/main.go b/devtools/gen/main.go new file mode 100644 index 0000000..401f194 --- /dev/null +++ b/devtools/gen/main.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path" + + "git.defalsify.org/vise.git/logging" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/common" +) + +var ( + logg = logging.NewVanilla() + scriptDir = path.Join("services", "registration") +) + +func main() { + config.LoadConfig() + + var dbDir string + var sessionId string + var database string + var engineDebug bool + + flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") + flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.BoolVar(&engineDebug, "d", false, "use engine debug output") + flag.Parse() + + ctx := context.Background() + ctx = context.WithValue(ctx, "SessionId", sessionId) + ctx = context.WithValue(ctx, "Database", database) + + resourceDir := scriptDir + menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + + store, err := menuStorageService.GetUserdataDb(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + userStore := common.UserDataStore{store} + + err = userStore.WriteEntry(ctx, sessionId, common.DATA_AMOUNT, []byte("1.0")) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + err = store.Close() + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + +} From 890f50704f20e518d9580dcf1436159ff047569a Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 8 Dec 2024 21:21:48 +0000 Subject: [PATCH 07/12] Add documentation comments for db subprefix types --- common/db.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/common/db.go b/common/db.go index 5532dc9..f20dbab 100644 --- a/common/db.go +++ b/common/db.go @@ -7,43 +7,88 @@ import ( "git.defalsify.org/vise.git/logging" ) +// DataType is a subprefix value used in association with vise/db.DATATYPE_USERDATA. +// +// All keys are used only within the context of a single account. Unless otherwise specified, the user context is the session id. +// +// * The first byte is vise/db.DATATYPE_USERDATA +// * The last 2 bytes are the DataTyp value, big-endian. +// * The intermediate bytes are the id of the user context. +// +// All values are strings type DataTyp uint16 const ( + // TODO: Seems unused DATA_ACCOUNT DataTyp = iota + // TODO: Seems unused, only read not written DATA_ACCOUNT_CREATED + // API Tracking id to follow status of account creation DATA_TRACKING_ID + // EVM address returned from API on account creation DATA_PUBLIC_KEY + // TODO: Seems unused DATA_CUSTODIAL_ID + // Currently active PIN used to authenticate ussd state change requests DATA_ACCOUNT_PIN + // TODO: Seems unused DATA_ACCOUNT_STATUS + // The first name of the user DATA_FIRST_NAME + // The last name of the user DATA_FAMILY_NAME + // The year-of-birth of the user DATA_YOB + // The location of the user DATA_LOCATION + // The gender of the user DATA_GENDER + // The offerings description of the user DATA_OFFERINGS + // The ethereum address of the recipient of an ongoing send request DATA_RECIPIENT + // The voucher value amount of an ongoing send request DATA_AMOUNT + // A general swap field for temporary values DATA_TEMPORARY_VALUE + // Currently active voucher symbol of user DATA_ACTIVE_SYM + // Voucher balance of user's currently active voucher DATA_ACTIVE_BAL + // String boolean indicating whether use of PIN is blocked DATA_BLOCKED_NUMBER + // Reverse mapping of a user's evm address to a session id. DATA_PUBLIC_KEY_REVERSE + // Decimal count of the currently active voucher DATA_ACTIVE_DECIMAL + // EVM address of the currently active voucher DATA_ACTIVE_ADDRESS - // Start the sub prefix data at 256 (0x0100) +) + +const ( + // List of valid voucher symbols in the user context. DATA_VOUCHER_SYMBOLS DataTyp = 256 + iota + // List of voucher balances for vouchers valid in the user context. DATA_VOUCHER_BALANCES + // List of voucher decimal counts for vouchers valid in the user context. DATA_VOUCHER_DECIMALS + // List of voucher EVM addresses for vouchers valid in the user context. DATA_VOUCHER_ADDRESSES + // List of senders for valid transactions in the user context. DATA_TX_SENDERS + // List of recipients for valid transactions in the user context. DATA_TX_RECIPIENTS + // List of voucher values for valid transactions in the user context. DATA_TX_VALUES + // List of voucher EVM addresses for valid transactions in the user context. DATA_TX_ADDRESSES + // List of valid transaction hashes in the user context. DATA_TX_HASHES + // List of transaction dates for valid transactions in the user context. DATA_TX_DATES + // List of voucher symbols for valid transactions in the user context. DATA_TX_SYMBOLS + // List of voucher decimal counts for valid transactions in the user context. DATA_TX_DECIMALS ) From a3e821fb16f5cb994175d042a39e4e43eca21121 Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 8 Dec 2024 21:57:53 +0000 Subject: [PATCH 08/12] Clarify gen cmd example with reverse public key lookup --- devtools/gen/main.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/devtools/gen/main.go b/devtools/gen/main.go index 401f194..6b58a50 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha1" "flag" "fmt" "os" @@ -46,7 +47,18 @@ func main() { } userStore := common.UserDataStore{store} - err = userStore.WriteEntry(ctx, sessionId, common.DATA_AMOUNT, []byte("1.0")) + h := sha1.New() + h.Write([]byte(sessionId)) + address := h.Sum(nil) + addressString := fmt.Sprintf("%x", address) + + err = userStore.WriteEntry(ctx, sessionId, common.DATA_PUBLIC_KEY, []byte(addressString)) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + err = userStore.WriteEntry(ctx, addressString, common.DATA_PUBLIC_KEY_REVERSE, []byte(sessionId)) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) From 604c16ec90ddcf2e97719c0893c0747a945bf681 Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 8 Dec 2024 22:48:39 +0000 Subject: [PATCH 09/12] Habilitate store dumper --- debug/db.go | 10 ++++++++++ devtools/gen/main.go | 6 ++++++ devtools/store/main.go | 11 +++++++++-- go.mod | 2 +- go.sum | 4 ++-- internal/storage/gdbm.go | 2 ++ 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/debug/db.go b/debug/db.go index eb62a41..ec9e58f 100644 --- a/debug/db.go +++ b/debug/db.go @@ -20,6 +20,16 @@ type KeyInfo struct { Description string } +func (k KeyInfo) String() string { + v := uint16(k.SubTyp) + s := subTypToString(k.SubTyp) + if s == "" { + v = uint16(k.Typ) + s = typToString(k.Typ) + } + return fmt.Sprintf("Session Id: %s\nTyp: %s (%d)\n", k.SessionId, s, v) +} + func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) { o := KeyInfo{} b := []byte(sessionId) diff --git a/devtools/gen/main.go b/devtools/gen/main.go index 6b58a50..b9e2aed 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -11,6 +11,7 @@ import ( "git.defalsify.org/vise.git/logging" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/common" ) @@ -19,6 +20,11 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + initializers.LoadEnvVariables() +} + + func main() { config.LoadConfig() diff --git a/devtools/store/main.go b/devtools/store/main.go index d6abec9..9262703 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -10,6 +10,7 @@ import ( "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/debug" "git.defalsify.org/vise.git/logging" ) @@ -30,6 +31,7 @@ func main() { var sessionId string var database string var engineDebug bool + flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&database, "db", "gdbm", "database to be used") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") @@ -37,7 +39,7 @@ func main() { flag.Parse() ctx := context.Background() - //ctx = context.WithValue(ctx, "SessionId", sessionId) + ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) resourceDir := scriptDir @@ -60,7 +62,12 @@ func main() { if k == nil { break } - fmt.Printf("%x %s %x\n", k, v) + o, err := debug.FromKey(k) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + fmt.Printf("%vValue: %v\n\n", o, v) } err = store.Close() diff --git a/go.mod b/go.mod index d89f0f3..aeac055 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a + git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 5532c1c..5949d8b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a h1:LvGKktk0kUnuRN3nF9r15D8OoV0sFaMmQr52kGq2gtE= -git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 h1:QI/6xmpFM9NsG1+vRPzlUkkjAQ1Z8WhNqS+2tCjBpxg= +git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= diff --git a/internal/storage/gdbm.go b/internal/storage/gdbm.go index 9879fe0..31ebf47 100644 --- a/internal/storage/gdbm.go +++ b/internal/storage/gdbm.go @@ -116,5 +116,7 @@ func(tdb *ThreadGdbmDb) Close() error { } func(tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) { + tdb.reserve() + defer tdb.release() return tdb.db.Dump(ctx, key) } From c820e89cb751c3b7b6285c74afb18570282dc077 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 11 Dec 2024 19:13:13 +0000 Subject: [PATCH 10/12] Segment tx userdata types, add internals docs --- common/db.go | 15 +++++---------- doc/data.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 doc/data.md diff --git a/common/db.go b/common/db.go index f20dbab..a5cf1c1 100644 --- a/common/db.go +++ b/common/db.go @@ -19,20 +19,12 @@ import ( type DataTyp uint16 const ( - // TODO: Seems unused - DATA_ACCOUNT DataTyp = iota - // TODO: Seems unused, only read not written - DATA_ACCOUNT_CREATED // API Tracking id to follow status of account creation - DATA_TRACKING_ID + DATA_TRACKING_ID = iota // EVM address returned from API on account creation DATA_PUBLIC_KEY - // TODO: Seems unused - DATA_CUSTODIAL_ID // Currently active PIN used to authenticate ussd state change requests DATA_ACCOUNT_PIN - // TODO: Seems unused - DATA_ACCOUNT_STATUS // The first name of the user DATA_FIRST_NAME // The last name of the user @@ -75,7 +67,10 @@ const ( // List of voucher EVM addresses for vouchers valid in the user context. DATA_VOUCHER_ADDRESSES // List of senders for valid transactions in the user context. - DATA_TX_SENDERS +) + +const ( + DATA_TX_SENDERS = 512 + iota // List of recipients for valid transactions in the user context. DATA_TX_RECIPIENTS // List of voucher values for valid transactions in the user context. diff --git a/doc/data.md b/doc/data.md new file mode 100644 index 0000000..7a41f0c --- /dev/null +++ b/doc/data.md @@ -0,0 +1,28 @@ +# Internals + +## Version + +This document describes component versions: + +* `urdt-ussd` `v0.5.0-beta` +* `go-vise` `v0.2.2` + + +## User profile data + +All user profile items are stored under keys matching the user's session id, prefixed with the 8-bit value `git.defalsify.org/vise.git/db.DATATYPE_USERDATA` (32), and followed with a 16-big big-endian value subprefix. + +For example, given the sessionId `+254123` and the key `git.grassecon.net/urdt-ussd/common.DATA_PUBLIC_KEY` (2) will be stored under the key: + +``` +0x322b3235343132330002 + +prefix sessionid subprefix +32 2b323534313233 0002 +``` + +### Sub-prefixes + +All sub-prefixes are defined as constants in the `git.grassecon.net/urdt-ussd/common` package. The constant names have the prefix `DATA_` + +Please refer to inline godoc documentation for the `git.grassecon.net/urdt-ussd/common` package for details on each data item. From 8d4d8a48e02367842b43fd5940bfbaf13a56a2ba Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 14 Dec 2024 11:56:31 +0000 Subject: [PATCH 11/12] Fix compile errors, test errors --- cmd/africastalking/main.go | 13 ++++++------- internal/handlers/ussd/menuhandler.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 7d1caa8..98864db 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -7,7 +7,6 @@ import ( "flag" "fmt" "io" - "log" "net/http" "os" "os/signal" @@ -45,14 +44,14 @@ type atRequestParser struct{} func (arp *atRequestParser) GetSessionId(rq any) (string, error) { rqv, ok := rq.(*http.Request) if !ok { - log.Printf("got an invalid request:", rq) + logg.Warnf("got an invalid request", "req", rq) return "", handlers.ErrInvalidRequest } // Capture body (if any) for logging body, err := io.ReadAll(rqv.Body) if err != nil { - log.Printf("failed to read request body:", err) + logg.Warnf("failed to read request body", "err", err) return "", fmt.Errorf("failed to read request body: %v", err) } // Reset the body for further reading @@ -62,13 +61,13 @@ func (arp *atRequestParser) GetSessionId(rq any) (string, error) { bodyLog := map[string]string{"body": string(body)} logBytes, err := json.Marshal(bodyLog) if err != nil { - log.Printf("failed to marshal request body:", err) + logg.Warnf("failed to marshal request body", "err", err) } else { - log.Printf("Received request:", string(logBytes)) + logg.Debugf("received request", "bytes", logBytes) } if err := rqv.ParseForm(); err != nil { - log.Printf("failed to parse form data: %v", err) + logg.Warnf("failed to parse form data", "err", err) return "", fmt.Errorf("failed to parse form data: %v", err) } @@ -79,7 +78,7 @@ func (arp *atRequestParser) GetSessionId(rq any) (string, error) { formattedNumber, err := common.FormatPhoneNumber(phoneNumber) if err != nil { - fmt.Printf("Error: %v\n", err) + logg.Warnf("failed to format phone number", "err", err) return "", fmt.Errorf("failed to format number") } diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index b47e31f..dff01d1 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -213,7 +213,7 @@ func (h *Handlers) CreateAccount(ctx context.Context, sym string, input []byte) return res, fmt.Errorf("missing session") } store := h.userdataStore - _, err = store.ReadEntry(ctx, sessionId, common.DATA_ACCOUNT_CREATED) + _, err = store.ReadEntry(ctx, sessionId, common.DATA_PUBLIC_KEY) if err != nil { if db.IsNotFound(err) { logg.InfoCtxf(ctx, "Creating an account because it doesn't exist") From 1ab49647f61f98bb1fd5c2b306ac53a27abf7e9e Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 14 Dec 2024 12:02:13 +0000 Subject: [PATCH 12/12] Upgrade vise --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aeac055..e1b7ddb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 + git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80 github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 5949d8b..ef7b782 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 h1:QI/6xmpFM9NsG1+vRPzlUkkjAQ1Z8WhNqS+2tCjBpxg= -git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80 h1:GYUVXRUtMpA40T4COeAduoay6CIgXjD5cfDYZOTFIKw= +git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=