From 10b3083647e90968dff2d1c5d73280515235ac20 Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sat, 2 Nov 2024 20:29:32 +0000
Subject: [PATCH 01/16] WIP Add db dumper tool

---
 devtools/store/main.go   | 64 ++++++++++++++++++++++++++++++++++++++++
 go.mod                   |  4 +--
 go.sum                   |  4 +--
 internal/storage/gdbm.go |  4 +++
 4 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 devtools/store/main.go

diff --git a/devtools/store/main.go b/devtools/store/main.go
new file mode 100644
index 0000000..df27bb1
--- /dev/null
+++ b/devtools/store/main.go
@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+
+	"git.grassecon.net/urdt/ussd/config"
+	"git.grassecon.net/urdt/ussd/initializers"
+	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.defalsify.org/vise.git/logging"
+)
+
+var (
+	logg      = logging.NewVanilla()
+	scriptDir = path.Join("services", "registration")
+)
+
+func init() {
+	initializers.LoadEnvVariables()
+}
+
+func main() {
+	config.LoadConfig()
+
+	var dbDir string
+	var sessionId string
+	var database string
+	var engineDebug bool
+	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
+	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
+	flag.Parse()
+
+	ctx := context.Background()
+	//ctx = context.WithValue(ctx, "SessionId", sessionId)
+	ctx = context.WithValue(ctx, "Database", database)
+
+	resourceDir := scriptDir
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
+	store, err := menuStorageService.GetUserdataDb(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	d, err := store.Dump(ctx, []byte(sessionId))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	for true {
+		k, v := d.Next(ctx)
+		if k == nil {
+			break
+		}
+		fmt.Printf("%x %s\n", k, v)
+	}
+}
diff --git a/go.mod b/go.mod
index 391c1a5..e30d42f 100644
--- a/go.mod
+++ b/go.mod
@@ -5,14 +5,14 @@ go 1.23.0
 toolchain go1.23.2
 
 require (
-	git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b
+	git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045
 	github.com/alecthomas/assert/v2 v2.2.2
 	github.com/grassrootseconomics/eth-custodial v1.3.0-beta
 	github.com/peteole/testdata-loader v0.3.0
 	gopkg.in/leonelquinteros/gotext.v1 v1.3.1
 )
 
-require github.com/grassrootseconomics/ussd-data-service v0.0.0-20241003123429-4904b4438a3a // indirect
+require github.com/grassrootseconomics/ussd-data-service v0.0.0-20241003123429-4904b4438a3a
 
 require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 0ba38c1..21dbfb2 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b h1:dxBplsIlzJHV+5EH+gzB+w08Blt7IJbb2jeRe1OEjLU=
-git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
+git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045 h1:uFFVEymnsJGVcuI2r//oNAqDD8l0HOkkEvqpOJHdJiY=
+git.defalsify.org/vise.git v0.2.1-0.20241102200556-5348e1988045/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
 github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
 github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=
diff --git a/internal/storage/gdbm.go b/internal/storage/gdbm.go
index 49de570..adb1d0f 100644
--- a/internal/storage/gdbm.go
+++ b/internal/storage/gdbm.go
@@ -114,3 +114,7 @@ func(tdb *ThreadGdbmDb) Close() error {
 	tdb.db = nil
 	return err
 }
+
+func(tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
+	return tdb.db.Dump(ctx, key)
+}

From 1a782c1db9b9d0339435f6e47d8c20e73ff23b7a Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 1 Dec 2024 17:32:06 +0000
Subject: [PATCH 02/16] Add debug package

---
 debug/db.go            | 47 ++++++++++++++++++++++++++++++++++++++++++
 debug/db_no.go         | 14 +++++++++++++
 debug/db_yes.go        | 42 +++++++++++++++++++++++++++++++++++++
 devtools/store/main.go |  3 ++-
 4 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100644 debug/db.go
 create mode 100644 debug/db_no.go
 create mode 100644 debug/db_yes.go

diff --git a/debug/db.go b/debug/db.go
new file mode 100644
index 0000000..9efad00
--- /dev/null
+++ b/debug/db.go
@@ -0,0 +1,47 @@
+package debug
+
+import (
+	"fmt"
+	"encoding/binary"
+
+	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/common"
+)
+
+type KeyInfo struct {
+	SessionId string
+	Typ uint8
+	SubTyp common.DataTyp
+	Label string
+	Description string
+}
+
+func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
+	o := KeyInfo{}
+	b := []byte(sessionId)
+	if len(k) <= len(b) {
+		return o, fmt.Errorf("storage key missing")
+	}
+
+	o.SessionId = sessionId
+
+	k = k[len(b):]
+	o.Typ = k[0]
+	k = k[1:]
+
+	if o.Typ == storage.DATATYPE_USERSUB {
+		if len(k) == 0 {
+			return o, fmt.Errorf("missing subtype key")
+		}
+		v := binary.BigEndian.Uint16(k[:2])
+		o.SubTyp = common.DataTyp(v)
+		o.Label = typToString(o.SubTyp)
+		k = k[2:]
+	}
+
+	if len(k) != 0 {
+		return o, fmt.Errorf("excess key information")
+	}
+
+	return o, nil
+}
diff --git a/debug/db_no.go b/debug/db_no.go
new file mode 100644
index 0000000..a01c78f
--- /dev/null
+++ b/debug/db_no.go
@@ -0,0 +1,14 @@
+// +build !debugdb
+
+package debug
+
+import (
+	"fmt"
+
+	"git.grassecon.net/urdt/ussd/common"
+)
+
+
+func typToString(v common.DataTyp) string {
+	return fmt.Sprintf("(%d)", v)
+}
diff --git a/debug/db_yes.go b/debug/db_yes.go
new file mode 100644
index 0000000..fa8186e
--- /dev/null
+++ b/debug/db_yes.go
@@ -0,0 +1,42 @@
+// +build debugdb
+
+package debug
+
+import (
+	"git.defalsify.org/vise.git/db"
+
+	"git.grassecon.net/urdt/ussd/common"
+)
+
+var (
+	dbTypStr map[common.DataTyp]string = map[common.DataTyp]string {
+		db.DATATYPE_USERDATA: "userdata",
+		common.DATA_ACCOUNT: "account",
+		common.DATA_ACCOUNT_CREATED: "account created",
+		common.DATA_TRACKING_ID: "tracking id",
+		common.DATA_PUBLIC_KEY: "public key",
+		common.DATA_CUSTODIAL_ID: "custodial id",
+		common.DATA_ACCOUNT_PIN: "account pin",
+		common.DATA_ACCOUNT_STATUS: "account status",
+		common.DATA_FIRST_NAME: "first name",
+		common.DATA_FAMILY_NAME: "family name",
+		common.DATA_YOB: "year of birth",
+		common.DATA_LOCATION: "location",
+		common.DATA_GENDER: "gender", 
+		common.DATA_OFFERINGS: "offerings",
+		common.DATA_RECIPIENT: "recipient",
+		common.DATA_AMOUNT: "amount",
+		common.DATA_TEMPORARY_VALUE: "temporary value",
+		common.DATA_ACTIVE_SYM: "active sym",
+		common.DATA_ACTIVE_BAL: "active bal",
+		common.DATA_BLOCKED_NUMBER: "blocked number",
+		common.DATA_PUBLIC_KEY_REVERSE: "public key reverse",
+		common.DATA_ACTIVE_DECIMAL: "active decimal",
+		common.DATA_ACTIVE_ADDRESS: "active address",
+		common.DATA_TRANSACTIONS: "transactions",
+	}
+)
+
+func typToString(v common.DataTyp) string {
+	return dbTypStr[v]
+}
diff --git a/devtools/store/main.go b/devtools/store/main.go
index df27bb1..35587ea 100644
--- a/devtools/store/main.go
+++ b/devtools/store/main.go
@@ -22,6 +22,7 @@ func init() {
 	initializers.LoadEnvVariables()
 }
 
+
 func main() {
 	config.LoadConfig()
 
@@ -59,6 +60,6 @@ func main() {
 		if k == nil {
 			break
 		}
-		fmt.Printf("%x %s\n", k, v)
+		fmt.Printf("%x %s %x\n", k, v)
 	}
 }

From 35cf3a1cd1d0586aadbb0dc68c43bab09a966bae Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 1 Dec 2024 18:41:28 +0000
Subject: [PATCH 03/16] Add debug label test, capability debug flag

---
 debug/cap.go    | 5 +++++
 debug/db_yes.go | 4 ++++
 2 files changed, 9 insertions(+)
 create mode 100644 debug/cap.go

diff --git a/debug/cap.go b/debug/cap.go
new file mode 100644
index 0000000..458bb48
--- /dev/null
+++ b/debug/cap.go
@@ -0,0 +1,5 @@
+package debug
+
+var (
+	DebugCap uint32
+)
diff --git a/debug/db_yes.go b/debug/db_yes.go
index fa8186e..c323366 100644
--- a/debug/db_yes.go
+++ b/debug/db_yes.go
@@ -37,6 +37,10 @@ var (
 	}
 )
 
+func init() {
+	DebugCap |= 1
+}
+
 func typToString(v common.DataTyp) string {
 	return dbTypStr[v]
 }

From 6d4f3109f8b312dc060f4c8c9d2b6f4791427b1a Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 1 Dec 2024 23:12:58 +0000
Subject: [PATCH 04/16] Consolidate subtyp and typ debug

---
 debug/db.go       | 17 +++++++++++++++-
 debug/db_debug.go | 39 ++++++++++++++++++++++++++++++++++++
 debug/db_no.go    | 14 -------------
 debug/db_test.go  | 50 +++++++++++++++++++++++++++++++++++++++++++++++
 debug/db_yes.go   | 46 -------------------------------------------
 5 files changed, 105 insertions(+), 61 deletions(-)
 create mode 100644 debug/db_debug.go
 delete mode 100644 debug/db_no.go
 create mode 100644 debug/db_test.go
 delete mode 100644 debug/db_yes.go

diff --git a/debug/db.go b/debug/db.go
index 9efad00..ccc8630 100644
--- a/debug/db.go
+++ b/debug/db.go
@@ -8,6 +8,10 @@ import (
 	"git.grassecon.net/urdt/ussd/common"
 )
 
+var (
+	dbTypStr map[common.DataTyp]string = make(map[common.DataTyp]string)
+)
+
 type KeyInfo struct {
 	SessionId string
 	Typ uint8
@@ -19,6 +23,7 @@ type KeyInfo struct {
 func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 	o := KeyInfo{}
 	b := []byte(sessionId)
+
 	if len(k) <= len(b) {
 		return o, fmt.Errorf("storage key missing")
 	}
@@ -35,8 +40,10 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 		}
 		v := binary.BigEndian.Uint16(k[:2])
 		o.SubTyp = common.DataTyp(v)
-		o.Label = typToString(o.SubTyp)
+		o.Label = subTypToString(o.SubTyp)
 		k = k[2:]
+	} else {
+		o.Label = typToString(o.Typ)
 	}
 
 	if len(k) != 0 {
@@ -45,3 +52,11 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 
 	return o, nil
 }
+
+func subTypToString(v common.DataTyp) string {
+	return dbTypStr[v + storage.DATATYPE_USERSUB + 1]
+}
+
+func typToString(v uint8) string {
+	return dbTypStr[common.DataTyp(uint16(v))]
+}
diff --git a/debug/db_debug.go b/debug/db_debug.go
new file mode 100644
index 0000000..e4a3949
--- /dev/null
+++ b/debug/db_debug.go
@@ -0,0 +1,39 @@
+// +build debugdb
+
+package debug
+
+import (
+	"git.defalsify.org/vise.git/db"
+
+	"git.grassecon.net/urdt/ussd/common"
+	"git.grassecon.net/urdt/ussd/internal/storage"
+)
+
+func init() {
+	DebugCap |= 1
+	dbTypStr[db.DATATYPE_STATE] = "internal_state"
+	dbTypStr[db.DATATYPE_USERDATA] = "userdata"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT] = "account"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_CREATED] = "account_created"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRACKING_ID] = "tracking id"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY] = "public key"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_CUSTODIAL_ID] = "custodial id"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_PIN] = "account pin"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_STATUS] = "account status"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FIRST_NAME] = "first name"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FAMILY_NAME] = "family name"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_YOB] = "year of birth"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_LOCATION] = "location"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_GENDER] = "gender"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_OFFERINGS] = "offerings"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_RECIPIENT] = "recipient"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_AMOUNT] = "amount"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_SYM] = "active sym"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_BAL] = "active bal"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_ADDRESS] = "active address"
+	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRANSACTIONS] = "transactions"
+}
diff --git a/debug/db_no.go b/debug/db_no.go
deleted file mode 100644
index a01c78f..0000000
--- a/debug/db_no.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// +build !debugdb
-
-package debug
-
-import (
-	"fmt"
-
-	"git.grassecon.net/urdt/ussd/common"
-)
-
-
-func typToString(v common.DataTyp) string {
-	return fmt.Sprintf("(%d)", v)
-}
diff --git a/debug/db_test.go b/debug/db_test.go
new file mode 100644
index 0000000..2dae1e8
--- /dev/null
+++ b/debug/db_test.go
@@ -0,0 +1,50 @@
+package debug
+
+import (
+	"testing"
+)
+
+func TestDebugDbSubKeyInfo(t *testing.T) {
+	s := "foo"
+	b := []byte(s)
+	b = append(b, []byte{0x40, 0x00, 0x02}...)
+	r, err := ToKeyInfo(b, s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.SessionId != s {
+		t.Fatalf("expected %s, got %s", s, r.SessionId)
+	}
+	if r.Typ != 64 {
+		t.Fatalf("expected 64, got %d", r.Typ)
+	}
+	if r.SubTyp != 2 {
+		t.Fatalf("expected 2, got %d", r.SubTyp)
+	}
+	if DebugCap & 1 > 0 {
+		if r.Label != "tracking id" {
+			t.Fatalf("expected 'tracking id', got '%s'", r.Label)
+		}
+	}
+}
+
+func TestDebugDbKeyInfo(t *testing.T) {
+	s := "bar"
+	b := []byte(s)
+	b = append(b, []byte{0x20}...)
+	r, err := ToKeyInfo(b, s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.SessionId != s {
+		t.Fatalf("expected %s, got %s", s, r.SessionId)
+	}
+	if r.Typ != 32 {
+		t.Fatalf("expected 64, got %d", r.Typ)
+	}
+	if DebugCap & 1 > 0 {
+		if r.Label != "userdata" {
+			t.Fatalf("expected 'userdata', got '%s'", r.Label)
+		}
+	}
+}
diff --git a/debug/db_yes.go b/debug/db_yes.go
deleted file mode 100644
index c323366..0000000
--- a/debug/db_yes.go
+++ /dev/null
@@ -1,46 +0,0 @@
-// +build debugdb
-
-package debug
-
-import (
-	"git.defalsify.org/vise.git/db"
-
-	"git.grassecon.net/urdt/ussd/common"
-)
-
-var (
-	dbTypStr map[common.DataTyp]string = map[common.DataTyp]string {
-		db.DATATYPE_USERDATA: "userdata",
-		common.DATA_ACCOUNT: "account",
-		common.DATA_ACCOUNT_CREATED: "account created",
-		common.DATA_TRACKING_ID: "tracking id",
-		common.DATA_PUBLIC_KEY: "public key",
-		common.DATA_CUSTODIAL_ID: "custodial id",
-		common.DATA_ACCOUNT_PIN: "account pin",
-		common.DATA_ACCOUNT_STATUS: "account status",
-		common.DATA_FIRST_NAME: "first name",
-		common.DATA_FAMILY_NAME: "family name",
-		common.DATA_YOB: "year of birth",
-		common.DATA_LOCATION: "location",
-		common.DATA_GENDER: "gender", 
-		common.DATA_OFFERINGS: "offerings",
-		common.DATA_RECIPIENT: "recipient",
-		common.DATA_AMOUNT: "amount",
-		common.DATA_TEMPORARY_VALUE: "temporary value",
-		common.DATA_ACTIVE_SYM: "active sym",
-		common.DATA_ACTIVE_BAL: "active bal",
-		common.DATA_BLOCKED_NUMBER: "blocked number",
-		common.DATA_PUBLIC_KEY_REVERSE: "public key reverse",
-		common.DATA_ACTIVE_DECIMAL: "active decimal",
-		common.DATA_ACTIVE_ADDRESS: "active address",
-		common.DATA_TRANSACTIONS: "transactions",
-	}
-)
-
-func init() {
-	DebugCap |= 1
-}
-
-func typToString(v common.DataTyp) string {
-	return dbTypStr[v]
-}

From c9deca11805f82c023d7026e8b37978ab319566c Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Mon, 2 Dec 2024 14:36:33 +0000
Subject: [PATCH 05/16] change userdata prefix on subprefix debug

---
 debug/db_debug.go      | 47 +++++++++++++++++++++---------------------
 devtools/store/main.go |  6 ++++++
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/debug/db_debug.go b/debug/db_debug.go
index e4a3949..7c17eed 100644
--- a/debug/db_debug.go
+++ b/debug/db_debug.go
@@ -12,28 +12,27 @@ import (
 func init() {
 	DebugCap |= 1
 	dbTypStr[db.DATATYPE_STATE] = "internal_state"
-	dbTypStr[db.DATATYPE_USERDATA] = "userdata"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT] = "account"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_CREATED] = "account_created"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRACKING_ID] = "tracking id"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY] = "public key"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_CUSTODIAL_ID] = "custodial id"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_PIN] = "account pin"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACCOUNT_STATUS] = "account status"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FIRST_NAME] = "first name"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_FAMILY_NAME] = "family name"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_YOB] = "year of birth"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_LOCATION] = "location"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_GENDER] = "gender"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_OFFERINGS] = "offerings"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_RECIPIENT] = "recipient"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_AMOUNT] = "amount"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_SYM] = "active sym"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_BAL] = "active bal"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_ACTIVE_ADDRESS] = "active address"
-	dbTypStr[storage.DATATYPE_USERSUB + 1 + common.DATA_TRANSACTIONS] = "transactions"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account_created"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address"
+	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRANSACTIONS] = "transactions"
 }
diff --git a/devtools/store/main.go b/devtools/store/main.go
index 35587ea..d6abec9 100644
--- a/devtools/store/main.go
+++ b/devtools/store/main.go
@@ -62,4 +62,10 @@ func main() {
 		}
 		fmt.Printf("%x %s %x\n", k, v)
 	}
+
+	err = store.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 }

From 3416fdf50c2ff82fadf9b4cc5afcf38b149de9c6 Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sat, 7 Dec 2024 23:06:03 +0000
Subject: [PATCH 06/16] Add keyinfo restore

---
 common/user_store.go |  4 +--
 debug/db.go          | 22 ++++++++++++----
 debug/db_debug.go    | 60 +++++++++++++++++++++++++------------------
 debug/db_test.go     | 44 ++++++++++++++++++++++++++------
 devtools/gen/main.go | 61 ++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 151 insertions(+), 40 deletions(-)
 create mode 100644 devtools/gen/main.go

diff --git a/common/user_store.go b/common/user_store.go
index 29796e2..6c770d8 100644
--- a/common/user_store.go
+++ b/common/user_store.go
@@ -20,7 +20,7 @@ type UserDataStore struct {
 func (store *UserDataStore) ReadEntry(ctx context.Context, sessionId string, typ DataTyp) ([]byte, error) {
 	store.SetPrefix(db.DATATYPE_USERDATA)
 	store.SetSession(sessionId)
-	k := PackKey(typ, []byte(sessionId))
+	k := ToBytes(typ)
 	return store.Get(ctx, k)
 }
 
@@ -29,6 +29,6 @@ func (store *UserDataStore) ReadEntry(ctx context.Context, sessionId string, typ
 func (store *UserDataStore) WriteEntry(ctx context.Context, sessionId string, typ DataTyp, value []byte) error {
 	store.SetPrefix(db.DATATYPE_USERDATA)
 	store.SetSession(sessionId)
-	k := PackKey(typ, []byte(sessionId))
+	k := ToBytes(typ)
 	return store.Put(ctx, k, value)
 }
diff --git a/debug/db.go b/debug/db.go
index ccc8630..eb62a41 100644
--- a/debug/db.go
+++ b/debug/db.go
@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"encoding/binary"
 
-	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/common"
+	"git.defalsify.org/vise.git/db"
 )
 
 var (
@@ -30,11 +30,12 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 
 	o.SessionId = sessionId
 
-	k = k[len(b):]
-	o.Typ = k[0]
+	o.Typ = uint8(k[0])
 	k = k[1:]
+	o.SessionId = string(k[:len(b)])
+	k = k[len(b):]
 
-	if o.Typ == storage.DATATYPE_USERSUB {
+	if o.Typ == db.DATATYPE_USERDATA {
 		if len(k) == 0 {
 			return o, fmt.Errorf("missing subtype key")
 		}
@@ -53,8 +54,19 @@ func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 	return o, nil
 }
 
+func FromKey(k []byte) (KeyInfo, error) {
+	o := KeyInfo{}
+
+	if len(k) < 4 {
+		return o, fmt.Errorf("insufficient key length")
+	}
+
+	sessionIdBytes := k[1:len(k)-2]
+	return ToKeyInfo(k, string(sessionIdBytes))
+}
+
 func subTypToString(v common.DataTyp) string {
-	return dbTypStr[v + storage.DATATYPE_USERSUB + 1]
+	return dbTypStr[v + db.DATATYPE_USERDATA + 1]
 }
 
 func typToString(v uint8) string {
diff --git a/debug/db_debug.go b/debug/db_debug.go
index 7c17eed..ed2dd66 100644
--- a/debug/db_debug.go
+++ b/debug/db_debug.go
@@ -6,33 +6,43 @@ import (
 	"git.defalsify.org/vise.git/db"
 
 	"git.grassecon.net/urdt/ussd/common"
-	"git.grassecon.net/urdt/ussd/internal/storage"
 )
 
 func init() {
 	DebugCap |= 1
-	dbTypStr[db.DATATYPE_STATE] = "internal_state"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account_created"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address"
-	dbTypStr[storage.DATATYPE_USERDATA + 1 + common.DATA_TRANSACTIONS] = "transactions"
+	dbTypStr[db.DATATYPE_STATE] = "internal state"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT] = "account"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_CREATED] = "account created"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TRACKING_ID] = "tracking id"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY] = "public key"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_CUSTODIAL_ID] = "custodial id"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_PIN] = "account pin"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACCOUNT_STATUS] = "account status"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_FIRST_NAME] = "first name"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_FAMILY_NAME] = "family name"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_YOB] = "year of birth"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_LOCATION] = "location"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_GENDER] = "gender"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_OFFERINGS] = "offerings"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_RECIPIENT] = "recipient"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_AMOUNT] = "amount"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TEMPORARY_VALUE] = "temporary value"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_SYM] = "active sym"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_BAL] = "active bal"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_BLOCKED_NUMBER] = "blocked number"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_PUBLIC_KEY_REVERSE] = "public_key_reverse"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_DECIMAL] = "active decimal"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_ACTIVE_ADDRESS] = "active address"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_SYMBOLS] = "voucher symbols"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_BALANCES] = "voucher balances"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_DECIMALS] = "voucher decimals"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_VOUCHER_ADDRESSES] = "voucher addresses"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_SENDERS] = "tx senders"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_RECIPIENTS] = "tx recipients"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_VALUES] = "tx values"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_ADDRESSES] = "tx addresses"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_HASHES] = "tx hashes"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_DATES] = "tx dates"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_SYMBOLS] = "tx symbols"
+	dbTypStr[db.DATATYPE_USERDATA + 1 + common.DATA_TX_DECIMALS] = "tx decimals"
 }
diff --git a/debug/db_test.go b/debug/db_test.go
index 2dae1e8..25f781d 100644
--- a/debug/db_test.go
+++ b/debug/db_test.go
@@ -2,12 +2,16 @@ package debug
 
 import (
 	"testing"
+
+	"git.grassecon.net/urdt/ussd/common"
+	"git.defalsify.org/vise.git/db"
 )
 
 func TestDebugDbSubKeyInfo(t *testing.T) {
 	s := "foo"
-	b := []byte(s)
-	b = append(b, []byte{0x40, 0x00, 0x02}...)
+	b := []byte{0x20}
+	b = append(b, []byte(s)...)
+	b = append(b, []byte{0x00, 0x02}...)
 	r, err := ToKeyInfo(b, s)
 	if err != nil {
 		t.Fatal(err)
@@ -15,7 +19,7 @@ func TestDebugDbSubKeyInfo(t *testing.T) {
 	if r.SessionId != s {
 		t.Fatalf("expected %s, got %s", s, r.SessionId)
 	}
-	if r.Typ != 64 {
+	if r.Typ != 32 {
 		t.Fatalf("expected 64, got %d", r.Typ)
 	}
 	if r.SubTyp != 2 {
@@ -30,8 +34,32 @@ func TestDebugDbSubKeyInfo(t *testing.T) {
 
 func TestDebugDbKeyInfo(t *testing.T) {
 	s := "bar"
-	b := []byte(s)
-	b = append(b, []byte{0x20}...)
+	b := []byte{0x10}
+	b = append(b, []byte(s)...)
+	r, err := ToKeyInfo(b, s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.SessionId != s {
+		t.Fatalf("expected %s, got %s", s, r.SessionId)
+	}
+	if r.Typ != 16 {
+		t.Fatalf("expected 16, got %d", r.Typ)
+	}
+	if DebugCap & 1 > 0 {
+		if r.Label != "internal state" {
+			t.Fatalf("expected 'internal_state', got '%s'", r.Label)
+		}
+	}
+}
+
+func TestDebugDbKeyInfoRestore(t *testing.T) {
+	s := "bar"
+	b := []byte{db.DATATYPE_USERDATA}
+	b = append(b, []byte(s)...)
+	k := common.ToBytes(common.DATA_ACTIVE_SYM)
+	b = append(b, k...)
+
 	r, err := ToKeyInfo(b, s)
 	if err != nil {
 		t.Fatal(err)
@@ -40,11 +68,11 @@ func TestDebugDbKeyInfo(t *testing.T) {
 		t.Fatalf("expected %s, got %s", s, r.SessionId)
 	}
 	if r.Typ != 32 {
-		t.Fatalf("expected 64, got %d", r.Typ)
+		t.Fatalf("expected 32, got %d", r.Typ)
 	}
 	if DebugCap & 1 > 0 {
-		if r.Label != "userdata" {
-			t.Fatalf("expected 'userdata', got '%s'", r.Label)
+		if r.Label != "active sym" {
+			t.Fatalf("expected 'active sym', got '%s'", r.Label)
 		}
 	}
 }
diff --git a/devtools/gen/main.go b/devtools/gen/main.go
new file mode 100644
index 0000000..401f194
--- /dev/null
+++ b/devtools/gen/main.go
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path"
+
+	"git.defalsify.org/vise.git/logging"
+	"git.grassecon.net/urdt/ussd/config"
+	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/common"
+)
+
+var (
+	logg      = logging.NewVanilla()
+	scriptDir = path.Join("services", "registration")
+)
+
+func main() {
+	config.LoadConfig()
+
+	var dbDir string
+	var sessionId string
+	var database string
+	var engineDebug bool
+
+	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
+	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
+	flag.Parse()
+
+	ctx := context.Background()
+	ctx = context.WithValue(ctx, "SessionId", sessionId)
+	ctx = context.WithValue(ctx, "Database", database)
+
+	resourceDir := scriptDir
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
+	store, err := menuStorageService.GetUserdataDb(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	userStore := common.UserDataStore{store}
+
+	err = userStore.WriteEntry(ctx, sessionId, common.DATA_AMOUNT, []byte("1.0"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	err = store.Close()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	
+}

From 890f50704f20e518d9580dcf1436159ff047569a Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 8 Dec 2024 21:21:48 +0000
Subject: [PATCH 07/16] Add documentation comments for db subprefix types

---
 common/db.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/common/db.go b/common/db.go
index 5532dc9..f20dbab 100644
--- a/common/db.go
+++ b/common/db.go
@@ -7,43 +7,88 @@ import (
 	"git.defalsify.org/vise.git/logging"
 )
 
+// DataType is a subprefix value used in association with vise/db.DATATYPE_USERDATA. 
+//
+// All keys are used only within the context of a single account. Unless otherwise specified, the user context is the session id.
+//
+// * The first byte is vise/db.DATATYPE_USERDATA
+// * The last 2 bytes are the DataTyp value, big-endian.
+// * The intermediate bytes are the id of the user context.
+//
+// All values are strings
 type DataTyp uint16
 
 const (
+	// TODO: Seems unused
 	DATA_ACCOUNT DataTyp = iota
+	// TODO: Seems unused, only read not written
 	DATA_ACCOUNT_CREATED
+	// API Tracking id to follow status of account creation
 	DATA_TRACKING_ID
+	// EVM address returned from API on account creation
 	DATA_PUBLIC_KEY
+	// TODO: Seems unused
 	DATA_CUSTODIAL_ID
+	// Currently active PIN used to authenticate ussd state change requests
 	DATA_ACCOUNT_PIN
+	// TODO: Seems unused
 	DATA_ACCOUNT_STATUS
+	// The first name of the user
 	DATA_FIRST_NAME
+	// The last name of the user
 	DATA_FAMILY_NAME
+	// The year-of-birth of the user
 	DATA_YOB
+	// The location of the user
 	DATA_LOCATION
+	// The gender of the user
 	DATA_GENDER
+	// The offerings description of the user
 	DATA_OFFERINGS
+	// The ethereum address of the recipient of an ongoing send request
 	DATA_RECIPIENT
+	// The voucher value amount of an ongoing send request
 	DATA_AMOUNT
+	// A general swap field for temporary values
 	DATA_TEMPORARY_VALUE
+	// Currently active voucher symbol of user
 	DATA_ACTIVE_SYM
+	// Voucher balance of user's currently active voucher
 	DATA_ACTIVE_BAL
+	// String boolean indicating whether use of PIN is blocked
 	DATA_BLOCKED_NUMBER
+	// Reverse mapping of a user's evm address to a session id.
 	DATA_PUBLIC_KEY_REVERSE
+	// Decimal count of the currently active voucher
 	DATA_ACTIVE_DECIMAL
+	// EVM address of the currently active voucher
 	DATA_ACTIVE_ADDRESS
-	// Start the sub prefix data at 256 (0x0100)
+)
+
+const (
+	// List of valid voucher symbols in the user context.
 	DATA_VOUCHER_SYMBOLS DataTyp = 256 + iota
+	// List of voucher balances for vouchers valid in the user context.
 	DATA_VOUCHER_BALANCES
+	// List of voucher decimal counts for vouchers valid in the user context.
 	DATA_VOUCHER_DECIMALS
+	// List of voucher EVM addresses for vouchers valid in the user context.
 	DATA_VOUCHER_ADDRESSES
+	// List of senders for valid transactions in the user context.
 	DATA_TX_SENDERS
+	// List of recipients for valid transactions in the user context.
 	DATA_TX_RECIPIENTS
+	// List of voucher values for valid transactions in the user context.
 	DATA_TX_VALUES
+	// List of voucher EVM addresses for valid transactions in the user context.
 	DATA_TX_ADDRESSES
+	// List of valid transaction hashes in the user context.
 	DATA_TX_HASHES
+	// List of transaction dates for valid transactions in the user context.
 	DATA_TX_DATES
+	// List of voucher symbols for valid transactions in the user context.
 	DATA_TX_SYMBOLS
+	// List of voucher decimal counts for valid transactions in the user context.
 	DATA_TX_DECIMALS
 )
 

From a3e821fb16f5cb994175d042a39e4e43eca21121 Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 8 Dec 2024 21:57:53 +0000
Subject: [PATCH 08/16] Clarify gen cmd example with reverse public key lookup

---
 devtools/gen/main.go | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/devtools/gen/main.go b/devtools/gen/main.go
index 401f194..6b58a50 100644
--- a/devtools/gen/main.go
+++ b/devtools/gen/main.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/sha1"
 	"flag"
 	"fmt"
 	"os"
@@ -46,7 +47,18 @@ func main() {
 	}
 	userStore := common.UserDataStore{store}
 
-	err = userStore.WriteEntry(ctx, sessionId, common.DATA_AMOUNT, []byte("1.0"))
+	h := sha1.New()
+	h.Write([]byte(sessionId))
+	address := h.Sum(nil)
+	addressString := fmt.Sprintf("%x", address)
+
+	err = userStore.WriteEntry(ctx, sessionId, common.DATA_PUBLIC_KEY, []byte(addressString))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	err = userStore.WriteEntry(ctx, addressString, common.DATA_PUBLIC_KEY_REVERSE, []byte(sessionId))
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)

From 604c16ec90ddcf2e97719c0893c0747a945bf681 Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sun, 8 Dec 2024 22:48:39 +0000
Subject: [PATCH 09/16] Habilitate store dumper

---
 debug/db.go              | 10 ++++++++++
 devtools/gen/main.go     |  6 ++++++
 devtools/store/main.go   | 11 +++++++++--
 go.mod                   |  2 +-
 go.sum                   |  4 ++--
 internal/storage/gdbm.go |  2 ++
 6 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/debug/db.go b/debug/db.go
index eb62a41..ec9e58f 100644
--- a/debug/db.go
+++ b/debug/db.go
@@ -20,6 +20,16 @@ type KeyInfo struct {
 	Description string
 }
 
+func (k KeyInfo) String() string {
+	v := uint16(k.SubTyp)
+	s := subTypToString(k.SubTyp)
+	if s == "" {
+		v = uint16(k.Typ)
+		s = typToString(k.Typ)
+	}
+	return fmt.Sprintf("Session Id: %s\nTyp: %s (%d)\n", k.SessionId, s, v)
+}
+
 func ToKeyInfo(k []byte, sessionId string) (KeyInfo, error) {
 	o := KeyInfo{}
 	b := []byte(sessionId)
diff --git a/devtools/gen/main.go b/devtools/gen/main.go
index 6b58a50..b9e2aed 100644
--- a/devtools/gen/main.go
+++ b/devtools/gen/main.go
@@ -11,6 +11,7 @@ import (
 	"git.defalsify.org/vise.git/logging"
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/common"
 )
 
@@ -19,6 +20,11 @@ var (
 	scriptDir = path.Join("services", "registration")
 )
 
+func init() {
+	initializers.LoadEnvVariables()
+}
+
+
 func main() {
 	config.LoadConfig()
 
diff --git a/devtools/store/main.go b/devtools/store/main.go
index d6abec9..9262703 100644
--- a/devtools/store/main.go
+++ b/devtools/store/main.go
@@ -10,6 +10,7 @@ import (
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/debug"
 	"git.defalsify.org/vise.git/logging"
 )
 
@@ -30,6 +31,7 @@ func main() {
 	var sessionId string
 	var database string
 	var engineDebug bool
+
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
 	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
@@ -37,7 +39,7 @@ func main() {
 	flag.Parse()
 
 	ctx := context.Background()
-	//ctx = context.WithValue(ctx, "SessionId", sessionId)
+	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 
 	resourceDir := scriptDir
@@ -60,7 +62,12 @@ func main() {
 		if k == nil {
 			break
 		}
-		fmt.Printf("%x %s %x\n", k, v)
+		o, err := debug.FromKey(k)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, err.Error())
+			os.Exit(1)
+		}
+		fmt.Printf("%vValue: %v\n\n", o, v)
 	}
 
 	err = store.Close()
diff --git a/go.mod b/go.mod
index d89f0f3..aeac055 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd
 go 1.23.0
 
 require (
-	git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a
+	git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5
 	github.com/alecthomas/assert/v2 v2.2.2
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/grassrootseconomics/eth-custodial v1.3.0-beta
diff --git a/go.sum b/go.sum
index 5532c1c..5949d8b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a h1:LvGKktk0kUnuRN3nF9r15D8OoV0sFaMmQr52kGq2gtE=
-git.defalsify.org/vise.git v0.2.1-0.20241122120231-9e9ee5bdfa7a/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
+git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 h1:QI/6xmpFM9NsG1+vRPzlUkkjAQ1Z8WhNqS+2tCjBpxg=
+git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
 github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
 github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=
diff --git a/internal/storage/gdbm.go b/internal/storage/gdbm.go
index 9879fe0..31ebf47 100644
--- a/internal/storage/gdbm.go
+++ b/internal/storage/gdbm.go
@@ -116,5 +116,7 @@ func(tdb *ThreadGdbmDb) Close() error {
 }
 
 func(tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
+	tdb.reserve()
+	defer tdb.release()
 	return tdb.db.Dump(ctx, key)
 }

From c820e89cb751c3b7b6285c74afb18570282dc077 Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Wed, 11 Dec 2024 19:13:13 +0000
Subject: [PATCH 10/16] Segment tx userdata types, add internals docs

---
 common/db.go | 15 +++++----------
 doc/data.md  | 28 ++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 10 deletions(-)
 create mode 100644 doc/data.md

diff --git a/common/db.go b/common/db.go
index f20dbab..a5cf1c1 100644
--- a/common/db.go
+++ b/common/db.go
@@ -19,20 +19,12 @@ import (
 type DataTyp uint16
 
 const (
-	// TODO: Seems unused
-	DATA_ACCOUNT DataTyp = iota
-	// TODO: Seems unused, only read not written
-	DATA_ACCOUNT_CREATED
 	// API Tracking id to follow status of account creation
-	DATA_TRACKING_ID
+	DATA_TRACKING_ID = iota
 	// EVM address returned from API on account creation
 	DATA_PUBLIC_KEY
-	// TODO: Seems unused
-	DATA_CUSTODIAL_ID
 	// Currently active PIN used to authenticate ussd state change requests
 	DATA_ACCOUNT_PIN
-	// TODO: Seems unused
-	DATA_ACCOUNT_STATUS
 	// The first name of the user
 	DATA_FIRST_NAME
 	// The last name of the user
@@ -75,7 +67,10 @@ const (
 	// List of voucher EVM addresses for vouchers valid in the user context.
 	DATA_VOUCHER_ADDRESSES
 	// List of senders for valid transactions in the user context.
-	DATA_TX_SENDERS
+)
+
+const (
+	DATA_TX_SENDERS = 512 + iota
 	// List of recipients for valid transactions in the user context.
 	DATA_TX_RECIPIENTS
 	// List of voucher values for valid transactions in the user context.
diff --git a/doc/data.md b/doc/data.md
new file mode 100644
index 0000000..7a41f0c
--- /dev/null
+++ b/doc/data.md
@@ -0,0 +1,28 @@
+# Internals
+
+## Version
+
+This document describes component versions:
+
+* `urdt-ussd` `v0.5.0-beta`
+* `go-vise` `v0.2.2`
+
+
+## User profile data
+
+All user profile items are stored under keys matching the user's session id, prefixed with the 8-bit value `git.defalsify.org/vise.git/db.DATATYPE_USERDATA` (32), and followed with a 16-big big-endian value subprefix.
+
+For example, given the sessionId `+254123` and the key `git.grassecon.net/urdt-ussd/common.DATA_PUBLIC_KEY` (2) will be stored under the key:
+
+```
+0x322b3235343132330002
+
+prefix   sessionid       subprefix
+32       2b323534313233  0002
+```
+
+### Sub-prefixes
+
+All sub-prefixes are defined as constants in the `git.grassecon.net/urdt-ussd/common` package. The constant names have the prefix `DATA_`
+
+Please refer to inline godoc documentation for the `git.grassecon.net/urdt-ussd/common` package for details on each data item.

From 8d4d8a48e02367842b43fd5940bfbaf13a56a2ba Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sat, 14 Dec 2024 11:56:31 +0000
Subject: [PATCH 11/16] Fix compile errors, test errors

---
 cmd/africastalking/main.go            | 13 ++++++-------
 internal/handlers/ussd/menuhandler.go |  2 +-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go
index 7d1caa8..98864db 100644
--- a/cmd/africastalking/main.go
+++ b/cmd/africastalking/main.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"os"
 	"os/signal"
@@ -45,14 +44,14 @@ type atRequestParser struct{}
 func (arp *atRequestParser) GetSessionId(rq any) (string, error) {
 	rqv, ok := rq.(*http.Request)
 	if !ok {
-		log.Printf("got an invalid request:", rq)
+		logg.Warnf("got an invalid request", "req", rq)
 		return "", handlers.ErrInvalidRequest
 	}
 
 	// Capture body (if any) for logging
 	body, err := io.ReadAll(rqv.Body)
 	if err != nil {
-		log.Printf("failed to read request body:", err)
+		logg.Warnf("failed to read request body", "err", err)
 		return "", fmt.Errorf("failed to read request body: %v", err)
 	}
 	// Reset the body for further reading
@@ -62,13 +61,13 @@ func (arp *atRequestParser) GetSessionId(rq any) (string, error) {
 	bodyLog := map[string]string{"body": string(body)}
 	logBytes, err := json.Marshal(bodyLog)
 	if err != nil {
-		log.Printf("failed to marshal request body:", err)
+		logg.Warnf("failed to marshal request body", "err", err)
 	} else {
-		log.Printf("Received request:", string(logBytes))
+		logg.Debugf("received request", "bytes", logBytes)
 	}
 
 	if err := rqv.ParseForm(); err != nil {
-		log.Printf("failed to parse form data: %v", err)
+		logg.Warnf("failed to parse form data", "err", err)
 		return "", fmt.Errorf("failed to parse form data: %v", err)
 	}
 
@@ -79,7 +78,7 @@ func (arp *atRequestParser) GetSessionId(rq any) (string, error) {
 
 	formattedNumber, err := common.FormatPhoneNumber(phoneNumber)
 	if err != nil {
-		fmt.Printf("Error: %v\n", err)
+		logg.Warnf("failed to format phone number", "err", err)
 		return "", fmt.Errorf("failed to format number")
 	}
 
diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go
index b47e31f..dff01d1 100644
--- a/internal/handlers/ussd/menuhandler.go
+++ b/internal/handlers/ussd/menuhandler.go
@@ -213,7 +213,7 @@ func (h *Handlers) CreateAccount(ctx context.Context, sym string, input []byte)
 		return res, fmt.Errorf("missing session")
 	}
 	store := h.userdataStore
-	_, err = store.ReadEntry(ctx, sessionId, common.DATA_ACCOUNT_CREATED)
+	_, err = store.ReadEntry(ctx, sessionId, common.DATA_PUBLIC_KEY)
 	if err != nil {
 		if db.IsNotFound(err) {
 			logg.InfoCtxf(ctx, "Creating an account because it doesn't exist")

From 1ab49647f61f98bb1fd5c2b306ac53a27abf7e9e Mon Sep 17 00:00:00 2001
From: lash <dev@holbrook.no>
Date: Sat, 14 Dec 2024 12:02:13 +0000
Subject: [PATCH 12/16] Upgrade vise

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index aeac055..e1b7ddb 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd
 go 1.23.0
 
 require (
-	git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5
+	git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80
 	github.com/alecthomas/assert/v2 v2.2.2
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/grassrootseconomics/eth-custodial v1.3.0-beta
diff --git a/go.sum b/go.sum
index 5949d8b..ef7b782 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5 h1:QI/6xmpFM9NsG1+vRPzlUkkjAQ1Z8WhNqS+2tCjBpxg=
-git.defalsify.org/vise.git v0.2.1-0.20241208224318-af1d15258dc5/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
+git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80 h1:GYUVXRUtMpA40T4COeAduoay6CIgXjD5cfDYZOTFIKw=
+git.defalsify.org/vise.git v0.2.1-0.20241212145627-683015d4df80/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
 github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
 github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=

From 549782f2309cb6acadb90698cf763514e099a940 Mon Sep 17 00:00:00 2001
From: Carlosokumu <carlosokumu254@gmail.com>
Date: Tue, 17 Dec 2024 15:12:38 +0300
Subject: [PATCH 13/16] add link to terms page

---
 services/registration/terms     | 3 ++-
 services/registration/terms_swa | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/services/registration/terms b/services/registration/terms
index 05b8c11..2081d41 100644
--- a/services/registration/terms
+++ b/services/registration/terms
@@ -1 +1,2 @@
-Do you agree to terms and conditions?
\ No newline at end of file
+Do you agree to terms and conditions?
+https://grassecon.org/pages/terms-and-conditions
\ No newline at end of file
diff --git a/services/registration/terms_swa b/services/registration/terms_swa
index 7113cd7..cd6f945 100644
--- a/services/registration/terms_swa
+++ b/services/registration/terms_swa
@@ -1 +1,2 @@
-Kwa kutumia hii huduma umekubali sheria na masharti?
\ No newline at end of file
+Kwa kutumia hii huduma umekubali sheria na masharti?
+https://grassecon.org/pages/terms-and-conditions
\ No newline at end of file

From fd6ff86579ab233921e534f6067eceb9067ebd52 Mon Sep 17 00:00:00 2001
From: Carlosokumu <carlosokumu254@gmail.com>
Date: Tue, 17 Dec 2024 15:24:15 +0300
Subject: [PATCH 14/16] add link to terms and conditions as expected content

---
 menutraversal_test/test_setup.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/menutraversal_test/test_setup.json b/menutraversal_test/test_setup.json
index 1f9d61d..caea216 100644
--- a/menutraversal_test/test_setup.json
+++ b/menutraversal_test/test_setup.json
@@ -11,7 +11,7 @@
                     },
                     {
                         "input": "0",
-                        "expectedContent": "Do you agree to terms and conditions?\n0:Yes\n1:No"
+                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n0:Yes\n1:No"
                     },
                     {
                         "input": "0",
@@ -44,7 +44,7 @@
                     },
                     {
                         "input": "0",
-                        "expectedContent": "Do you agree to terms and conditions?\n0:Yes\n1:No"
+                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n0:Yes\n1:No"
                     },
                     {
                         "input": "1",

From 70b2fa4ac2de1fa6893e9aabbd22e06ba0c649ec Mon Sep 17 00:00:00 2001
From: Carlosokumu <carlosokumu254@gmail.com>
Date: Tue, 17 Dec 2024 15:46:28 +0300
Subject: [PATCH 15/16] add  spacing after link

---
 services/registration/terms     | 2 +-
 services/registration/terms_swa | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/registration/terms b/services/registration/terms
index 2081d41..8af5b06 100644
--- a/services/registration/terms
+++ b/services/registration/terms
@@ -1,2 +1,2 @@
 Do you agree to terms and conditions?
-https://grassecon.org/pages/terms-and-conditions
\ No newline at end of file
+https://grassecon.org/pages/terms-and-conditions
diff --git a/services/registration/terms_swa b/services/registration/terms_swa
index cd6f945..5678186 100644
--- a/services/registration/terms_swa
+++ b/services/registration/terms_swa
@@ -1,2 +1,2 @@
 Kwa kutumia hii huduma umekubali sheria na masharti?
-https://grassecon.org/pages/terms-and-conditions
\ No newline at end of file
+https://grassecon.org/pages/terms-and-conditions

From 9b33117cb15957a97667e2efeda1a4f3a9ce89d3 Mon Sep 17 00:00:00 2001
From: Carlosokumu <carlosokumu254@gmail.com>
Date: Tue, 17 Dec 2024 16:02:35 +0300
Subject: [PATCH 16/16] add space on expected content

---
 menutraversal_test/test_setup.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/menutraversal_test/test_setup.json b/menutraversal_test/test_setup.json
index caea216..c5860b4 100644
--- a/menutraversal_test/test_setup.json
+++ b/menutraversal_test/test_setup.json
@@ -11,7 +11,7 @@
                     },
                     {
                         "input": "0",
-                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n0:Yes\n1:No"
+                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n\n0:Yes\n1:No"
                     },
                     {
                         "input": "0",
@@ -44,7 +44,7 @@
                     },
                     {
                         "input": "0",
-                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n0:Yes\n1:No"
+                        "expectedContent": "Do you agree to terms and conditions?\nhttps://grassecon.org/pages/terms-and-conditions\n\n0:Yes\n1:No"
                     },
                     {
                         "input": "1",