Merge branch 'master' into lash/stalecache

.env.example

@@ -6,9 +6,13 @@ HOST=127.0.0.1
 AT_ENDPOINT=/ussd/africastalking
 
 #PostgreSQL
-DB_CONN=postgres://postgres:strongpass@localhost:5432/urdt_ussd
-#DB_TIMEZONE=Africa/Nairobi
-#DB_SCHEMA=vise
+DB_HOST=localhost
+DB_USER=postgres
+DB_PASSWORD=strongpass
+DB_NAME=urdt_ussd
+DB_PORT=5432
+DB_SSLMODE=disable
+DB_TIMEZONE=Africa/Nairobi
 
 #External API Calls
 CUSTODIAL_URL_BASE=http://localhost:5003

cmd/africastalking/main.go

@@ -19,6 +19,7 @@ import (
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/http/at"
+	httpserver "git.grassecon.net/urdt/ussd/internal/http/at"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
 )
@@ -33,37 +34,26 @@ var (
 func init() {
 	initializers.LoadEnvVariables()
 }
-
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "build", build, "conn", connData, "resourcedir", resourceDir, "outputsize", size)
+	logg.Infof("start command", "build", build, "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
@@ -80,13 +70,14 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	rs, err := menuStorageService.GetResource(ctx)
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -130,9 +121,11 @@ func main() {
 	}
 	defer stateStore.Close()
 
-	rp := &at.ATRequestParser{}
+	rp := &at.ATRequestParser{
+		Context: ctx,
+	}
 	bsh := handlers.NewBaseSessionHandler(cfg, rs, stateStore, userdataStore, rp, hl)
-	sh := at.NewATSessionHandler(bsh)
+	sh := httpserver.NewATSessionHandler(bsh)
 
 	mux := http.NewServeMux()
 	mux.Handle(initializers.GetEnv("AT_ENDPOINT", "/"), sh)

cmd/async/main.go

@@ -21,8 +21,8 @@ import (
 )
 
 var (
-	logg          = logging.NewVanilla()
-	scriptDir     = path.Join("services", "registration")
+	logg      = logging.NewVanilla()
+	scriptDir = path.Join("services", "registration")
 	menuSeparator = ": "
 )
 
@@ -35,7 +35,7 @@ type asyncRequestParser struct {
 	input     []byte
 }
 
-func (p *asyncRequestParser) GetSessionId(ctx context.Context, r any) (string, error) {
+func (p *asyncRequestParser) GetSessionId(r any) (string, error) {
 	return p.sessionId, nil
 }
 
@@ -46,35 +46,25 @@ func (p *asyncRequestParser) GetInput(r any) ([]byte, error) {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
 	var sessionId string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
+	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
@@ -91,18 +81,18 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, err.Error())
-		os.Exit(1)
-	}
-
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
+	err = menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {

cmd/http/main.go

@@ -36,33 +36,23 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size)
+	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
@@ -79,14 +69,19 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
+	err = menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
 	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())

cmd/main.go

@@ -30,31 +30,19 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var size uint
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var resourceDir string
-	var err error
-
-	flag.StringVar(&resourceDir, "resourcedir", scriptDir, "resource dir")
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData, "outputsize", size)
+	logg.Infof("start command", "dbdir", dbDir, "outputsize", size)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
@@ -69,7 +57,14 @@ func main() {
 		MenuSeparator: menuSeparator,
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	resourceDir := scriptDir
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
+	err := menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {

cmd/ssh/README.md

@@ -1,34 +0,0 @@
-# URDT-USSD SSH server
-
-An SSH server entry point for the vise engine.
-
-
-## Adding public keys for access
-
-Map your (client) public key to a session identifier (e.g. phone number)
-
-```
-go run -v -tags logtrace ./cmd/ssh/sshkey/main.go -i <session_id> [--dbdir <dbpath>] <client_publickey_filepath>
-```
-
-
-## Create a private key for the server
-
-```
-ssh-keygen -N "" -f <server_privatekey_filepath>
-```
-
-
-## Run the server
-
-
-```
-go run -v -tags logtrace ./cmd/ssh/main.go -h <host> -p <port> [--dbdir <dbpath>] <server_privatekey_filepath>
-```
-
-
-## Connect to the server
-
-```
-ssh [-v] -T -p <port> -i <client_publickey_filepath> <host>
-```

cmd/ssh/main.go

@@ -1,133 +0,0 @@
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"path"
-	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-
-	"git.defalsify.org/vise.git/db"
-	"git.defalsify.org/vise.git/engine"
-	"git.defalsify.org/vise.git/logging"
-
-	"git.grassecon.net/urdt/ussd/config"
-	"git.grassecon.net/urdt/ussd/initializers"
-	"git.grassecon.net/urdt/ussd/internal/ssh"
-	"git.grassecon.net/urdt/ussd/internal/storage"
-)
-
-var (
-	wg sync.WaitGroup
-	keyStore db.Db
-	logg      = logging.NewVanilla()
-	scriptDir = path.Join("services", "registration")
-)
-
-func init() {
-	initializers.LoadEnvVariables()
-}
-
-func main() {
-	config.LoadConfig()
-
-	var connStr string
-	var dbDir string
-	var resourceDir string
-	var size uint
-	var engineDebug bool
-	var stateDebug bool
-	var host string
-	var port uint
-	flag.StringVar(&connStr, "c", "", "connection string")
-	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
-	flag.UintVar(&size, "s", 160, "max size of output")
-	flag.StringVar(&host, "h", "127.0.0.1", "socket host")
-	flag.UintVar(&port, "p", 7122, "socket port")
-	flag.Parse()
-
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	sshKeyFile := flag.Arg(0)
-	_, err = os.Stat(sshKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "cannot open ssh server private key file: %v\n", err)
-		os.Exit(1)
-	}
-
-	ctx := context.Background()
-	logg.WarnCtxf(ctx, "!!!!! WARNING WARNING WARNING")
-	logg.WarnCtxf(ctx, "!!!!! =======================")
-	logg.WarnCtxf(ctx, "!!!!! This is not a production ready server!")
-	logg.WarnCtxf(ctx, "!!!!! Do not expose to internet and only use with tunnel!")
-	logg.WarnCtxf(ctx, "!!!!! (See ssh -L <...>)")
-
-	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "keyfile", sshKeyFile, "host", host, "port", port)
-
-	pfp := path.Join(scriptDir, "pp.csv")
-
-	cfg := engine.Config{
-		Root:       "root",
-		OutputSize: uint32(size),
-		FlagCount:  uint32(16),
-	}
-	if stateDebug {
-		cfg.StateDebug = true
-	}
-	if engineDebug {
-		cfg.EngineDebug = true
-	}
-
-	authKeyStore, err := ssh.NewSshKeyStore(ctx, dbDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "keystore file open error: %v", err)
-		os.Exit(1)
-	}
-	defer func () {
-		logg.TraceCtxf(ctx, "shutdown auth key store reached")
-		err = authKeyStore.Close()
-		if err != nil {
-			logg.ErrorCtxf(ctx, "keystore close error", "err", err)
-		}
-	}()
-
-	cint := make(chan os.Signal)
-	cterm := make(chan os.Signal)
-	signal.Notify(cint, os.Interrupt, syscall.SIGINT)
-	signal.Notify(cterm, os.Interrupt, syscall.SIGTERM)
-
-	runner := &ssh.SshRunner{
-		Cfg: cfg,
-		Debug: engineDebug,
-		FlagFile: pfp,
-		Conn: connData,
-		ResourceDir: resourceDir,
-		SrvKeyFile: sshKeyFile,
-		Host: host,
-		Port: port,
-	}
-	go func() {
-		select {
-		case _ = <-cint:
-		case _ = <-cterm:
-		}
-		logg.TraceCtxf(ctx, "shutdown runner reached")
-		err := runner.Stop()
-		if err != nil {
-			logg.ErrorCtxf(ctx, "runner stop error", "err", err)
-		}
-		
-	}()
-	runner.Run(ctx, authKeyStore)
-}

cmd/ssh/sshkey/main.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"os"
-
-	"git.grassecon.net/urdt/ussd/internal/ssh"
-)
-
-func main() {
-	var dbDir string
-	var sessionId string
-	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
-	flag.StringVar(&sessionId, "i", "", "session id")
-	flag.Parse()
-
-	if sessionId == "" {
-		fmt.Fprintf(os.Stderr, "empty session id\n")
-		os.Exit(1)
-	}
-
-	ctx := context.Background()
-
-	sshKeyFile := flag.Arg(0)
-	if sshKeyFile == "" {
-		fmt.Fprintf(os.Stderr, "missing key file argument\n")
-		os.Exit(1)
-	}
-
-	store, err := ssh.NewSshKeyStore(ctx, dbDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-	defer store.Close()
-
-	err = store.AddFromFile(ctx, sshKeyFile, sessionId)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-}

common/storage.go

@@ -23,17 +23,17 @@ type StorageServices interface {
 	GetPersister(ctx context.Context) (*persist.Persister, error)
 	GetUserdataDb(ctx context.Context) (db.Db, error)
 	GetResource(ctx context.Context) (resource.Resource, error)
+	EnsureDbDir() error
 }
 
 type StorageService struct {
 	svc *storage.MenuStorageService
 }
 
-func NewStorageService(conn storage.ConnData) (*StorageService, error) {
-	svc := &StorageService{
-		svc: storage.NewMenuStorageService(conn, ""),
+func NewStorageService(dbDir string) *StorageService {
+	return &StorageService{
+		svc: storage.NewMenuStorageService(dbDir, ""),
 	}
-	return svc, nil
 }
 
 func(ss *StorageService) GetPersister(ctx context.Context) (*persist.Persister, error) {
@@ -47,3 +47,7 @@ func(ss *StorageService) GetUserdataDb(ctx context.Context) (db.Db, error) {
 func(ss *StorageService) GetResource(ctx context.Context) (resource.Resource, error) {
 	return nil, errors.New("not implemented")
 }
+
+func(ss *StorageService) EnsureDbDir() error {
+	return ss.svc.EnsureDbDir()
+}

config/config.go

@@ -34,7 +34,6 @@ var (
 	VoucherTransfersURL string
 	VoucherDataURL      string
 	CheckAliasURL       string
-	DbConn		string
 )
 
 func setBase() error {
@@ -44,20 +43,14 @@ func setBase() error {
 	dataURLBase = initializers.GetEnv("DATA_URL_BASE", "http://localhost:5006")
 	BearerToken = initializers.GetEnv("BEARER_TOKEN", "")
 
-	_, err = url.Parse(custodialURLBase)
+	_, err = url.JoinPath(custodialURLBase, "/foo")
 	if err != nil {
 		return err
 	}
-	_, err = url.Parse(dataURLBase)
+	_, err = url.JoinPath(dataURLBase, "/bar")
 	if err != nil {
 		return err
 	}
-
-	return nil
-}
-
-func setConn() error {
-	DbConn = initializers.GetEnv("DB_CONN", "")
 	return nil
 }
 
@@ -67,10 +60,6 @@ func LoadConfig() error {
 	if err != nil {
 		return err
 	}
-	err = setConn()
-	if err != nil {
-		return err
-	}
 	CreateAccountURL, _ = url.JoinPath(custodialURLBase, createAccountPath)
 	TrackStatusURL, _ = url.JoinPath(custodialURLBase, trackStatusPath)
 	BalanceURL, _ = url.JoinPath(custodialURLBase, balancePathPrefix)

devtools/gen/main.go

@@ -28,35 +28,24 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var err error
 
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData)
-
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
 	store, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())

devtools/store/main.go

@@ -28,34 +28,23 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var err error
 
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", ".state", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData)
-
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 
 	store, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {

internal/handlers/single.go

@@ -6,9 +6,9 @@ import (
 	"io"
 
 	"git.defalsify.org/vise.git/engine"
-	"git.defalsify.org/vise.git/logging"
-	"git.defalsify.org/vise.git/persist"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/persist"
+	"git.defalsify.org/vise.git/logging"
 
 	"git.grassecon.net/urdt/ussd/internal/storage"
 )
@@ -20,33 +20,33 @@ var (
 var (
 	ErrInvalidRequest = errors.New("invalid request for context")
 	ErrSessionMissing = errors.New("missing session")
-	ErrInvalidInput   = errors.New("invalid input")
-	ErrStorage        = errors.New("storage retrieval fail")
-	ErrEngineType     = errors.New("incompatible engine")
-	ErrEngineInit     = errors.New("engine init fail")
-	ErrEngineExec     = errors.New("engine exec fail")
+	ErrInvalidInput = errors.New("invalid input")
+	ErrStorage = errors.New("storage retrieval fail")
+	ErrEngineType = errors.New("incompatible engine")
+	ErrEngineInit = errors.New("engine init fail")
+	ErrEngineExec = errors.New("engine exec fail")
 )
 
 type RequestSession struct {
-	Ctx      context.Context
-	Config   engine.Config
-	Engine   engine.Engine
-	Input    []byte
-	Storage  *storage.Storage
-	Writer   io.Writer
+	Ctx context.Context
+	Config engine.Config
+	Engine engine.Engine
+	Input []byte
+	Storage *storage.Storage
+	Writer io.Writer
 	Continue bool
 }
 
 // TODO: seems like can remove this.
 type RequestParser interface {
-	GetSessionId(context context.Context, rq any) (string, error)
+	GetSessionId(rq any) (string, error)
 	GetInput(rq any) ([]byte, error)
 }
 
 type RequestHandler interface {
 	GetConfig() engine.Config
 	GetRequestParser() RequestParser
-	GetEngine(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine
+	GetEngine(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine 
 	Process(rs RequestSession) (RequestSession, error)
 	Output(rs RequestSession) (RequestSession, error)
 	Reset(rs RequestSession) (RequestSession, error)

internal/handlers/ussd/menuhandler.go

@@ -28,7 +28,7 @@ import (
 )
 
 var (
-	logg           = logging.NewVanilla().WithDomain("ussdmenuhandler").WithContextKey("SessionId")
+	logg           = logging.NewVanilla().WithDomain("ussdmenuhandler").WithContextKey("session-id")
 	scriptDir      = path.Join("services", "registration")
 	translationDir = path.Join(scriptDir, "locale")
 )
@@ -124,7 +124,7 @@ func (h *Handlers) Init(ctx context.Context, sym string, input []byte) (resource
 
 	sessionId, ok := ctx.Value("SessionId").(string)
 	if ok {
-		ctx = context.WithValue(ctx, "SessionId", sessionId)
+		context.WithValue(ctx, "session-id", sessionId)
 	}
 
 	flag_admin_privilege, _ := h.flagManager.GetFlag("flag_admin_privilege")

internal/http/at/parse.go

@@ -15,14 +15,16 @@ import (
 )
 
 type ATRequestParser struct {
+	Context context.Context
 }
 
-func (arp *ATRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) {
+func (arp *ATRequestParser) GetSessionId(rq any) (string, error) {
 	rqv, ok := rq.(*http.Request)
 	if !ok {
 		logg.Warnf("got an invalid request", "req", rq)
 		return "", handlers.ErrInvalidRequest
 	}
+
 	// Capture body (if any) for logging
 	body, err := io.ReadAll(rqv.Body)
 	if err != nil {
@@ -41,9 +43,9 @@ func (arp *ATRequestParser) GetSessionId(ctx context.Context, rq any) (string, e
 		decodedStr := string(logBytes)
 		sessionId, err := extractATSessionId(decodedStr)
 		if err != nil {
-			ctx = context.WithValue(ctx, "AT-SessionId", sessionId)
+			context.WithValue(arp.Context, "at-session-id", sessionId)
 		}
-		logg.DebugCtxf(ctx, "Received request:", decodedStr)
+		logg.Debugf("Received request:", decodedStr)
 	}
 
 	if err := rqv.ParseForm(); err != nil {

internal/http/at/server.go

@@ -10,7 +10,7 @@ import (
 )
 
 var (
-	logg = logging.NewVanilla().WithDomain("atserver").WithContextKey("SessionId").WithContextKey("AT-SessionId")
+	logg = logging.NewVanilla().WithDomain("atserver")
 )
 
 type ATSessionHandler struct {
@@ -34,7 +34,7 @@ func (ash *ATSessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
 
 	rp := ash.GetRequestParser()
 	cfg := ash.GetConfig()
-	cfg.SessionId, err = rp.GetSessionId(req.Context(), req)
+	cfg.SessionId, err = rp.GetSessionId(req)
 	if err != nil {
 		logg.ErrorCtxf(rqs.Ctx, "", "header processing error", err)
 		ash.WriteError(w, 400, err)
@@ -48,7 +48,7 @@ func (ash *ATSessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)
 		return
 	}
 
-	rqs, err = ash.Process(rqs)
+	rqs, err = ash.Process(rqs) 
 	switch err {
 	case nil: // set code to 200 if no err
 		code = 200

internal/http/parse.go

@@ -1,7 +1,6 @@
 package http
 
 import (
-	"context"
 	"io/ioutil"
 	"net/http"
 
@@ -11,7 +10,7 @@ import (
 type DefaultRequestParser struct {
 }
 
-func (rp *DefaultRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) {
+func (rp *DefaultRequestParser) GetSessionId(rq any) (string, error) {
 	rqv, ok := rq.(*http.Request)
 	if !ok {
 		return "", handlers.ErrInvalidRequest
@@ -35,3 +34,5 @@ func (rp *DefaultRequestParser) GetInput(rq any) ([]byte, error) {
 	}
 	return v, nil
 }
+
+

internal/http/server.go

@@ -46,7 +46,7 @@ func (f *SessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 	rp := f.GetRequestParser()
 	cfg := f.GetConfig()
-	cfg.SessionId, err = rp.GetSessionId(req.Context(), req)
+	cfg.SessionId, err = rp.GetSessionId(req)
 	if err != nil {
 		logg.ErrorCtxf(rqs.Ctx, "", "header processing error", err)
 		f.WriteError(w, 400, err)

internal/http/server_test.go

@@ -2,7 +2,6 @@ package http
 
 import (
 	"bytes"
-	"context"
 	"errors"
 	"net/http"
 	"net/http/httptest"
@@ -162,7 +161,7 @@ func TestDefaultRequestParser_GetSessionId(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			id, err := parser.GetSessionId(context.Background(),tt.request)
+			id, err := parser.GetSessionId(tt.request)
 
 			if id != tt.expectedID {
 				t.Errorf("Expected session ID %s, got %s", tt.expectedID, id)

internal/ssh/keystore.go

@@ -1,65 +0,0 @@
-package ssh
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path"
-
-	"golang.org/x/crypto/ssh"
-
-	"git.defalsify.org/vise.git/db"
-
-	"git.grassecon.net/urdt/ussd/internal/storage"
-	dbstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm"
-)
-
-type SshKeyStore struct {
-	store db.Db
-}
-
-func NewSshKeyStore(ctx context.Context, dbDir string) (*SshKeyStore, error) {
-	keyStore := &SshKeyStore{}
-	keyStoreFile := path.Join(dbDir, "ssh_authorized_keys.gdbm")
-	keyStore.store = dbstorage.NewThreadGdbmDb()
-	err := keyStore.store.Connect(ctx, keyStoreFile)
-	if err != nil {
-		return nil, err
-	}
-	return keyStore, nil
-}
-
-func(s *SshKeyStore) AddFromFile(ctx context.Context, fp string, sessionId string) error {
-	_, err := os.Stat(fp)
-	if err != nil {
-		return fmt.Errorf("cannot open ssh server public key file: %v\n", err)
-	}
-
-	publicBytes, err := os.ReadFile(fp)
-	if err != nil {
-		return fmt.Errorf("Failed to load public key: %v", err)
-	}
-	pubKey, _, _, _, err := ssh.ParseAuthorizedKey(publicBytes)
-	if err != nil {
-		return fmt.Errorf("Failed to parse public key: %v", err)
-	}
-	k := append([]byte{0x01}, pubKey.Marshal()...)
-	s.store.SetPrefix(storage.DATATYPE_EXTEND)
-	logg.Infof("Added key", "sessionId", sessionId, "public key", string(publicBytes))
-	return s.store.Put(ctx, k, []byte(sessionId))
-}
-
-func(s *SshKeyStore) Get(ctx context.Context, pubKey ssh.PublicKey) (string, error) {
-	s.store.SetLanguage(nil)
-	s.store.SetPrefix(storage.DATATYPE_EXTEND)
-	k := append([]byte{0x01}, pubKey.Marshal()...)
-	v, err := s.store.Get(ctx, k)
-	if err != nil {
-		return "", err
-	}
-	return string(v), nil
-}
-
-func(s *SshKeyStore) Close() error {
-	return s.store.Close()
-}

internal/ssh/ssh.go

@@ -1,282 +0,0 @@
-package ssh
-
-import (
-	"context"
-	"encoding/hex"
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"net"
-	"os"
-	"sync"
-
-	"golang.org/x/crypto/ssh"
-
-	"git.defalsify.org/vise.git/engine"
-	"git.defalsify.org/vise.git/logging"
-	"git.defalsify.org/vise.git/resource"
-	"git.defalsify.org/vise.git/state"
-
-	"git.grassecon.net/urdt/ussd/internal/handlers"
-	"git.grassecon.net/urdt/ussd/internal/storage"
-	"git.grassecon.net/urdt/ussd/remote"
-)
-
-var (
-	logg = logging.NewVanilla().WithDomain("ssh")
-)
-
-type auther struct {
-	Ctx context.Context
-	keyStore *SshKeyStore
-	auth map[string]string
-}
-
-func NewAuther(ctx context.Context, keyStore *SshKeyStore) *auther {
-	return &auther{
-		Ctx: ctx,
-		keyStore: keyStore,
-		auth: make(map[string]string),
-	}
-}
-
-func(a *auther) Check(conn ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
-	va, err := a.keyStore.Get(a.Ctx, pubKey)
-	if err != nil {
-		return nil, err
-	}
-	ka := hex.EncodeToString(conn.SessionID())
-	a.auth[ka] = va 
-	fmt.Fprintf(os.Stderr, "connect: %s -> %s\n", ka, va)
-	return nil, nil
-}
-
-func(a *auther) FromConn(c *ssh.ServerConn) (string, error) {
-	if c == nil {
-		return "", errors.New("nil server conn")
-	}
-	if c.Conn == nil {
-		return "", errors.New("nil underlying conn")
-	}
-	return a.Get(c.Conn.SessionID())
-}
-
-
-func(a *auther) Get(k []byte) (string, error) {
-	ka := hex.EncodeToString(k)
-	v, ok := a.auth[ka]
-	if !ok {
-		return "", errors.New("not found")
-	}
-	return v, nil
-}
-
-type SshRunner struct {
-	Ctx context.Context
-	Cfg engine.Config
-	FlagFile string
-	Conn storage.ConnData
-	ResourceDir string
-	Debug bool
-	SrvKeyFile string
-	Host string
-	Port uint
-	wg sync.WaitGroup
-	lst net.Listener
-}
-
-func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChannel, en engine.Engine) error {
-	if ch == nil {
-		return errors.New("nil channel")
-	}
-	if ch.ChannelType() != "session" {
-		ch.Reject(ssh.UnknownChannelType, "that is not the channel you are looking for")
-		return errors.New("not a session")
-	}
-	channel, requests, err := ch.Accept()
-	if err != nil {
-		panic(err)
-	}
-	defer channel.Close()
-	s.wg.Add(1)
-	go func(reqIn <-chan *ssh.Request) {
-		defer s.wg.Done()
-		for req := range reqIn {
-			req.Reply(req.Type == "shell", nil)	
-		}
-		_ = requests
-	}(requests)
-
-	cont, err := en.Exec(ctx, []byte{})
-	if err != nil {
-		return fmt.Errorf("initial engine exec err: %v", err)
-	}
-
-	var input [state.INPUT_LIMIT]byte
-	for cont {
-		c, err := en.Flush(ctx, channel)
-		if err != nil {
-			return fmt.Errorf("flush err: %v", err)
-		}
-		_, err = channel.Write([]byte{0x0a})
-		if err != nil {
-			return fmt.Errorf("newline err: %v", err)
-		}
-		c, err = channel.Read(input[:])
-		if err != nil {
-			return fmt.Errorf("read input fail: %v", err)
-		}
-		logg.TraceCtxf(ctx, "input read", "c", c, "input", input[:c-1])
-		cont, err = en.Exec(ctx, input[:c-1])
-		if err != nil {
-			return fmt.Errorf("engine exec err: %v", err)
-		}
-		logg.TraceCtxf(ctx, "exec cont", "cont", cont, "en", en)
-		_ = c
-	}
-	c, err := en.Flush(ctx, channel)
-	if err != nil {
-		return fmt.Errorf("last flush err: %v", err)
-	}
-	_ = c
-	return nil
-}
-
-func(s *SshRunner) Stop() error {
-	return s.lst.Close()
-}
-
-func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) {
-	ctx := s.Ctx
-	menuStorageService := storage.NewMenuStorageService(s.Conn, s.ResourceDir)
-
-	rs, err := menuStorageService.GetResource(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	pe, err := menuStorageService.GetPersister(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	userdatastore, err := menuStorageService.GetUserdataDb(ctx)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	dbResource, ok := rs.(*resource.DbResource)
-	if !ok {
-		return nil, nil, err
-	}
-
-	lhs, err := handlers.NewLocalHandlerService(ctx, s.FlagFile, true, dbResource, s.Cfg, rs)
-	lhs.SetDataStore(&userdatastore)
-	lhs.SetPersister(pe)
-	lhs.Cfg.SessionId = sessionId
-
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// TODO: clear up why pointer here and by-value other cmds
-	accountService := &remote.AccountService{}
-	hl, err := lhs.GetHandler(accountService)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	en := lhs.GetEngine()
-	en = en.WithFirst(hl.Init)
-	if s.Debug {
-		en = en.WithDebug(nil)
-	}
-	// TODO: this is getting very hacky!
-	closer := func() {
-		err := menuStorageService.Close()
-		if err != nil {
-			logg.ErrorCtxf(ctx, "menu storage service cleanup fail", "err", err)
-		}
-	}
-	return en, closer, nil
-}
-
-// adapted example from crypto/ssh package, NewServerConn doc
-func(s *SshRunner) Run(ctx context.Context, keyStore *SshKeyStore) {
-	running := true
-
-	// TODO: waitgroup should probably not be global
-	defer s.wg.Wait()
-
-	auth := NewAuther(ctx, keyStore)
-	cfg := ssh.ServerConfig{
-		PublicKeyCallback: auth.Check,
-	}
-
-	privateBytes, err := os.ReadFile(s.SrvKeyFile)
-	if err != nil {
-		logg.ErrorCtxf(ctx, "Failed to load private key", "err", err)
-	}
-	private, err := ssh.ParsePrivateKey(privateBytes)
-	if err != nil {
-		logg.ErrorCtxf(ctx, "Failed to parse private key", "err", err)
-	}
-	srvPub := private.PublicKey()
-	srvPubStr := base64.StdEncoding.EncodeToString(srvPub.Marshal())
-	logg.InfoCtxf(ctx, "have server key", "type", srvPub.Type(), "public", srvPubStr)
-	cfg.AddHostKey(private)
-
-	s.lst, err = net.Listen("tcp", fmt.Sprintf("%s:%d", s.Host, s.Port))
-	if err != nil {
-		panic(err)
-	}
-
-	for running {
-		conn, err := s.lst.Accept()
-		if err != nil {
-			logg.ErrorCtxf(ctx, "ssh accept error", "err", err)
-			running = false
-			continue
-		}
-
-		go func(conn net.Conn) {
-			defer conn.Close()
-			for true {
-				srvConn, nC, rC, err := ssh.NewServerConn(conn, &cfg)
-				if err != nil {
-					logg.InfoCtxf(ctx, "rejected client", "err", err)
-					return
-				}
-				logg.DebugCtxf(ctx, "ssh client connected", "conn", srvConn)
-
-				s.wg.Add(1)
-				go func() {
-					ssh.DiscardRequests(rC)
-					s.wg.Done()
-				}()
-				
-				sessionId, err := auth.FromConn(srvConn)
-				if err != nil {
-					logg.ErrorCtxf(ctx, "Cannot find authentication")
-					return
-				}
-				en, closer, err := s.GetEngine(sessionId)
-				if err != nil {
-					logg.ErrorCtxf(ctx, "engine won't start", "err", err)
-					return
-				}
-				defer func() {
-					err := en.Finish()
-					if err != nil {
-						logg.ErrorCtxf(ctx, "engine won't stop", "err", err)
-					}
-					closer()
-				}()
-				for ch := range nC {
-					err = s.serve(ctx, sessionId, ch, en)
-					logg.ErrorCtxf(ctx, "ssh server finish", "err", err)
-				}
-			}
-		}(conn)
-	}
-}

internal/storage/db/gdbm/gdbm.go

@@ -17,6 +17,7 @@ var (
 	dbC map[string]chan db.Db
 )
 
+
 type ThreadGdbmDb struct {
 	db db.Db
 	connStr string

internal/storage/db/sub_prefix_db.go

@@ -6,6 +6,11 @@ import (
 	"git.defalsify.org/vise.git/db"
 )
 
+const (
+	DATATYPE_USERSUB = 64
+	SUBPREFIX_TIME = uint16(1)
+)
+
 // PrefixDb interface abstracts the database operations.
 type PrefixDb interface {
 	Get(ctx context.Context, key []byte) ([]byte, error)
@@ -26,8 +31,12 @@ func NewSubPrefixDb(store db.Db, pfx []byte) *SubPrefixDb {
 	}
 }
 
-func (s *SubPrefixDb) toKey(k []byte) []byte {
-	return append(s.pfx, k...)
+func(s *SubPrefixDb) SetSession(sessionId string) {
+	s.store.SetSession(sessionId)	
+}
+
+func(s *SubPrefixDb) toKey(k []byte) []byte {
+        return append(s.pfx, k...)
 }
 
 func (s *SubPrefixDb) Get(ctx context.Context, key []byte) ([]byte, error) {

internal/storage/parse.go

@@ -1,69 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"net/url"
-	"path"
-)
-
-const (
-	DBTYPE_MEM = iota
-	DBTYPE_GDBM
-	DBTYPE_POSTGRES
-)
-
-type ConnData struct {
-	typ int
-	str string
-}
-
-func (cd *ConnData) DbType() int {
-	return cd.typ
-}
-
-func (cd *ConnData) String() string {
-	return cd.str
-}
-
-func probePostgres(s string) (string, bool) {
-	v, err := url.Parse(s)
-	if err != nil {
-		return "", false
-	}
-	if v.Scheme != "postgres" {
-		return "", false
-	}
-	return s, true
-}
-
-func probeGdbm(s string) (string, bool) {
-	if !path.IsAbs(s) {
-		return "", false
-	}
-	s = path.Clean(s)
-	return s, true
-}
-
-func ToConnData(connStr string) (ConnData, error) {
-	var o ConnData
-
-	if connStr == "" {
-		return o, nil
-	}
-
-	v, ok := probePostgres(connStr)
-	if ok {
-		o.typ = DBTYPE_POSTGRES
-		o.str = v
-		return o, nil
-	}
-
-	v, ok = probeGdbm(connStr)
-	if ok {
-		o.typ = DBTYPE_GDBM
-		o.str = v
-		return o, nil
-	}
-
-	return o, fmt.Errorf("invalid connection string: %s", connStr)
-}

internal/storage/parse_test.go

@@ -1,28 +0,0 @@
-package storage
-
-import (
-	"testing"
-)
-
-func TestParseConnStr(t *testing.T) {
-	_, err := ToConnData("postgres://foo:bar@localhost:5432/baz")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("/foo/bar")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("/foo/bar/")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("foo/bar")
-	if err == nil {
-		t.Fatalf("expected error")
-	}
-	_, err = ToConnData("http://foo/bar")
-	if err == nil {
-		t.Fatalf("expected error")
-	}
-}

internal/storage/storage.go

@@ -5,10 +5,6 @@ import (
 	"git.defalsify.org/vise.git/persist"
 )
 
-const (
-	DATATYPE_EXTEND = 128
-)
-
 type Storage struct {
 	Persister *persist.Persister
 	UserdataDb db.Db	

internal/storage/storageservice.go

@@ -12,6 +12,7 @@ import (
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/persist"
 	"git.defalsify.org/vise.git/resource"
+	"git.grassecon.net/urdt/ussd/initializers"
 	gdbmstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm"
 )
 
@@ -23,48 +24,63 @@ type StorageService interface {
 	GetPersister(ctx context.Context) (*persist.Persister, error)
 	GetUserdataDb(ctx context.Context) db.Db
 	GetResource(ctx context.Context) (resource.Resource, error)
+	EnsureDbDir() error
 }
 
 type MenuStorageService struct {
-	conn ConnData
+	dbDir         string
 	resourceDir   string
 	resourceStore db.Db
 	stateStore    db.Db
 	userDataStore db.Db
 }
 
-func NewMenuStorageService(conn ConnData, resourceDir string) *MenuStorageService {
+func buildConnStr() string {
+	host := initializers.GetEnv("DB_HOST", "localhost")
+	user := initializers.GetEnv("DB_USER", "postgres")
+	password := initializers.GetEnv("DB_PASSWORD", "")
+	dbName := initializers.GetEnv("DB_NAME", "")
+	port := initializers.GetEnv("DB_PORT", "5432")
+
+	connString := fmt.Sprintf(
+		"postgres://%s:%s@%s:%s/%s",
+		user, password, host, port, dbName,
+	)
+	logg.Debugf("pg conn string", "conn", connString)
+
+	return connString
+}
+
+func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService {
 	return &MenuStorageService{
-		conn: conn,
+		dbDir:       dbDir,
 		resourceDir: resourceDir,
 	}
 }
 
-func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) {
-	var newDb db.Db
-	var err error
+func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) {
+	database, ok := ctx.Value("Database").(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to select the database")
+	}
 
 	if existingDb != nil {
 		return existingDb, nil
 	}
 
+	var newDb db.Db
+	var err error
 
-	connStr := ms.conn.String()
-	dbTyp := ms.conn.DbType()
-	if dbTyp == DBTYPE_POSTGRES {
+	if database == "postgres" {
 		newDb = postgres.NewPgDb()
-	} else if dbTyp == DBTYPE_GDBM {
-		err = ms.ensureDbDir()
-		if err != nil {
-			return nil, err
-		}
-		connStr = path.Join(connStr, section)
-		newDb = gdbmstorage.NewThreadGdbmDb()
+		connStr := buildConnStr()
+		err = newDb.Connect(ctx, connStr)
 	} else {
-		return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String())
+		newDb = gdbmstorage.NewThreadGdbmDb()
+		storeFile := path.Join(ms.dbDir, fileName)
+		err = newDb.Connect(ctx, storeFile)
 	}
-	logg.DebugCtxf(ctx, "connecting to db", "conn", connStr)
-	err = newDb.Connect(ctx, connStr)
+
 	if err != nil {
 		return nil, err
 	}
@@ -121,8 +137,8 @@ func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error)
 	return ms.stateStore, nil
 }
 
-func (ms *MenuStorageService) ensureDbDir() error {
-	err := os.MkdirAll(ms.conn.String(), 0700)
+func (ms *MenuStorageService) EnsureDbDir() error {
+	err := os.MkdirAll(ms.dbDir, 0700)
 	if err != nil {
 		return fmt.Errorf("state dir create exited with error: %v\n", err)
 	}

internal/storage/timed.go

@@ -0,0 +1,109 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"time"
+	"encoding/binary"
+
+	"git.defalsify.org/vise.git/db"
+)
+
+type TimedDb struct {
+	db.Db
+	tdb *SubPrefixDb
+	ttl time.Duration
+	parentPfx uint8
+	parentSession []byte
+	matchPfx map[uint8][][]byte
+}
+
+func NewTimedDb(db db.Db, ttl time.Duration) *TimedDb {
+	var b [2]byte
+	binary.BigEndian.PutUint16(b[:], SUBPREFIX_TIME)
+	sdb := NewSubPrefixDb(db, b[:])
+	return &TimedDb{
+		Db: db,
+		tdb: sdb,
+		ttl: ttl,
+	}
+}
+
+func(tib *TimedDb) WithMatch(pfx uint8, keyPart []byte) *TimedDb {
+	if tib.matchPfx == nil {
+		tib.matchPfx = make(map[uint8][][]byte)
+	}
+	tib.matchPfx[pfx] = append(tib.matchPfx[pfx], keyPart)
+	return tib
+}
+
+func(tib *TimedDb) checkPrefix(pfx uint8, key []byte) bool {
+	var v []byte
+	if tib.matchPfx == nil {
+		return true
+	}
+	for _, v = range(tib.matchPfx[pfx]) {
+		l := len(v)
+		if l > len(key) {
+			continue
+		}
+		if bytes.Equal(v, key[:l]) {
+			return true	
+		}
+	}
+	return false
+}
+
+func(tib *TimedDb) SetPrefix(pfx uint8) {
+	tib.Db.SetPrefix(pfx)
+	tib.parentPfx = pfx
+}
+
+func(tib *TimedDb) SetSession(session string) {
+	tib.Db.SetSession(session)
+	tib.parentSession = []byte(session)
+}
+
+func(tib *TimedDb) Put(ctx context.Context, key []byte, val []byte) error {
+	t := time.Now()	
+	b, err := t.MarshalBinary()
+	if err != nil {
+		return err
+	}
+	err = tib.Db.Put(ctx, key, val)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		tib.parentPfx = 0
+		tib.parentSession = nil
+	}()
+	if tib.checkPrefix(tib.parentPfx, key) {
+		tib.tdb.SetSession("")
+		k := db.ToSessionKey(tib.parentPfx, []byte(tib.parentSession), key)
+		k = append([]byte{tib.parentPfx}, k...)
+		err = tib.tdb.Put(ctx, k, b)
+		if err != nil {
+			logg.ErrorCtxf(ctx, "failed to update timestamp of record", err)
+		}
+	}
+	return nil
+}
+
+func(tib *TimedDb) Stale(ctx context.Context, pfx uint8, sessionId string, key []byte) bool {
+	tib.tdb.SetSession("")
+	b := db.ToSessionKey(pfx, []byte(sessionId), key)
+	b = append([]byte{pfx}, b...)
+	v, err := tib.tdb.Get(ctx, b)
+	if err != nil {
+		logg.WarnCtxf(ctx, "no time entry", "key", key, "b", b)
+		return false
+	}
+	t_now := time.Now()	
+	t_then := time.Time{}
+	err = t_then.UnmarshalBinary(v)
+	if err != nil {
+		return false
+	}
+	return t_now.After(t_then.Add(tib.ttl))
+}

internal/storage/timed_test.go

@@ -0,0 +1,125 @@
+package storage
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.defalsify.org/vise.git/db"
+	memdb "git.defalsify.org/vise.git/db/mem"
+)
+
+func TestStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	k := []byte("foo")
+	err = tdb.Put(ctx, k, []byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "", k) {
+		t.Fatal("expected not stale")
+	}
+	time.Sleep(time.Millisecond)
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "", k) {
+		t.Fatal("expected stale")
+	}
+}
+
+func TestFilteredStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k := []byte("foo")
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb = tdb.WithMatch(db.DATATYPE_STATE, []byte("fo"))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("inky")
+	err = tdb.Put(ctx, k, []byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_STATE)
+	tdb.SetSession("inky")
+	err = tdb.Put(ctx, k, []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetSession("blinky")
+	err = tdb.Put(ctx, k, []byte("clyde"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "blinky", k) {
+		t.Fatal("expected not stale")
+	}
+	time.Sleep(time.Millisecond)
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if !tdb.Stale(ctx, db.DATATYPE_STATE, "inky", k) {
+		t.Fatal("expected stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "blinky", k) {
+		t.Fatal("expected not stale")
+	}
+}
+
+func TestFilteredSameKeypartStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb = tdb.WithMatch(db.DATATYPE_USERDATA, []byte("ba"))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("bar"), []byte("inky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("baz"), []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("foo"), []byte("blinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Millisecond)
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("bar")) {
+		t.Fatal("expected stale")
+	}
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("baz")) {
+	t.Fatal("expected stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("foo")) {
+		t.Fatal("expected not stale")
+	}
+}

internal/testutil/TestEngine.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 	"time"
 
 	"git.defalsify.org/vise.git/engine"
@@ -28,6 +27,7 @@ var (
 func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
+	ctx = context.WithValue(ctx, "Database", "gdbm")
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	var eventChannel = make(chan bool)
@@ -39,40 +39,37 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
 		FlagCount:  uint32(128),
 	}
 
-	connStr, err := filepath.Abs(".test_state/state.gdbm")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-	conn, err := storage.ToConnData(connStr)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr parse err: %v", err)
-		os.Exit(1)
-	}
+	dbDir := ".test_state"
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(conn, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
+	err := menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "resource error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	pe, err := menuStorageService.GetPersister(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "persister error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	userDataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "userdb error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	dbResource, ok := rs.(*resource.DbResource)
 	if !ok {
-		fmt.Fprintf(os.Stderr, "dbresource cast error")
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 

internal/testutil/engine_test.go

@@ -1,15 +0,0 @@
-package testutil
-
-import (
-	"testing"
-)
-
-func TestCreateEngine(t *testing.T) {
-	o, clean, eventC := TestEngine("foo")
-	defer clean()
-	defer func() {
-		<-eventC
-		close(eventC)
-	}()
-	_ = o
-}

internal/testutil/mocks/httpmocks/requestparsermock.go

@@ -1,14 +1,12 @@
 package httpmocks
 
-import "context"
-
 // MockRequestParser implements the handlers.RequestParser interface for testing
 type MockRequestParser struct {
 	GetSessionIdFunc func(any) (string, error)
 	GetInputFunc     func(any) ([]byte, error)
 }
 
-func (m *MockRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) {
+func (m *MockRequestParser) GetSessionId(rq any) (string, error) {
 	return m.GetSessionIdFunc(rq)
 }
 

menutraversal_test/menu_traversal_test.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"math/rand"
 	"os"
-	"path/filepath"
 	"regexp"
 	"testing"
 
@@ -18,6 +17,7 @@ import (
 
 var (
 	testData  = driver.ReadData()
+	testStore = ".test_state"
 	sessionID string
 	src       = rand.NewSource(42)
 	g         = rand.New(src)
@@ -25,11 +25,6 @@ var (
 
 var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests")
 
-func testStore() string {
-	v, _ :=  filepath.Abs(".test_state/state.gdbm")
-	return v
-}
-
 func GenerateSessionId() string {
 	uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g))
 	v, err := uu.NewV4()
@@ -86,8 +81,8 @@ func extractSendAmount(response []byte) string {
 func TestMain(m *testing.M) {
 	sessionID = GenerateSessionId()
 	defer func() {
-		if err := os.RemoveAll(testStore()); err != nil {
-			log.Fatalf("Failed to delete state store %s: %v", testStore(), err)
+		if err := os.RemoveAll(testStore); err != nil {
+			log.Fatalf("Failed to delete state store %s: %v", testStore, err)
 		}
 	}()
 	m.Run()