Merge pull request 'trim any leading whitespace in the input' (#258) from send-input-fix into master

Reviewed-on: urdt/ussd#258
Reviewed-by: lash <accounts-grassrootseconomics@holbrook.no>

.dockerignore

@@ -1,5 +1,6 @@
 /**
 !/cmd/africastalking
+!/cmd/ssh
 !/common
 !/config
 !/initializers

.env.example

@@ -6,11 +6,19 @@ HOST=127.0.0.1
 AT_ENDPOINT=/ussd/africastalking
 
 #PostgreSQL
-DB_CONN=postgres://postgres:strongpass@localhost:5432/urdt_ussd
-#DB_TIMEZONE=Africa/Nairobi
-#DB_SCHEMA=vise
+DB_HOST=localhost
+DB_USER=postgres
+DB_PASSWORD=strongpass
+DB_NAME=urdt_ussd
+DB_PORT=5432
+DB_SSLMODE=disable
+DB_TIMEZONE=Africa/Nairobi
 
 #External API Calls
 CUSTODIAL_URL_BASE=http://localhost:5003
 BEARER_TOKEN=eyJeSIsInRcCI6IkpXVCJ.yJwdWJsaWNLZXkiOiIwrrrrrr
 DATA_URL_BASE=http://localhost:5006
+
+#Language
+DEFAULT_LANGUAGE=eng
+LANGUAGES=eng, swa

Dockerfile

@@ -19,6 +19,7 @@ WORKDIR /build
 RUN echo "Building on $BUILDPLATFORM, building for $TARGETPLATFORM"
 RUN go mod download
 RUN go build -tags logtrace -o ussd-africastalking -ldflags="-X main.build=${BUILD} -s -w" cmd/africastalking/main.go
+RUN go build -tags logtrace -o ussd-ssh -ldflags="-X main.build=${BUILD} -s -w" cmd/ssh/main.go
 
 FROM debian:bookworm-slim
 
@@ -30,6 +31,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/*
 WORKDIR /service
 
 COPY --from=build /build/ussd-africastalking .
+COPY --from=build /build/ussd-ssh .
 COPY --from=build /build/LICENSE .
 COPY --from=build /build/README.md .
 COPY --from=build /build/services ./services
@@ -37,5 +39,6 @@ COPY --from=build /build/.env.example .
 RUN mv .env.example .env
 
 EXPOSE 7123
+EXPOSE 7122
 
 CMD ["./ussd-africastalking"]

cmd/africastalking/main.go

@@ -14,13 +14,16 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/http/at"
+	httpserver "git.grassecon.net/urdt/ussd/internal/http/at"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -33,40 +36,40 @@ var (
 func init() {
 	initializers.LoadEnvVariables()
 }
-
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
+	var gettextDir string
+	var langs args.LangVar
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "build", build, "conn", connData, "resourcedir", resourceDir, "outputsize", size)
+	logg.Infof("start command", "build", build, "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
@@ -80,13 +83,14 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	rs, err := menuStorageService.GetResource(ctx)
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -132,7 +136,7 @@ func main() {
 
 	rp := &at.ATRequestParser{}
 	bsh := handlers.NewBaseSessionHandler(cfg, rs, stateStore, userdataStore, rp, hl)
-	sh := at.NewATSessionHandler(bsh)
+	sh := httpserver.NewATSessionHandler(bsh)
 
 	mux := http.NewServeMux()
 	mux.Handle(initializers.GetEnv("AT_ENDPOINT", "/"), sh)

cmd/async/main.go

@@ -12,12 +12,14 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -46,38 +48,40 @@ func (p *asyncRequestParser) GetInput(r any) ([]byte, error) {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
 	var sessionId string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
+	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
@@ -91,18 +95,18 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, err.Error())
-		os.Exit(1)
-	}
-
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
+	err = menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {

cmd/http/main.go

@@ -14,6 +14,7 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
@@ -21,6 +22,7 @@ import (
 	httpserver "git.grassecon.net/urdt/ussd/internal/http"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -36,36 +38,38 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var resourceDir string
 	var size uint
 	var database string
 	var engineDebug bool
 	var host string
 	var port uint
-	var err error
-
+	var gettextDir string
+	var langs args.LangVar
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size)
+	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
@@ -79,14 +83,19 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
+	err = menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
 	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())

cmd/main.go

@@ -10,10 +10,12 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/internal/args"
 	"git.grassecon.net/urdt/ussd/remote"
 )
 
@@ -27,38 +29,43 @@ func init() {
 	initializers.LoadEnvVariables()
 }
 
+// TODO: external script automatically generate language handler list from select language vise code OR consider dynamic menu generation script possibility
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var size uint
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var resourceDir string
-	var err error
-
-	flag.StringVar(&resourceDir, "resourcedir", scriptDir, "resource dir")
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
+	logg.Infof("start command", "dbdir", dbDir, "outputsize", size)
 
-	logg.Infof("start command", "conn", connData, "outputsize", size)
+	if len(langs.Langs()) == 0 {
+		langs.Set(config.DefaultLanguage)
+	}
 
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
@@ -69,7 +76,17 @@ func main() {
 		MenuSeparator: menuSeparator,
 	}
 
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	resourceDir := scriptDir
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+	if gettextDir != "" {
+		menuStorageService = menuStorageService.WithGettext(gettextDir, langs.Langs())
+	}
+
+	err = menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {

cmd/ssh/main.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"path"
 	"os"
 	"os/signal"
+	"path"
 	"sync"
 	"syscall"
 
@@ -14,27 +14,19 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 
-	"git.grassecon.net/urdt/ussd/config"
-	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/ssh"
-	"git.grassecon.net/urdt/ussd/internal/storage"
 )
 
 var (
-	wg sync.WaitGroup
-	keyStore db.Db
+	wg        sync.WaitGroup
+	keyStore  db.Db
 	logg      = logging.NewVanilla()
 	scriptDir = path.Join("services", "registration")
+
+	build = "dev"
 )
 
-func init() {
-	initializers.LoadEnvVariables()
-}
-
 func main() {
-	config.LoadConfig()
-
-	var connStr string
 	var dbDir string
 	var resourceDir string
 	var size uint
@@ -42,25 +34,17 @@ func main() {
 	var stateDebug bool
 	var host string
 	var port uint
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
-	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
+	flag.BoolVar(&engineDebug, "engine-debug", false, "use engine debug output")
+	flag.BoolVar(&stateDebug, "state-debug", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
-	flag.StringVar(&host, "h", "127.0.0.1", "socket host")
-	flag.UintVar(&port, "p", 7122, "socket port")
+	flag.StringVar(&host, "h", "127.0.0.1", "http host")
+	flag.UintVar(&port, "p", 7122, "http port")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
 	sshKeyFile := flag.Arg(0)
-	_, err = os.Stat(sshKeyFile)
+	_, err := os.Stat(sshKeyFile)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "cannot open ssh server private key file: %v\n", err)
 		os.Exit(1)
@@ -94,7 +78,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "keystore file open error: %v", err)
 		os.Exit(1)
 	}
-	defer func () {
+	defer func() {
 		logg.TraceCtxf(ctx, "shutdown auth key store reached")
 		err = authKeyStore.Close()
 		if err != nil {
@@ -108,14 +92,14 @@ func main() {
 	signal.Notify(cterm, os.Interrupt, syscall.SIGTERM)
 
 	runner := &ssh.SshRunner{
-		Cfg: cfg,
-		Debug: engineDebug,
-		FlagFile: pfp,
-		Conn: connData,
+		Cfg:         cfg,
+		Debug:       engineDebug,
+		FlagFile:    pfp,
+		DbDir:       dbDir,
 		ResourceDir: resourceDir,
-		SrvKeyFile: sshKeyFile,
-		Host: host,
-		Port: port,
+		SrvKeyFile:  sshKeyFile,
+		Host:        host,
+		Port:        port,
 	}
 	go func() {
 		select {
@@ -127,7 +111,7 @@ func main() {
 		if err != nil {
 			logg.ErrorCtxf(ctx, "runner stop error", "err", err)
 		}
-		
+
 	}()
 	runner.Run(ctx, authKeyStore)
 }

common/storage.go

@@ -23,17 +23,17 @@ type StorageServices interface {
 	GetPersister(ctx context.Context) (*persist.Persister, error)
 	GetUserdataDb(ctx context.Context) (db.Db, error)
 	GetResource(ctx context.Context) (resource.Resource, error)
+	EnsureDbDir() error
 }
 
 type StorageService struct {
 	svc *storage.MenuStorageService
 }
 
-func NewStorageService(conn storage.ConnData) (*StorageService, error) {
-	svc := &StorageService{
-		svc: storage.NewMenuStorageService(conn, ""),
+func NewStorageService(dbDir string) *StorageService {
+	return &StorageService{
+		svc: storage.NewMenuStorageService(dbDir, ""),
 	}
-	return svc, nil
 }
 
 func(ss *StorageService) GetPersister(ctx context.Context) (*persist.Persister, error) {
@@ -47,3 +47,7 @@ func(ss *StorageService) GetUserdataDb(ctx context.Context) (db.Db, error) {
 func(ss *StorageService) GetResource(ctx context.Context) (resource.Resource, error) {
 	return nil, errors.New("not implemented")
 }
+
+func(ss *StorageService) EnsureDbDir() error {
+	return ss.svc.EnsureDbDir()
+}

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"net/url"
+	"strings"
 
 	"git.grassecon.net/urdt/ussd/initializers"
 )
@@ -18,6 +19,11 @@ const (
 	AliasPrefix                = "api/v1/alias"
 )
 
+var (
+	defaultLanguage		   = "eng"
+	languages []string
+)
+
 var (
 	custodialURLBase string
 	dataURLBase      string
@@ -34,9 +40,28 @@ var (
 	VoucherTransfersURL string
 	VoucherDataURL      string
 	CheckAliasURL       string
-	DbConn		string
+	DefaultLanguage	    string
+	Languages	[]string
 )
 
+func setLanguage() error {
+	defaultLanguage = initializers.GetEnv("DEFAULT_LANGUAGE", defaultLanguage)
+	languages = strings.Split(initializers.GetEnv("LANGUAGES", defaultLanguage), ",")
+	haveDefaultLanguage := false
+	for i, v := range(languages) {
+		languages[i] = strings.ReplaceAll(v, " ", "")
+		if languages[i] == defaultLanguage {
+			haveDefaultLanguage = true
+		}
+	}
+
+	if !haveDefaultLanguage {
+		languages = append([]string{defaultLanguage}, languages...)
+	}
+
+	return nil
+}
+
 func setBase() error {
 	var err error
 
@@ -44,20 +69,14 @@ func setBase() error {
 	dataURLBase = initializers.GetEnv("DATA_URL_BASE", "http://localhost:5006")
 	BearerToken = initializers.GetEnv("BEARER_TOKEN", "")
 
-	_, err = url.Parse(custodialURLBase)
+	_, err = url.JoinPath(custodialURLBase, "/foo")
 	if err != nil {
 		return err
 	}
-	_, err = url.Parse(dataURLBase)
+	_, err = url.JoinPath(dataURLBase, "/bar")
 	if err != nil {
 		return err
 	}
-
-	return nil
-}
-
-func setConn() error {
-	DbConn = initializers.GetEnv("DB_CONN", "")
 	return nil
 }
 
@@ -67,7 +86,7 @@ func LoadConfig() error {
 	if err != nil {
 		return err
 	}
-	err = setConn()
+	err = setLanguage()
 	if err != nil {
 		return err
 	}
@@ -80,6 +99,8 @@ func LoadConfig() error {
 	VoucherTransfersURL, _ = url.JoinPath(dataURLBase, voucherTransfersPathPrefix)
 	VoucherDataURL, _ = url.JoinPath(dataURLBase, voucherDataPathPrefix)
 	CheckAliasURL, _ = url.JoinPath(dataURLBase, AliasPrefix)
+	DefaultLanguage = defaultLanguage
+	Languages = languages
 
 	return nil
 }

devtools/lang/main.go

@@ -0,0 +1,126 @@
+// create language files from environment
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"git.defalsify.org/vise.git/logging"
+	"git.defalsify.org/vise.git/lang"
+	"git.grassecon.net/urdt/ussd/config"
+	"git.grassecon.net/urdt/ussd/initializers"
+)
+
+const (
+	
+	changeHeadSrc = `LOAD reset_account_authorized 0
+LOAD reset_incorrect 0
+CATCH incorrect_pin flag_incorrect_pin 1
+CATCH pin_entry flag_account_authorized 0
+` 
+
+	selectSrc = `LOAD set_language 6
+RELOAD set_language
+CATCH terms flag_account_created 0
+MOVE language_changed
+`
+)
+
+var (
+	logg = logging.NewVanilla()
+	mouts string
+	incmps string
+)
+
+func init() {
+	initializers.LoadEnvVariables()
+}
+
+func toLanguageLabel(ln lang.Language) string {
+	s := ln.Name
+	v := strings.Split(s, " (")
+	if len(v) > 1 {
+		s = v[0]
+	}
+	return s
+}
+
+func toLanguageKey(ln lang.Language) string {
+	s := toLanguageLabel(ln)
+	return strings.ToLower(s)
+}
+
+func main() {
+	var srcDir string
+
+	flag.StringVar(&srcDir, "o", ".", "resource dir write to")
+	flag.Parse()
+
+	logg.Infof("start command", "dir", srcDir)
+
+	err := config.LoadConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "config load error: %v", err)
+		os.Exit(1)
+	}
+	logg.Tracef("using languages", "lang", config.Languages)
+
+	for i, v := range(config.Languages) {
+		ln, err := lang.LanguageFromCode(v)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error parsing language: %s\n", v)
+			os.Exit(1)
+		}
+		n := i + 1
+		s := toLanguageKey(ln)
+		mouts += fmt.Sprintf("MOUT %s %v\n", s, n)
+		v = "set_" + ln.Code
+		incmps += fmt.Sprintf("INCMP %s %v\n", v, n)
+
+		p := path.Join(srcDir, v)
+		w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed open language set template output: %v\n", err)
+			os.Exit(1)
+		}
+		s = toLanguageLabel(ln)
+		defer w.Close()
+		_, err = w.Write([]byte(s))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+			os.Exit(1)
+		}
+	}
+	src := mouts + "HALT\n" + incmps
+	src += "INCMP . *\n"
+
+	p := path.Join(srcDir, "select_language.vis")
+	w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+	defer w.Close()
+	_, err = w.Write([]byte(src))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+
+	src = changeHeadSrc + src
+	p = path.Join(srcDir, "change_language.vis")
+	w, err = os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+	defer w.Close()
+	_, err = w.Write([]byte(src))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+}

devtools/store/dump/main.go

@@ -25,37 +25,35 @@ func init() {
 }
 
 
+func formatItem(k []byte, v []byte) (string, error) {
+	o, err := debug.FromKey(k)
+	if err != nil {
+		return "", err
+	}
+	s := fmt.Sprintf("%vValue: %v\n\n", o, string(v))
+	return s, nil
+}
+
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var err error
 
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", ".state", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData)
-
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 
 	store, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
@@ -75,12 +73,12 @@ func main() {
 		if k == nil {
 			break
 		}
-		o, err := debug.FromKey(k)
+		r, err := formatItem(k, v)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
+			fmt.Fprintf(os.Stderr, "format db item error: %v", err)
 			os.Exit(1)
 		}
-		fmt.Printf("%vValue: %v\n\n", o, string(v))
+		fmt.Printf(r)
 	}
 
 	err = store.Close()

devtools/store/generate/main.go

@@ -28,35 +28,24 @@ func init() {
 func main() {
 	config.LoadConfig()
 
-	var connStr string
+	var dbDir string
 	var sessionId string
 	var database string
 	var engineDebug bool
-	var err error
 
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
-	flag.StringVar(&connStr, "c", "", "connection string")
+	flag.StringVar(&database, "db", "gdbm", "database to be used")
+	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.Parse()
 
-	if connStr != "" {
-		connStr = config.DbConn
-	}
-	connData, err := storage.ToConnData(config.DbConn)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-
-	logg.Infof("start command", "conn", connData)
-
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(connData, resourceDir)
-	
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
 	store, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())

internal/args/lang.go

@@ -0,0 +1,34 @@
+package args
+
+import (
+	"strings"
+
+	"git.defalsify.org/vise.git/lang"
+)
+
+type LangVar struct {
+	v []lang.Language
+}
+
+func(lv *LangVar) Set(s string) error {
+	v, err := lang.LanguageFromCode(s)
+	if err != nil {
+		return err
+	}
+	lv.v = append(lv.v, v)
+	return err
+}
+
+func(lv *LangVar) String() string {
+	var s []string
+	for _, v := range(lv.v) {
+		s = append(s, v.Code)
+	}
+	return strings.Join(s, ",")
+}
+
+func(lv *LangVar) Langs() []lang.Language {
+	return lv.v
+}
+
+

internal/http/at/parse.go

@@ -81,7 +81,8 @@ func (arp *ATRequestParser) GetInput(rq any) ([]byte, error) {
 		return nil, fmt.Errorf("no input found")
 	}
 
-	return []byte(parts[len(parts)-1]), nil
+	trimmedInput := strings.TrimSpace(parts[len(parts)-1])
+	return []byte(trimmedInput), nil
 }
 
 func parseQueryParams(query string) map[string]string {

internal/ssh/ssh.go

@@ -71,20 +71,6 @@ func(a *auther) Get(k []byte) (string, error) {
 	return v, nil
 }
 
-type SshRunner struct {
-	Ctx context.Context
-	Cfg engine.Config
-	FlagFile string
-	Conn storage.ConnData
-	ResourceDir string
-	Debug bool
-	SrvKeyFile string
-	Host string
-	Port uint
-	wg sync.WaitGroup
-	lst net.Listener
-}
-
 func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChannel, en engine.Engine) error {
 	if ch == nil {
 		return errors.New("nil channel")
@@ -142,13 +128,32 @@ func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChanne
 	return nil
 }
 
+type SshRunner struct {
+	Ctx context.Context
+	Cfg engine.Config
+	FlagFile string
+	DbDir string
+	ResourceDir string
+	Debug bool
+	SrvKeyFile string
+	Host string
+	Port uint
+	wg sync.WaitGroup
+	lst net.Listener
+}
+
 func(s *SshRunner) Stop() error {
 	return s.lst.Close()
 }
 
 func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) {
 	ctx := s.Ctx
-	menuStorageService := storage.NewMenuStorageService(s.Conn, s.ResourceDir)
+	menuStorageService := storage.NewMenuStorageService(s.DbDir, s.ResourceDir)
+
+	err := menuStorageService.EnsureDbDir()
+	if err != nil {
+		return nil, nil, err
+	}
 
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {

internal/storage/parse.go

@@ -1,69 +0,0 @@
-package storage
-
-import (
-	"fmt"
-	"net/url"
-	"path"
-)
-
-const (
-	DBTYPE_MEM = iota
-	DBTYPE_GDBM
-	DBTYPE_POSTGRES
-)
-
-type ConnData struct {
-	typ int
-	str string
-}
-
-func (cd *ConnData) DbType() int {
-	return cd.typ
-}
-
-func (cd *ConnData) String() string {
-	return cd.str
-}
-
-func probePostgres(s string) (string, bool) {
-	v, err := url.Parse(s)
-	if err != nil {
-		return "", false
-	}
-	if v.Scheme != "postgres" {
-		return "", false
-	}
-	return s, true
-}
-
-func probeGdbm(s string) (string, bool) {
-	if !path.IsAbs(s) {
-		return "", false
-	}
-	s = path.Clean(s)
-	return s, true
-}
-
-func ToConnData(connStr string) (ConnData, error) {
-	var o ConnData
-
-	if connStr == "" {
-		return o, nil
-	}
-
-	v, ok := probePostgres(connStr)
-	if ok {
-		o.typ = DBTYPE_POSTGRES
-		o.str = v
-		return o, nil
-	}
-
-	v, ok = probeGdbm(connStr)
-	if ok {
-		o.typ = DBTYPE_GDBM
-		o.str = v
-		return o, nil
-	}
-
-	return o, fmt.Errorf("invalid connection string: %s", connStr)
-}

internal/storage/parse_test.go

@@ -1,28 +0,0 @@
-package storage
-
-import (
-	"testing"
-)
-
-func TestParseConnStr(t *testing.T) {
-	_, err := ToConnData("postgres://foo:bar@localhost:5432/baz")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("/foo/bar")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("/foo/bar/")
-	if err != nil {
-		t.Fatal(err)	
-	}
-	_, err = ToConnData("foo/bar")
-	if err == nil {
-		t.Fatalf("expected error")
-	}
-	_, err = ToConnData("http://foo/bar")
-	if err == nil {
-		t.Fatalf("expected error")
-	}
-}

internal/storage/storageservice.go

@@ -9,9 +9,11 @@ import (
 	"git.defalsify.org/vise.git/db"
 	fsdb "git.defalsify.org/vise.git/db/fs"
 	"git.defalsify.org/vise.git/db/postgres"
+	"git.defalsify.org/vise.git/lang"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/persist"
 	"git.defalsify.org/vise.git/resource"
+	"git.grassecon.net/urdt/ussd/initializers"
 	gdbmstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm"
 )
 
@@ -23,48 +25,86 @@ type StorageService interface {
 	GetPersister(ctx context.Context) (*persist.Persister, error)
 	GetUserdataDb(ctx context.Context) db.Db
 	GetResource(ctx context.Context) (resource.Resource, error)
+	EnsureDbDir() error
 }
 
 type MenuStorageService struct {
-	conn ConnData
+	dbDir         string
 	resourceDir   string
+	poResource    resource.Resource
 	resourceStore db.Db
 	stateStore    db.Db
 	userDataStore db.Db
 }
 
-func NewMenuStorageService(conn ConnData, resourceDir string) *MenuStorageService {
+func buildConnStr() string {
+	host := initializers.GetEnv("DB_HOST", "localhost")
+	user := initializers.GetEnv("DB_USER", "postgres")
+	password := initializers.GetEnv("DB_PASSWORD", "")
+	dbName := initializers.GetEnv("DB_NAME", "")
+	port := initializers.GetEnv("DB_PORT", "5432")
+
+	connString := fmt.Sprintf(
+		"postgres://%s:%s@%s:%s/%s",
+		user, password, host, port, dbName,
+	)
+	logg.Debugf("pg conn string", "conn", connString)
+
+	return connString
+}
+
+func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService {
 	return &MenuStorageService{
-		conn: conn,
+		dbDir:       dbDir,
 		resourceDir: resourceDir,
 	}
 }
 
-func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) {
-	var newDb db.Db
-	var err error
+// WithGettext triggers use of gettext for translation of templates and menus.
+//
+// The first language in `lns` will be used as default language, to resolve node keys to 
+// language strings.
+//
+// If `lns` is an empty array, gettext will not be used.
+func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *MenuStorageService {
+	if len(lns) == 0 {
+		logg.Warnf("Gettext requested but no languages supplied")
+		return ms
+	}
+	rs := resource.NewPoResource(lns[0], path)
+
+	for _, ln := range(lns) {
+		rs = rs.WithLanguage(ln)
+	}
+
+	ms.poResource = rs
+
+	return ms
+}
+
+func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) {
+	database, ok := ctx.Value("Database").(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to select the database")
+	}
 
 	if existingDb != nil {
 		return existingDb, nil
 	}
 
+	var newDb db.Db
+	var err error
 
-	connStr := ms.conn.String()
-	dbTyp := ms.conn.DbType()
-	if dbTyp == DBTYPE_POSTGRES {
+	if database == "postgres" {
 		newDb = postgres.NewPgDb()
-	} else if dbTyp == DBTYPE_GDBM {
-		err = ms.ensureDbDir()
-		if err != nil {
-			return nil, err
-		}
-		connStr = path.Join(connStr, section)
-		newDb = gdbmstorage.NewThreadGdbmDb()
+		connStr := buildConnStr()
+		err = newDb.Connect(ctx, connStr)
 	} else {
-		return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String())
+		newDb = gdbmstorage.NewThreadGdbmDb()
+		storeFile := path.Join(ms.dbDir, fileName)
+		err = newDb.Connect(ctx, storeFile)
 	}
-	logg.DebugCtxf(ctx, "connecting to db", "conn", connStr)
-	err = newDb.Connect(ctx, connStr)
+
 	if err != nil {
 		return nil, err
 	}
@@ -104,6 +144,11 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc
 		return nil, err
 	}
 	rfs := resource.NewDbResource(ms.resourceStore)
+	if ms.poResource != nil {
+		logg.InfoCtxf(ctx, "using poresource for menu and template")
+		rfs.WithMenuGetter(ms.poResource.GetMenu)
+		rfs.WithTemplateGetter(ms.poResource.GetTemplate)
+	}
 	return rfs, nil
 }
 
@@ -121,8 +166,8 @@ func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error)
 	return ms.stateStore, nil
 }
 
-func (ms *MenuStorageService) ensureDbDir() error {
-	err := os.MkdirAll(ms.conn.String(), 0700)
+func (ms *MenuStorageService) EnsureDbDir() error {
+	err := os.MkdirAll(ms.dbDir, 0700)
 	if err != nil {
 		return fmt.Errorf("state dir create exited with error: %v\n", err)
 	}

internal/testutil/TestEngine.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 	"time"
 
 	"git.defalsify.org/vise.git/engine"
@@ -28,6 +27,7 @@ var (
 func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
+	ctx = context.WithValue(ctx, "Database", "gdbm")
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	var eventChannel = make(chan bool)
@@ -39,40 +39,37 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
 		FlagCount:  uint32(128),
 	}
 
-	connStr, err := filepath.Abs(".test_state/state.gdbm")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr err: %v", err)
-		os.Exit(1)
-	}
-	conn, err := storage.ToConnData(connStr)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "connstr parse err: %v", err)
-		os.Exit(1)
-	}
+	dbDir := ".test_state"
 	resourceDir := scriptDir
-	menuStorageService := storage.NewMenuStorageService(conn, resourceDir)
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+
+	err := menuStorageService.EnsureDbDir()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, err.Error())
+		os.Exit(1)
+	}
 
 	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "resource error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	pe, err := menuStorageService.GetPersister(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "persister error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	userDataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "userdb error: %v", err)
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
 	dbResource, ok := rs.(*resource.DbResource)
 	if !ok {
-		fmt.Fprintf(os.Stderr, "dbresource cast error")
+		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 

internal/testutil/engine_test.go

@@ -1,15 +0,0 @@
-package testutil
-
-import (
-	"testing"
-)
-
-func TestCreateEngine(t *testing.T) {
-	o, clean, eventC := TestEngine("foo")
-	defer clean()
-	defer func() {
-		<-eventC
-		close(eventC)
-	}()
-	_ = o
-}

menutraversal_test/menu_traversal_test.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"math/rand"
 	"os"
-	"path/filepath"
 	"regexp"
 	"testing"
 
@@ -18,6 +17,7 @@ import (
 
 var (
 	testData  = driver.ReadData()
+	testStore = ".test_state"
 	sessionID string
 	src       = rand.NewSource(42)
 	g         = rand.New(src)
@@ -25,11 +25,6 @@ var (
 
 var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests")
 
-func testStore() string {
-	v, _ :=  filepath.Abs(".test_state/state.gdbm")
-	return v
-}
-
 func GenerateSessionId() string {
 	uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g))
 	v, err := uu.NewV4()
@@ -86,8 +81,8 @@ func extractSendAmount(response []byte) string {
 func TestMain(m *testing.M) {
 	sessionID = GenerateSessionId()
 	defer func() {
-		if err := os.RemoveAll(testStore()); err != nil {
-			log.Fatalf("Failed to delete state store %s: %v", testStore(), err)
+		if err := os.RemoveAll(testStore); err != nil {
+			log.Fatalf("Failed to delete state store %s: %v", testStore, err)
 		}
 	}()
 	m.Run()