Merge branch 'master' into postgres-switch-for-tests

.dockerignore

@@ -1,5 +1,6 @@
 /**
 !/cmd/africastalking
+!/cmd/ssh
 !/common
 !/config
 !/initializers

.env.example

@@ -18,3 +18,7 @@ DB_TIMEZONE=Africa/Nairobi
 CUSTODIAL_URL_BASE=http://localhost:5003
 BEARER_TOKEN=eyJeSIsInRcCI6IkpXVCJ.yJwdWJsaWNLZXkiOiIwrrrrrr
 DATA_URL_BASE=http://localhost:5006
+
+#Language
+DEFAULT_LANGUAGE=eng
+LANGUAGES=eng, swa

Dockerfile

@@ -39,5 +39,6 @@ COPY --from=build /build/.env.example .
 RUN mv .env.example .env
 
 EXPOSE 7123
+EXPOSE 7122
 
 CMD ["./ussd-africastalking"]

cmd/africastalking/main.go

@@ -14,6 +14,7 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 	testdataloader "github.com/peteole/testdata-loader"
 
 	"git.grassecon.net/urdt/ussd/config"
@@ -23,6 +24,7 @@ import (
 	httpserver "git.grassecon.net/urdt/ussd/internal/http/at"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -47,6 +49,8 @@ func main() {
 	var engineDebug bool
 	var host string
 	var port uint
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
 	flag.StringVar(&database, "db", "gdbm", "database to be used")
@@ -55,6 +59,8 @@ func main() {
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
 	logg.Infof("start command", "build", build, "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
@@ -62,6 +68,13 @@ func main() {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
 	ctx = context.WithValue(ctx, "Schema", dbSchema)
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{

cmd/async/main.go

@@ -12,6 +12,7 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 	testdataloader "github.com/peteole/testdata-loader"
 
 	"git.grassecon.net/urdt/ussd/config"
@@ -19,6 +20,7 @@ import (
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -57,6 +59,8 @@ func main() {
 	var engineDebug bool
 	var host string
 	var port uint
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
 	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
@@ -66,6 +70,8 @@ func main() {
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
 	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
@@ -73,6 +79,14 @@ func main() {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
 	ctx = context.WithValue(ctx, "Schema", dbSchema)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{

cmd/http/main.go

@@ -14,6 +14,7 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 	testdataloader "github.com/peteole/testdata-loader"
 
 	"git.grassecon.net/urdt/ussd/config"
@@ -22,6 +23,7 @@ import (
 	httpserver "git.grassecon.net/urdt/ussd/internal/http"
 	"git.grassecon.net/urdt/ussd/internal/storage"
 	"git.grassecon.net/urdt/ussd/remote"
+	"git.grassecon.net/urdt/ussd/internal/args"
 )
 
 var (
@@ -46,6 +48,8 @@ func main() {
 	var engineDebug bool
 	var host string
 	var port uint
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir")
 	flag.StringVar(&database, "db", "gdbm", "database to be used")
@@ -54,6 +58,8 @@ func main() {
 	flag.UintVar(&size, "s", 160, "max size of output")
 	flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host")
 	flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
 	logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size)
@@ -61,6 +67,14 @@ func main() {
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "Database", database)
 	ctx = context.WithValue(ctx, "Schema", dbSchema)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{

cmd/main.go

@@ -10,10 +10,12 @@ import (
 	"git.defalsify.org/vise.git/engine"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/lang"
 	"git.grassecon.net/urdt/ussd/config"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/handlers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
+	"git.grassecon.net/urdt/ussd/internal/args"
 	"git.grassecon.net/urdt/ussd/remote"
 	testdataloader "github.com/peteole/testdata-loader"
 )
@@ -29,6 +31,7 @@ func init() {
 	initializers.LoadEnvVariables(baseDir)
 }
 
+// TODO: external script automatically generate language handler list from select language vise code OR consider dynamic menu generation script possibility
 func main() {
 	config.LoadConfig()
 
@@ -38,20 +41,36 @@ func main() {
 	var database string
 	var dbSchema string
 	var engineDebug bool
+	var gettextDir string
+	var langs args.LangVar
 	flag.StringVar(&sessionId, "session-id", "075xx2123", "session id")
 	flag.StringVar(&database, "db", "gdbm", "database to be used")
 	flag.StringVar(&dbSchema, "schema", "public", "database schema to be used")
 	flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from")
 	flag.BoolVar(&engineDebug, "d", false, "use engine debug output")
 	flag.UintVar(&size, "s", 160, "max size of output")
+	flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory")
+	flag.Var(&langs, "language", "add symbol resolution for language")
 	flag.Parse()
 
 	logg.Infof("start command", "dbdir", dbDir, "outputsize", size)
 
+	if len(langs.Langs()) == 0 {
+		langs.Set(config.DefaultLanguage)
+	}
+
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, "SessionId", sessionId)
 	ctx = context.WithValue(ctx, "Database", database)
 	ctx = context.WithValue(ctx, "Schema", dbSchema)
+
+	ln, err := lang.LanguageFromCode(config.DefaultLanguage)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "default language set error: %v", err)
+		os.Exit(1)
+	}
+	ctx = context.WithValue(ctx, "Language", ln)
+
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
@@ -64,8 +83,11 @@ func main() {
 
 	resourceDir := scriptDir
 	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
+	if gettextDir != "" {
+		menuStorageService = menuStorageService.WithGettext(gettextDir, langs.Langs())
+	}
 
-	err := menuStorageService.EnsureDbDir()
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)

cmd/ssh/main.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"path"
 	"os"
 	"os/signal"
+	"path"
 	"sync"
 	"syscall"
 
@@ -22,6 +22,8 @@ var (
 	keyStore  db.Db
 	logg      = logging.NewVanilla()
 	scriptDir = path.Join("services", "registration")
+
+	build = "dev"
 )
 
 func main() {
@@ -76,7 +78,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "keystore file open error: %v", err)
 		os.Exit(1)
 	}
-	defer func () {
+	defer func() {
 		logg.TraceCtxf(ctx, "shutdown auth key store reached")
 		err = authKeyStore.Close()
 		if err != nil {

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"net/url"
+	"strings"
 
 	"git.grassecon.net/urdt/ussd/initializers"
 )
@@ -18,6 +19,11 @@ const (
 	AliasPrefix                = "api/v1/alias"
 )
 
+var (
+	defaultLanguage		   = "eng"
+	languages []string
+)
+
 var (
 	custodialURLBase string
 	dataURLBase      string
@@ -34,8 +40,28 @@ var (
 	VoucherTransfersURL string
 	VoucherDataURL      string
 	CheckAliasURL       string
+	DefaultLanguage	    string
+	Languages	[]string
 )
 
+func setLanguage() error {
+	defaultLanguage = initializers.GetEnv("DEFAULT_LANGUAGE", defaultLanguage)
+	languages = strings.Split(initializers.GetEnv("LANGUAGES", defaultLanguage), ",")
+	haveDefaultLanguage := false
+	for i, v := range(languages) {
+		languages[i] = strings.ReplaceAll(v, " ", "")
+		if languages[i] == defaultLanguage {
+			haveDefaultLanguage = true
+		}
+	}
+
+	if !haveDefaultLanguage {
+		languages = append([]string{defaultLanguage}, languages...)
+	}
+
+	return nil
+}
+
 func setBase() error {
 	var err error
 
@@ -60,6 +86,10 @@ func LoadConfig() error {
 	if err != nil {
 		return err
 	}
+	err = setLanguage()
+	if err != nil {
+		return err
+	}
 	CreateAccountURL, _ = url.JoinPath(custodialURLBase, createAccountPath)
 	TrackStatusURL, _ = url.JoinPath(custodialURLBase, trackStatusPath)
 	BalanceURL, _ = url.JoinPath(custodialURLBase, balancePathPrefix)
@@ -69,6 +99,8 @@ func LoadConfig() error {
 	VoucherTransfersURL, _ = url.JoinPath(dataURLBase, voucherTransfersPathPrefix)
 	VoucherDataURL, _ = url.JoinPath(dataURLBase, voucherDataPathPrefix)
 	CheckAliasURL, _ = url.JoinPath(dataURLBase, AliasPrefix)
+	DefaultLanguage = defaultLanguage
+	Languages = languages
 
 	return nil
 }

devtools/lang/main.go

@@ -0,0 +1,126 @@
+// create language files from environment
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	"git.defalsify.org/vise.git/logging"
+	"git.defalsify.org/vise.git/lang"
+	"git.grassecon.net/urdt/ussd/config"
+	"git.grassecon.net/urdt/ussd/initializers"
+)
+
+const (
+	
+	changeHeadSrc = `LOAD reset_account_authorized 0
+LOAD reset_incorrect 0
+CATCH incorrect_pin flag_incorrect_pin 1
+CATCH pin_entry flag_account_authorized 0
+` 
+
+	selectSrc = `LOAD set_language 6
+RELOAD set_language
+CATCH terms flag_account_created 0
+MOVE language_changed
+`
+)
+
+var (
+	logg = logging.NewVanilla()
+	mouts string
+	incmps string
+)
+
+func init() {
+	initializers.LoadEnvVariables()
+}
+
+func toLanguageLabel(ln lang.Language) string {
+	s := ln.Name
+	v := strings.Split(s, " (")
+	if len(v) > 1 {
+		s = v[0]
+	}
+	return s
+}
+
+func toLanguageKey(ln lang.Language) string {
+	s := toLanguageLabel(ln)
+	return strings.ToLower(s)
+}
+
+func main() {
+	var srcDir string
+
+	flag.StringVar(&srcDir, "o", ".", "resource dir write to")
+	flag.Parse()
+
+	logg.Infof("start command", "dir", srcDir)
+
+	err := config.LoadConfig()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "config load error: %v", err)
+		os.Exit(1)
+	}
+	logg.Tracef("using languages", "lang", config.Languages)
+
+	for i, v := range(config.Languages) {
+		ln, err := lang.LanguageFromCode(v)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error parsing language: %s\n", v)
+			os.Exit(1)
+		}
+		n := i + 1
+		s := toLanguageKey(ln)
+		mouts += fmt.Sprintf("MOUT %s %v\n", s, n)
+		v = "set_" + ln.Code
+		incmps += fmt.Sprintf("INCMP %s %v\n", v, n)
+
+		p := path.Join(srcDir, v)
+		w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed open language set template output: %v\n", err)
+			os.Exit(1)
+		}
+		s = toLanguageLabel(ln)
+		defer w.Close()
+		_, err = w.Write([]byte(s))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+			os.Exit(1)
+		}
+	}
+	src := mouts + "HALT\n" + incmps
+	src += "INCMP . *\n"
+
+	p := path.Join(srcDir, "select_language.vis")
+	w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+	defer w.Close()
+	_, err = w.Write([]byte(src))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+
+	src = changeHeadSrc + src
+	p = path.Join(srcDir, "change_language.vis")
+	w, err = os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+	defer w.Close()
+	_, err = w.Write([]byte(src))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err)
+		os.Exit(1)
+	}
+}

devtools/store/dump/main.go

@@ -7,23 +7,31 @@ import (
 	"os"
 	"path"
 
-	"git.defalsify.org/vise.git/db"
-	"git.defalsify.org/vise.git/logging"
 	"git.grassecon.net/urdt/ussd/config"
-	"git.grassecon.net/urdt/ussd/debug"
 	"git.grassecon.net/urdt/ussd/initializers"
 	"git.grassecon.net/urdt/ussd/internal/storage"
-	testdataloader "github.com/peteole/testdata-loader"
+	"git.grassecon.net/urdt/ussd/debug"
+	"git.defalsify.org/vise.git/db"
+	"git.defalsify.org/vise.git/logging"
 )
 
 var (
 	logg      = logging.NewVanilla()
-	baseDir   = testdataloader.GetBasePath()
 	scriptDir = path.Join("services", "registration")
 )
 
 func init() {
-	initializers.LoadEnvVariables(baseDir)
+	initializers.LoadEnvVariables()
+}
+
+
+func formatItem(k []byte, v []byte) (string, error) {
+	o, err := debug.FromKey(k)
+	if err != nil {
+		return "", err
+	}
+	s := fmt.Sprintf("%vValue: %v\n\n", o, string(v))
+	return s, nil
 }
 
 func main() {
@@ -65,12 +73,12 @@ func main() {
 		if k == nil {
 			break
 		}
-		o, err := debug.FromKey(k)
+		r, err := formatItem(k, v)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, err.Error())
+			fmt.Fprintf(os.Stderr, "format db item error: %v", err)
 			os.Exit(1)
 		}
-		fmt.Printf("%vValue: %v\n\n", o, string(v))
+		fmt.Printf(r)
 	}
 
 	err = store.Close()

internal/args/lang.go

@@ -0,0 +1,34 @@
+package args
+
+import (
+	"strings"
+
+	"git.defalsify.org/vise.git/lang"
+)
+
+type LangVar struct {
+	v []lang.Language
+}
+
+func(lv *LangVar) Set(s string) error {
+	v, err := lang.LanguageFromCode(s)
+	if err != nil {
+		return err
+	}
+	lv.v = append(lv.v, v)
+	return err
+}
+
+func(lv *LangVar) String() string {
+	var s []string
+	for _, v := range(lv.v) {
+		s = append(s, v.Code)
+	}
+	return strings.Join(s, ",")
+}
+
+func(lv *LangVar) Langs() []lang.Language {
+	return lv.v
+}
+
+

internal/handlers/ussd/menuhandler.go

@@ -835,7 +835,7 @@ func (h *Handlers) QuitWithHelp(ctx context.Context, sym string, input []byte) (
 	l := gotext.NewLocale(translationDir, code)
 	l.AddDomain("default")
 
-	res.Content = l.Get("For more help,please call: 0757628885")
+	res.Content = l.Get("For more help, please call: 0757628885")
 	res.FlagReset = append(res.FlagReset, flag_account_authorized)
 	return res, nil
 }

internal/http/at/parse.go

@@ -81,7 +81,8 @@ func (arp *ATRequestParser) GetInput(rq any) ([]byte, error) {
 		return nil, fmt.Errorf("no input found")
 	}
 
-	return []byte(parts[len(parts)-1]), nil
+	trimmedInput := strings.TrimSpace(parts[len(parts)-1])
+	return []byte(trimmedInput), nil
 }
 
 func parseQueryParams(query string) map[string]string {

internal/storage/storageservice.go

@@ -9,6 +9,7 @@ import (
 	"git.defalsify.org/vise.git/db"
 	fsdb "git.defalsify.org/vise.git/db/fs"
 	"git.defalsify.org/vise.git/db/postgres"
+	"git.defalsify.org/vise.git/lang"
 	"git.defalsify.org/vise.git/logging"
 	"git.defalsify.org/vise.git/persist"
 	"git.defalsify.org/vise.git/resource"
@@ -31,6 +32,7 @@ type StorageService interface {
 type MenuStorageService struct {
 	dbDir         string
 	resourceDir   string
+	poResource    resource.Resource
 	resourceStore db.Db
 	stateStore    db.Db
 	userDataStore db.Db
@@ -59,6 +61,28 @@ func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService
 	}
 }
 
+// WithGettext triggers use of gettext for translation of templates and menus.
+//
+// The first language in `lns` will be used as default language, to resolve node keys to 
+// language strings.
+//
+// If `lns` is an empty array, gettext will not be used.
+func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *MenuStorageService {
+	if len(lns) == 0 {
+		logg.Warnf("Gettext requested but no languages supplied")
+		return ms
+	}
+	rs := resource.NewPoResource(lns[0], path)
+
+	for _, ln := range(lns) {
+		rs = rs.WithLanguage(ln)
+	}
+
+	ms.poResource = rs
+
+	return ms
+}
+
 func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) {
 	database, ok := ctx.Value("Database").(string)
 	if !ok {
@@ -150,6 +174,11 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc
 		return nil, err
 	}
 	rfs := resource.NewDbResource(ms.resourceStore)
+	if ms.poResource != nil {
+		logg.InfoCtxf(ctx, "using poresource for menu and template")
+		rfs.WithMenuGetter(ms.poResource.GetMenu)
+		rfs.WithTemplateGetter(ms.poResource.GetTemplate)
+	}
 	return rfs, nil
 }
 

services/registration/locale/swa/default.po

@@ -7,8 +7,8 @@ msgstr "Ombi lako limetumwa. %s atapokea %s %s kutoka kwa %s."
 msgid "Thank you for using Sarafu. Goodbye!"
 msgstr "Asante kwa kutumia huduma ya Sarafu. Kwaheri!"
 
-msgid "For more help,please call: 0757628885"
-msgstr "Kwa usaidizi zaidi,piga: 0757628885"
+msgid "For more help, please call: 0757628885"
+msgstr "Kwa usaidizi zaidi, piga: 0757628885"
 
 msgid "Balance: %s\n"
 msgstr "Salio: %s\n"