From 6f65c33be4b8e416dac0baaa5f8935b2cc61820f Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 26 Sep 2024 15:15:06 +0100 Subject: [PATCH 01/81] Re-add ssh --- cmd/ssh/README.md | 34 +++++ cmd/ssh/main.go | 115 ++++++++++++++++ cmd/ssh/sshkey/main.go | 44 ++++++ internal/ssh/keystore.go | 64 +++++++++ internal/ssh/ssh.go | 284 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 541 insertions(+) create mode 100644 cmd/ssh/README.md create mode 100644 cmd/ssh/main.go create mode 100644 cmd/ssh/sshkey/main.go create mode 100644 internal/ssh/keystore.go create mode 100644 internal/ssh/ssh.go diff --git a/cmd/ssh/README.md b/cmd/ssh/README.md new file mode 100644 index 0000000..ff325d7 --- /dev/null +++ b/cmd/ssh/README.md @@ -0,0 +1,34 @@ +# URDT-USSD SSH server + +An SSH server entry point for the vise engine. + + +## Adding public keys for access + +Map your (client) public key to a session identifier (e.g. phone number) + +``` +go run -v -tags logtrace ./cmd/ssh/sshkey/main.go -i [--dbdir ] +``` + + +## Create a private key for the server + +``` +ssh-keygen -N "" -f +``` + + +## Run the server + + +``` +go run -v -tags logtrace ./cmd/ssh/main.go -h -p [--dbdir ] +``` + + +## Connect to the server + +``` +ssh [-v] -T -p -i +``` diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go new file mode 100644 index 0000000..0227616 --- /dev/null +++ b/cmd/ssh/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "flag" + "fmt" + "path" + "os" + "os/signal" + "sync" + "syscall" + + "git.defalsify.org/vise.git/db" + "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/logging" + + "git.grassecon.net/urdt/ussd/internal/ssh" +) + +var ( + wg sync.WaitGroup + keyStore db.Db + logg = logging.NewVanilla() + scriptDir = path.Join("services", "registration") +) + +func main() { + var dbDir string + var resourceDir string + var size uint + var engineDebug bool + var stateDebug bool + var host string + var port uint + flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") + flag.BoolVar(&engineDebug, "engine-debug", false, "use engine debug output") + flag.BoolVar(&stateDebug, "state-debug", false, "use engine debug output") + flag.UintVar(&size, "s", 160, "max size of output") + flag.StringVar(&host, "h", "127.0.0.1", "http host") + flag.UintVar(&port, "p", 7122, "http port") + flag.Parse() + + sshKeyFile := flag.Arg(0) + _, err := os.Stat(sshKeyFile) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot open ssh server private key file: %v\n", err) + os.Exit(1) + } + + ctx := context.Background() + logg.WarnCtxf(ctx, "!!!!! WARNING WARNING WARNING") + logg.WarnCtxf(ctx, "!!!!! =======================") + logg.WarnCtxf(ctx, "!!!!! This is not a production ready server!") + logg.WarnCtxf(ctx, "!!!!! Do not expose to internet and only use with tunnel!") + logg.WarnCtxf(ctx, "!!!!! (See ssh -L <...>)") + + logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "keyfile", sshKeyFile, "host", host, "port", port) + + pfp := path.Join(scriptDir, "pp.csv") + + cfg := engine.Config{ + Root: "root", + OutputSize: uint32(size), + FlagCount: uint32(16), + } + if stateDebug { + cfg.StateDebug = true + } + if engineDebug { + cfg.EngineDebug = true + } + + authKeyStore, err := ssh.NewSshKeyStore(ctx, dbDir) + if err != nil { + fmt.Fprintf(os.Stderr, "keystore file open error: %v", err) + os.Exit(1) + } + defer func () { + logg.TraceCtxf(ctx, "shutdown auth key store reached") + err = authKeyStore.Close() + if err != nil { + logg.ErrorCtxf(ctx, "keystore close error", "err", err) + } + }() + + cint := make(chan os.Signal) + cterm := make(chan os.Signal) + signal.Notify(cint, os.Interrupt, syscall.SIGINT) + signal.Notify(cterm, os.Interrupt, syscall.SIGTERM) + + runner := &ssh.SshRunner{ + Cfg: cfg, + Debug: engineDebug, + FlagFile: pfp, + DbDir: dbDir, + ResourceDir: resourceDir, + SrvKeyFile: sshKeyFile, + Host: host, + Port: port, + } + go func() { + select { + case _ = <-cint: + case _ = <-cterm: + } + logg.TraceCtxf(ctx, "shutdown runner reached") + err := runner.Stop() + if err != nil { + logg.ErrorCtxf(ctx, "runner stop error", "err", err) + } + + }() + runner.Run(ctx, authKeyStore) +} diff --git a/cmd/ssh/sshkey/main.go b/cmd/ssh/sshkey/main.go new file mode 100644 index 0000000..87b89a3 --- /dev/null +++ b/cmd/ssh/sshkey/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + + "git.grassecon.net/urdt/ussd/internal/ssh" +) + +func main() { + var dbDir string + var sessionId string + flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&sessionId, "i", "", "session id") + flag.Parse() + + if sessionId == "" { + fmt.Fprintf(os.Stderr, "empty session id\n") + os.Exit(1) + } + + ctx := context.Background() + + sshKeyFile := flag.Arg(0) + if sshKeyFile == "" { + fmt.Fprintf(os.Stderr, "missing key file argument\n") + os.Exit(1) + } + + store, err := ssh.NewSshKeyStore(ctx, dbDir) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + defer store.Close() + + err = store.AddFromFile(ctx, sshKeyFile, sessionId) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/internal/ssh/keystore.go b/internal/ssh/keystore.go new file mode 100644 index 0000000..fed7233 --- /dev/null +++ b/internal/ssh/keystore.go @@ -0,0 +1,64 @@ +package ssh + +import ( + "context" + "fmt" + "os" + "path" + + "golang.org/x/crypto/ssh" + + "git.defalsify.org/vise.git/db" + + "git.grassecon.net/urdt/ussd/internal/storage" +) + +type SshKeyStore struct { + store db.Db +} + +func NewSshKeyStore(ctx context.Context, dbDir string) (*SshKeyStore, error) { + keyStore := &SshKeyStore{} + keyStoreFile := path.Join(dbDir, "ssh_authorized_keys.gdbm") + keyStore.store = storage.NewThreadGdbmDb() + err := keyStore.store.Connect(ctx, keyStoreFile) + if err != nil { + return nil, err + } + return keyStore, nil +} + +func(s *SshKeyStore) AddFromFile(ctx context.Context, fp string, sessionId string) error { + _, err := os.Stat(fp) + if err != nil { + return fmt.Errorf("cannot open ssh server public key file: %v\n", err) + } + + publicBytes, err := os.ReadFile(fp) + if err != nil { + return fmt.Errorf("Failed to load public key: %v", err) + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey(publicBytes) + if err != nil { + return fmt.Errorf("Failed to parse public key: %v", err) + } + k := append([]byte{0x01}, pubKey.Marshal()...) + s.store.SetPrefix(storage.DATATYPE_CUSTOM) + logg.Infof("Added key", "sessionId", sessionId, "public key", string(publicBytes)) + return s.store.Put(ctx, k, []byte(sessionId)) +} + +func(s *SshKeyStore) Get(ctx context.Context, pubKey ssh.PublicKey) (string, error) { + s.store.SetLanguage(nil) + s.store.SetPrefix(storage.DATATYPE_CUSTOM) + k := append([]byte{0x01}, pubKey.Marshal()...) + v, err := s.store.Get(ctx, k) + if err != nil { + return "", err + } + return string(v), nil +} + +func(s *SshKeyStore) Close() error { + return s.store.Close() +} diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go new file mode 100644 index 0000000..394f55f --- /dev/null +++ b/internal/ssh/ssh.go @@ -0,0 +1,284 @@ +package ssh + +import ( + "context" + "encoding/hex" + "encoding/base64" + "errors" + "fmt" + "net" + "os" + "sync" + + "golang.org/x/crypto/ssh" + + "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/logging" + "git.defalsify.org/vise.git/resource" + "git.defalsify.org/vise.git/state" + + "git.grassecon.net/urdt/ussd/internal/handlers" + "git.grassecon.net/urdt/ussd/internal/storage" +) + +var ( + logg = logging.NewVanilla().WithDomain("ssh") +) + +type auther struct { + Ctx context.Context + keyStore *SshKeyStore + auth map[string]string +} + +func NewAuther(ctx context.Context, keyStore *SshKeyStore) *auther { + return &auther{ + Ctx: ctx, + keyStore: keyStore, + auth: make(map[string]string), + } +} + +func(a *auther) Check(conn ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + va, err := a.keyStore.Get(a.Ctx, pubKey) + if err != nil { + return nil, err + } + ka := hex.EncodeToString(conn.SessionID()) + a.auth[ka] = va + fmt.Fprintf(os.Stderr, "connect: %s -> %s\n", ka, va) + return nil, nil +} + +func(a *auther) FromConn(c *ssh.ServerConn) (string, error) { + if c == nil { + return "", errors.New("nil server conn") + } + if c.Conn == nil { + return "", errors.New("nil underlying conn") + } + return a.Get(c.Conn.SessionID()) +} + + +func(a *auther) Get(k []byte) (string, error) { + ka := hex.EncodeToString(k) + v, ok := a.auth[ka] + if !ok { + return "", errors.New("not found") + } + return v, nil +} + +func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChannel, en engine.Engine) error { + if ch == nil { + return errors.New("nil channel") + } + if ch.ChannelType() != "session" { + ch.Reject(ssh.UnknownChannelType, "that is not the channel you are looking for") + return errors.New("not a session") + } + channel, requests, err := ch.Accept() + if err != nil { + panic(err) + } + defer channel.Close() + s.wg.Add(1) + go func(reqIn <-chan *ssh.Request) { + defer s.wg.Done() + for req := range reqIn { + req.Reply(req.Type == "shell", nil) + } + _ = requests + }(requests) + + cont, err := en.Exec(ctx, []byte{}) + if err != nil { + return fmt.Errorf("initial engine exec err: %v", err) + } + + var input [state.INPUT_LIMIT]byte + for cont { + c, err := en.Flush(ctx, channel) + if err != nil { + return fmt.Errorf("flush err: %v", err) + } + _, err = channel.Write([]byte{0x0a}) + if err != nil { + return fmt.Errorf("newline err: %v", err) + } + c, err = channel.Read(input[:]) + if err != nil { + return fmt.Errorf("read input fail: %v", err) + } + logg.TraceCtxf(ctx, "input read", "c", c, "input", input[:c-1]) + cont, err = en.Exec(ctx, input[:c-1]) + if err != nil { + return fmt.Errorf("engine exec err: %v", err) + } + logg.TraceCtxf(ctx, "exec cont", "cont", cont, "en", en) + _ = c + } + c, err := en.Flush(ctx, channel) + if err != nil { + return fmt.Errorf("last flush err: %v", err) + } + _ = c + return nil +} + +type SshRunner struct { + Ctx context.Context + Cfg engine.Config + FlagFile string + DbDir string + ResourceDir string + Debug bool + SrvKeyFile string + Host string + Port uint + wg sync.WaitGroup + lst net.Listener +} + +func(s *SshRunner) Stop() error { + return s.lst.Close() +} + +func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) { + ctx := s.Ctx + menuStorageService := storage.NewMenuStorageService(s.DbDir, s.ResourceDir) + + err := menuStorageService.EnsureDbDir() + if err != nil { + return nil, nil, err + } + + rs, err := menuStorageService.GetResource(ctx) + if err != nil { + return nil, nil, err + } + + pe, err := menuStorageService.GetPersister(ctx) + if err != nil { + return nil, nil, err + } + + userdatastore, err := menuStorageService.GetUserdataDb(ctx) + if err != nil { + return nil, nil, err + } + + dbResource, ok := rs.(*resource.DbResource) + if !ok { + return nil, nil, err + } + + lhs, err := handlers.NewLocalHandlerService(s.FlagFile, true, dbResource, s.Cfg, rs) + lhs.SetDataStore(&userdatastore) + lhs.SetPersister(pe) + lhs.Cfg.SessionId = sessionId + + if err != nil { + return nil, nil, err + } + + hl, err := lhs.GetHandler() + if err != nil { + return nil, nil, err + } + + en := lhs.GetEngine() + en = en.WithFirst(hl.Init) + if s.Debug { + en = en.WithDebug(nil) + } + // TODO: this is getting very hacky! + closer := func() { + err := menuStorageService.Close() + if err != nil { + logg.ErrorCtxf(ctx, "menu storage service cleanup fail", "err", err) + } + } + return en, closer, nil +} + +// adapted example from crypto/ssh package, NewServerConn doc +func(s *SshRunner) Run(ctx context.Context, keyStore *SshKeyStore) { + running := true + + // TODO: waitgroup should probably not be global + defer s.wg.Wait() + + auth := NewAuther(ctx, keyStore) + cfg := ssh.ServerConfig{ + PublicKeyCallback: auth.Check, + } + + privateBytes, err := os.ReadFile(s.SrvKeyFile) + if err != nil { + logg.ErrorCtxf(ctx, "Failed to load private key", "err", err) + } + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + logg.ErrorCtxf(ctx, "Failed to parse private key", "err", err) + } + srvPub := private.PublicKey() + srvPubStr := base64.StdEncoding.EncodeToString(srvPub.Marshal()) + logg.InfoCtxf(ctx, "have server key", "type", srvPub.Type(), "public", srvPubStr) + cfg.AddHostKey(private) + + s.lst, err = net.Listen("tcp", fmt.Sprintf("%s:%d", s.Host, s.Port)) + if err != nil { + panic(err) + } + + for running { + conn, err := s.lst.Accept() + if err != nil { + logg.ErrorCtxf(ctx, "ssh accept error", "err", err) + running = false + continue + } + + go func(conn net.Conn) { + defer conn.Close() + for true { + srvConn, nC, rC, err := ssh.NewServerConn(conn, &cfg) + if err != nil { + logg.InfoCtxf(ctx, "rejected client", "err", err) + return + } + logg.DebugCtxf(ctx, "ssh client connected", "conn", srvConn) + + s.wg.Add(1) + go func() { + ssh.DiscardRequests(rC) + s.wg.Done() + }() + + sessionId, err := auth.FromConn(srvConn) + if err != nil { + logg.ErrorCtxf(ctx, "Cannot find authentication") + return + } + en, closer, err := s.GetEngine(sessionId) + if err != nil { + logg.ErrorCtxf(ctx, "engine won't start", "err", err) + return + } + defer func() { + err := en.Finish() + if err != nil { + logg.ErrorCtxf(ctx, "engine won't stop", "err", err) + } + closer() + }() + for ch := range nC { + err = s.serve(ctx, sessionId, ch, en) + logg.ErrorCtxf(ctx, "ssh server finish", "err", err) + } + } + }(conn) + } +} From d246cdee51cc537884fd541ca6db6e58a1c1ab6c Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 27 Sep 2024 21:25:21 +0100 Subject: [PATCH 02/81] Rename datatype const name for ssh prefix --- internal/ssh/keystore.go | 4 ++-- internal/storage/storage.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/ssh/keystore.go b/internal/ssh/keystore.go index fed7233..b80727b 100644 --- a/internal/ssh/keystore.go +++ b/internal/ssh/keystore.go @@ -43,14 +43,14 @@ func(s *SshKeyStore) AddFromFile(ctx context.Context, fp string, sessionId strin return fmt.Errorf("Failed to parse public key: %v", err) } k := append([]byte{0x01}, pubKey.Marshal()...) - s.store.SetPrefix(storage.DATATYPE_CUSTOM) + s.store.SetPrefix(storage.DATATYPE_EXTEND) logg.Infof("Added key", "sessionId", sessionId, "public key", string(publicBytes)) return s.store.Put(ctx, k, []byte(sessionId)) } func(s *SshKeyStore) Get(ctx context.Context, pubKey ssh.PublicKey) (string, error) { s.store.SetLanguage(nil) - s.store.SetPrefix(storage.DATATYPE_CUSTOM) + s.store.SetPrefix(storage.DATATYPE_EXTEND) k := append([]byte{0x01}, pubKey.Marshal()...) v, err := s.store.Get(ctx, k) if err != nil { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 53f4392..231a1db 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,6 +5,10 @@ import ( "git.defalsify.org/vise.git/persist" ) +const ( + DATATYPE_EXTEND = 128 +) + type Storage struct { Persister *persist.Persister UserdataDb db.Db From 6ee2c88fe25d0bdab9305c3d14a388831473cb29 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 09:39:49 +0000 Subject: [PATCH 03/81] Implement gettext spec in local vm cmd --- cmd/main.go | 23 +++++++++++++++++++- config/config.go | 8 +++++++ internal/args/lang.go | 34 ++++++++++++++++++++++++++++++ internal/storage/storageservice.go | 22 +++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 internal/args/lang.go diff --git a/cmd/main.go b/cmd/main.go index 4fd084f..e6f96ab 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,10 +10,12 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/remote" ) @@ -31,22 +33,38 @@ func main() { config.LoadConfig() var dbDir string + var gettextDir string var size uint var sessionId string var database string var engineDebug bool + var langs args.LangVar flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&database, "db", "gdbm", "database to be used") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") + flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory") + flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "outputsize", size) + if len(langs.Langs()) == 0 { + langs.Set(config.DefaultLanguage) + } + ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) + + ln, err := lang.LanguageFromCode(config.DefaultLanguage) + if err != nil { + fmt.Fprintf(os.Stderr, "default language set error: %v", err) + os.Exit(1) + } + ctx = context.WithValue(ctx, "Language", ln) + pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ @@ -59,8 +77,11 @@ func main() { resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + if gettextDir != "" { + menuStorageService = menuStorageService.WithGettext(gettextDir, langs.Langs()) + } - err := menuStorageService.EnsureDbDir() + err = menuStorageService.EnsureDbDir() if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/config/config.go b/config/config.go index 3a8e8ed..6575310 100644 --- a/config/config.go +++ b/config/config.go @@ -18,6 +18,10 @@ const ( AliasPrefix = "api/v1/alias" ) +var ( + defaultLanguage = "eng" +) + var ( custodialURLBase string dataURLBase string @@ -34,6 +38,7 @@ var ( VoucherTransfersURL string VoucherDataURL string CheckAliasURL string + DefaultLanguage string ) func setBase() error { @@ -51,6 +56,8 @@ func setBase() error { if err != nil { return err } + + defaultLanguage = initializers.GetEnv("DEFAULT_LANGUAGE", defaultLanguage) return nil } @@ -69,6 +76,7 @@ func LoadConfig() error { VoucherTransfersURL, _ = url.JoinPath(dataURLBase, voucherTransfersPathPrefix) VoucherDataURL, _ = url.JoinPath(dataURLBase, voucherDataPathPrefix) CheckAliasURL, _ = url.JoinPath(dataURLBase, AliasPrefix) + DefaultLanguage = defaultLanguage return nil } diff --git a/internal/args/lang.go b/internal/args/lang.go new file mode 100644 index 0000000..f9afdc9 --- /dev/null +++ b/internal/args/lang.go @@ -0,0 +1,34 @@ +package args + +import ( + "strings" + + "git.defalsify.org/vise.git/lang" +) + +type LangVar struct { + v []lang.Language +} + +func(lv *LangVar) Set(s string) error { + v, err := lang.LanguageFromCode(s) + if err != nil { + return err + } + lv.v = append(lv.v, v) + return err +} + +func(lv *LangVar) String() string { + var s []string + for _, v := range(lv.v) { + s = append(s, v.Code) + } + return strings.Join(s, ",") +} + +func(lv *LangVar) Langs() []lang.Language { + return lv.v +} + + diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index ca28bbb..e420dee 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -9,6 +9,7 @@ import ( "git.defalsify.org/vise.git/db" fsdb "git.defalsify.org/vise.git/db/fs" "git.defalsify.org/vise.git/db/postgres" + "git.defalsify.org/vise.git/lang" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" @@ -29,6 +30,7 @@ type StorageService interface { type MenuStorageService struct { dbDir string resourceDir string + poResource resource.Resource resourceStore db.Db stateStore db.Db userDataStore db.Db @@ -57,6 +59,22 @@ func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService } } +func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *MenuStorageService { + ln := lang.Language{ + Code: "xxx", + Name: "Translation key", + } + rs := resource.NewPoResource(ln, path) + + for _, ln = range(lns) { + rs = rs.WithLanguage(ln) + } + + ms.poResource = rs + + return ms +} + func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) { database, ok := ctx.Value("Database").(string) if !ok { @@ -119,6 +137,10 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc return nil, err } rfs := resource.NewDbResource(ms.resourceStore) + if ms.poResource != nil { + rfs.WithMenuGetter(ms.poResource.GetMenu) + rfs.WithTemplateGetter(ms.poResource.GetTemplate) + } return rfs, nil } From 06230dc557b71e304a824f0eaf42b66bf5233310 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 14:31:13 +0000 Subject: [PATCH 04/81] Add todo comment --- cmd/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/main.go b/cmd/main.go index e6f96ab..2737c84 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,6 +29,7 @@ func init() { initializers.LoadEnvVariables() } +// TODO: external script automatically generate language handler list from select language vise code OR consider dynamic menu generation script possibility func main() { config.LoadConfig() From 43b2c3b78d4d49f792f143ac91faf265095454d0 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 18:13:37 +0000 Subject: [PATCH 05/81] Rehabilitate gettext resource --- go.mod | 2 +- go.sum | 4 ++-- internal/storage/storageservice.go | 17 ++++++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 16ccdc3..09d2cd5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.3-0.20241231085136-8582c7e157d9 + git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 9086cd8..29b3893 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.3-0.20241231085136-8582c7e157d9 h1:O3m+NgWDWtJm8OculT99c4bDMAO4xLe2c8hpCKpsd9g= -git.defalsify.org/vise.git v0.2.3-0.20241231085136-8582c7e157d9/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be h1:bh+VQanntFrMjQA/ZPRv5Wlist4e743EbiBvK1l9Qac= +git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index e420dee..451358e 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -59,14 +59,20 @@ func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService } } +// WithGettext triggers use of gettext for translation of templates and menus. +// +// The first language in `lns` will be used as default language, to resolve node keys to +// language strings. +// +// If `lns` is an empty array, gettext will not be used. func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *MenuStorageService { - ln := lang.Language{ - Code: "xxx", - Name: "Translation key", + if len(lns) == 0 { + logg.Warnf("Gettext requested but no languages supplied") + return ms } - rs := resource.NewPoResource(ln, path) + rs := resource.NewPoResource(lns[0], path) - for _, ln = range(lns) { + for _, ln := range(lns) { rs = rs.WithLanguage(ln) } @@ -138,6 +144,7 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc } rfs := resource.NewDbResource(ms.resourceStore) if ms.poResource != nil { + logg.InfoCtxf(ctx, "using poresource for menu and template") rfs.WithMenuGetter(ms.poResource.GetMenu) rfs.WithTemplateGetter(ms.poResource.GetTemplate) } From b888af446d44c0dace1387f7ec56f94e2a391c7c Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 18:49:16 +0000 Subject: [PATCH 06/81] update govise --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 09d2cd5..780f0e6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be + git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57 github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 29b3893..812dc15 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be h1:bh+VQanntFrMjQA/ZPRv5Wlist4e743EbiBvK1l9Qac= -git.defalsify.org/vise.git v0.2.3-0.20250102175429-e5b5bc34a2be/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57 h1:fBGbWr7AMPj5gEyZmMwLaaaIFNvxAhY7fteDFolGMlw= +git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= From 6723884103f93827457c3294fa5984c157aec745 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 21:02:01 +0000 Subject: [PATCH 07/81] Update go-vise --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 780f0e6..9862ff0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57 + git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7 github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 812dc15..59b2258 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57 h1:fBGbWr7AMPj5gEyZmMwLaaaIFNvxAhY7fteDFolGMlw= -git.defalsify.org/vise.git v0.2.3-0.20250102184614-1c71081a3b57/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7 h1:/PLEm5DCJXwlgDsQ36s20qfdlKIwdBfTGmFYqsIy29E= +git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= From c1e0617bb317e398487db8d33744c1542611e100 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 2 Jan 2025 21:13:06 +0000 Subject: [PATCH 08/81] Update go-vise --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9862ff0..6ddb811 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.23.0 require ( - git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7 + git.defalsify.org/vise.git v0.2.3-0.20250102211200-fd3db37d2626 github.com/alecthomas/assert/v2 v2.2.2 github.com/gofrs/uuid v4.4.0+incompatible github.com/grassrootseconomics/eth-custodial v1.3.0-beta diff --git a/go.sum b/go.sum index 59b2258..58b405f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7 h1:/PLEm5DCJXwlgDsQ36s20qfdlKIwdBfTGmFYqsIy29E= -git.defalsify.org/vise.git v0.2.3-0.20250102205756-049511818ec7/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= +git.defalsify.org/vise.git v0.2.3-0.20250102211200-fd3db37d2626 h1:ANfeAGxCtjDtkzHNesttnI7CtF0zjhhxsTZj0+ipq9s= +git.defalsify.org/vise.git v0.2.3-0.20250102211200-fd3db37d2626/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= From 1b12f0ba5f0f62ff006a8a0b6e05edcf6592901f Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 10:00:52 +0000 Subject: [PATCH 09/81] Add po language alternative to all runners --- cmd/africastalking/main.go | 13 +++++++++++++ cmd/async/main.go | 14 ++++++++++++++ cmd/http/main.go | 14 ++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index ca88978..9f42199 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -18,6 +18,7 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/common" "git.grassecon.net/urdt/ussd/config" @@ -26,6 +27,7 @@ import ( httpserver "git.grassecon.net/urdt/ussd/internal/http" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" + "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -114,6 +116,8 @@ func main() { var engineDebug bool var host string var port uint + var gettextDir string + var langs args.LangVar flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.StringVar(&database, "db", "gdbm", "database to be used") @@ -121,12 +125,21 @@ func main() { flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") + flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory") + flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() logg.Infof("start command", "build", build, "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + ln, err := lang.LanguageFromCode(config.DefaultLanguage) + if err != nil { + fmt.Fprintf(os.Stderr, "default language set error: %v", err) + os.Exit(1) + } + ctx = context.WithValue(ctx, "Language", ln) + pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/async/main.go b/cmd/async/main.go index 9cd04b3..de2a85b 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -12,12 +12,14 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" + "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -54,6 +56,8 @@ func main() { var engineDebug bool var host string var port uint + var gettextDir string + var langs args.LangVar flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") @@ -62,12 +66,22 @@ func main() { flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") + flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory") + flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + + ln, err := lang.LanguageFromCode(config.DefaultLanguage) + if err != nil { + fmt.Fprintf(os.Stderr, "default language set error: %v", err) + os.Exit(1) + } + ctx = context.WithValue(ctx, "Language", ln) + pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/http/main.go b/cmd/http/main.go index 6ddfded..c47447a 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -14,6 +14,7 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -21,6 +22,7 @@ import ( httpserver "git.grassecon.net/urdt/ussd/internal/http" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" + "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -43,6 +45,8 @@ func main() { var engineDebug bool var host string var port uint + var gettextDir string + var langs args.LangVar flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.StringVar(&database, "db", "gdbm", "database to be used") @@ -50,12 +54,22 @@ func main() { flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") + flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory") + flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + + ln, err := lang.LanguageFromCode(config.DefaultLanguage) + if err != nil { + fmt.Fprintf(os.Stderr, "default language set error: %v", err) + os.Exit(1) + } + ctx = context.WithValue(ctx, "Language", ln) + pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ From e16b7445e8841f5fe4f4cc00f7095b2c320cb296 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 10:28:27 +0000 Subject: [PATCH 10/81] Move arg var to same spot as other runners --- cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index 2737c84..a61176d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,11 +34,11 @@ func main() { config.LoadConfig() var dbDir string - var gettextDir string var size uint var sessionId string var database string var engineDebug bool + var gettextDir string var langs args.LangVar flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&database, "db", "gdbm", "database to be used") From 056d0566138fac247bcf25d53003ad41d4fa2bdb Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 14:43:08 +0000 Subject: [PATCH 11/81] Add language source and template file generator --- config/config.go | 28 +++++++++- devtools/lang/main.go | 126 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 devtools/lang/main.go diff --git a/config/config.go b/config/config.go index 6575310..02fdaab 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "net/url" + "strings" "git.grassecon.net/urdt/ussd/initializers" ) @@ -20,6 +21,7 @@ const ( var ( defaultLanguage = "eng" + languages []string ) var ( @@ -39,8 +41,27 @@ var ( VoucherDataURL string CheckAliasURL string DefaultLanguage string + Languages []string ) +func setLanguage() error { + defaultLanguage = initializers.GetEnv("DEFAULT_LANGUAGE", defaultLanguage) + languages = strings.Split(initializers.GetEnv("LANGUAGES", defaultLanguage), ",") + haveDefaultLanguage := false + for i, v := range(languages) { + languages[i] = strings.ReplaceAll(v, " ", "") + if languages[i] == defaultLanguage { + haveDefaultLanguage = true + } + } + + if !haveDefaultLanguage { + languages = append([]string{defaultLanguage}, languages...) + } + + return nil +} + func setBase() error { var err error @@ -56,8 +77,6 @@ func setBase() error { if err != nil { return err } - - defaultLanguage = initializers.GetEnv("DEFAULT_LANGUAGE", defaultLanguage) return nil } @@ -67,6 +86,10 @@ func LoadConfig() error { if err != nil { return err } + err = setLanguage() + if err != nil { + return err + } CreateAccountURL, _ = url.JoinPath(custodialURLBase, createAccountPath) TrackStatusURL, _ = url.JoinPath(custodialURLBase, trackStatusPath) BalanceURL, _ = url.JoinPath(custodialURLBase, balancePathPrefix) @@ -77,6 +100,7 @@ func LoadConfig() error { VoucherDataURL, _ = url.JoinPath(dataURLBase, voucherDataPathPrefix) CheckAliasURL, _ = url.JoinPath(dataURLBase, AliasPrefix) DefaultLanguage = defaultLanguage + Languages = languages return nil } diff --git a/devtools/lang/main.go b/devtools/lang/main.go new file mode 100644 index 0000000..81655f0 --- /dev/null +++ b/devtools/lang/main.go @@ -0,0 +1,126 @@ +package main + +import ( + "flag" + "fmt" + "os" + "path" + "strings" + + "git.defalsify.org/vise.git/logging" + "git.defalsify.org/vise.git/lang" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" +) + +const ( + + changeHeadSrc = `LOAD reset_account_authorized 0 +LOAD reset_incorrect 0 +CATCH incorrect_pin flag_incorrect_pin 1 +CATCH pin_entry flag_account_authorized 0 +` + + selectSrc = `LOAD set_language 6 +RELOAD set_language +CATCH terms flag_account_created 0 +MOVE language_changed +` +) + +var ( + logg = logging.NewVanilla() + mouts string + incmps string +) + +func init() { + initializers.LoadEnvVariables() +} + +func toLanguageLabel(ln lang.Language) string { + s := ln.Name + v := strings.Split(s, " (") + if len(v) > 1 { + s = v[0] + } + return s +} + +func toLanguageKey(ln lang.Language) string { + s := toLanguageLabel(ln) + return strings.ToLower(s) +} + +func main() { + var srcDir string + + flag.StringVar(&srcDir, "o", ".", "resource dir write to") + flag.Parse() + + logg.Infof("start command", "dir", srcDir) + + err := config.LoadConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "config load error: %v", err) + os.Exit(1) + } + logg.Tracef("using languages", "lang", config.Languages) + + for i, v := range(config.Languages) { + ln, err := lang.LanguageFromCode(v) + if err != nil { + fmt.Fprintf(os.Stderr, "error parsing language: %s", v) + os.Exit(1) + } + n := i + 1 + s := toLanguageKey(ln) + mouts += fmt.Sprintf("MOUT %s %v\n", s, n) + //incmp += fmt.Sprintf("INCMP set_%s %u\n", + v = "set_" + ln.Code + incmps += fmt.Sprintf("INCMP %s %v\n", v, n) + + p := path.Join(srcDir, v) + w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + if err != nil { + fmt.Fprintf(os.Stderr, "failed open language set template output: %v", err) + os.Exit(1) + } + s = toLanguageLabel(ln) + defer w.Close() + _, err = w.Write([]byte(s)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + os.Exit(1) + } + } + src := mouts + "HALT\n" + incmps + src += "INCMP . *\n" + + p := path.Join(srcDir, "select_language.vis") + w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + if err != nil { + fmt.Fprintf(os.Stderr, "failed open select language vis output: %v", err) + os.Exit(1) + } + defer w.Close() + _, err = w.Write([]byte(src)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + os.Exit(1) + } + + src = changeHeadSrc + src + p = path.Join(srcDir, "change_language.vis") + w, err = os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + if err != nil { + fmt.Fprintf(os.Stderr, "failed open select language vis output: %v", err) + os.Exit(1) + } + defer w.Close() + _, err = w.Write([]byte(src)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + os.Exit(1) + } +} From 9013cc3618e2968029c2b3923b646fbe68766dd1 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 15:10:20 +0000 Subject: [PATCH 12/81] Improve error messages --- devtools/lang/main.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/devtools/lang/main.go b/devtools/lang/main.go index 81655f0..83c68b3 100644 --- a/devtools/lang/main.go +++ b/devtools/lang/main.go @@ -1,3 +1,4 @@ +// create language files from environment package main import ( @@ -70,27 +71,26 @@ func main() { for i, v := range(config.Languages) { ln, err := lang.LanguageFromCode(v) if err != nil { - fmt.Fprintf(os.Stderr, "error parsing language: %s", v) + fmt.Fprintf(os.Stderr, "error parsing language: %s\n", v) os.Exit(1) } n := i + 1 s := toLanguageKey(ln) mouts += fmt.Sprintf("MOUT %s %v\n", s, n) - //incmp += fmt.Sprintf("INCMP set_%s %u\n", v = "set_" + ln.Code incmps += fmt.Sprintf("INCMP %s %v\n", v, n) p := path.Join(srcDir, v) - w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600) if err != nil { - fmt.Fprintf(os.Stderr, "failed open language set template output: %v", err) + fmt.Fprintf(os.Stderr, "failed open language set template output: %v\n", err) os.Exit(1) } s = toLanguageLabel(ln) defer w.Close() _, err = w.Write([]byte(s)) if err != nil { - fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err) os.Exit(1) } } @@ -98,29 +98,29 @@ func main() { src += "INCMP . *\n" p := path.Join(srcDir, "select_language.vis") - w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + w, err := os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600) if err != nil { - fmt.Fprintf(os.Stderr, "failed open select language vis output: %v", err) + fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err) os.Exit(1) } defer w.Close() _, err = w.Write([]byte(src)) if err != nil { - fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err) os.Exit(1) } src = changeHeadSrc + src p = path.Join(srcDir, "change_language.vis") - w, err = os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) + w, err = os.OpenFile(p, os.O_WRONLY | os.O_CREATE | os.O_EXCL, 0600) if err != nil { - fmt.Fprintf(os.Stderr, "failed open select language vis output: %v", err) + fmt.Fprintf(os.Stderr, "failed open select language vis output: %v\n", err) os.Exit(1) } defer w.Close() _, err = w.Write([]byte(src)) if err != nil { - fmt.Fprintf(os.Stderr, "failed write select language vis output: %v", err) + fmt.Fprintf(os.Stderr, "failed write select language vis output: %v\n", err) os.Exit(1) } } From ac0c43cb43a652ccfa0a649f7d4bce56fed72a76 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 17:18:23 +0000 Subject: [PATCH 13/81] Factor out formatting method --- devtools/store/main.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/devtools/store/main.go b/devtools/store/main.go index 8bd4d16..dfb9089 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -25,6 +25,15 @@ func init() { } +func formatItem(k []byte, v []byte) (string, error) { + o, err := debug.FromKey(k) + if err != nil { + return "", err + } + s := fmt.Sprintf("%vValue: %v\n\n", o, string(v)) + return s, nil +} + func main() { config.LoadConfig() @@ -64,12 +73,12 @@ func main() { if k == nil { break } - o, err := debug.FromKey(k) + r, err := formatItem(k, v) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "format db item error: %v", err) os.Exit(1) } - fmt.Printf("%vValue: %v\n\n", o, string(v)) + fmt.Printf(r) } err = store.Close() From daec816a3e0ba47629f6ee400ea7b5fe4cce51d5 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 3 Jan 2025 17:21:52 +0000 Subject: [PATCH 14/81] Move store devtools location --- devtools/store/{ => dump}/main.go | 0 devtools/{gen => store/generate}/main.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename devtools/store/{ => dump}/main.go (100%) rename devtools/{gen => store/generate}/main.go (100%) diff --git a/devtools/store/main.go b/devtools/store/dump/main.go similarity index 100% rename from devtools/store/main.go rename to devtools/store/dump/main.go diff --git a/devtools/gen/main.go b/devtools/store/generate/main.go similarity index 100% rename from devtools/gen/main.go rename to devtools/store/generate/main.go From dc1674ec551c9ba24a16886d7076a3a91eefdd64 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 08:40:43 +0000 Subject: [PATCH 15/81] WIP add connection string parser --- internal/storage/parse.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 internal/storage/parse.go diff --git a/internal/storage/parse.go b/internal/storage/parse.go new file mode 100644 index 0000000..0745a72 --- /dev/null +++ b/internal/storage/parse.go @@ -0,0 +1,21 @@ +package storage + +const ( + DBTYPE_MEM = iota + DBTYPE_GDBM + DBTYPE_POSTGRES +) + +type connData struct { + typ int + str string +} + +func toConnData(s string) connData { + var o connData + + if s == "" { + return o + } + return o +} From 2e30739ec9223d2028c558c4b56e1bf03394bec3 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 09:37:12 +0000 Subject: [PATCH 16/81] Implement connstr --- cmd/africastalking/main.go | 3 +- cmd/async/main.go | 31 +++++++++++----- cmd/http/main.go | 23 ++++++++---- cmd/main.go | 22 ++++++++--- common/storage.go | 14 ++++--- devtools/gen/main.go | 24 ++++++++++-- devtools/store/main.go | 22 +++++++++-- internal/storage/parse.go | 59 ++++++++++++++++++++++++++++-- internal/storage/parse_test.go | 37 +++++++++++++++++++ internal/storage/storageservice.go | 51 ++++++++++++++++---------- internal/testutil/TestEngine.go | 6 ++- 11 files changed, 227 insertions(+), 65 deletions(-) create mode 100644 internal/storage/parse_test.go diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 2696b51..11c6512 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -15,11 +15,10 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.grassecon.net/urdt/ussd/common" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" - httpserver "git.grassecon.net/urdt/ussd/internal/http/at" + at "git.grassecon.net/urdt/ussd/internal/http/at" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" ) diff --git a/cmd/async/main.go b/cmd/async/main.go index 9cd04b3..a9a0b1e 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -7,6 +7,7 @@ import ( "os" "os/signal" "path" + "path/filepath" "syscall" "git.defalsify.org/vise.git/engine" @@ -47,24 +48,33 @@ func main() { config.LoadConfig() var sessionId string - var dbDir string + var connStr string var resourceDir string var size uint var database string var engineDebug bool var host string var port uint + var err error + flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&connStr, "c", ".", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + + logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) @@ -81,18 +91,19 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + menuStorageService := storage.NewMenuStorageService(resourceDir) + err = menuStorageService.SetConn(connStr) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } - err = menuStorageService.EnsureDbDir() - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } userdataStore, err := menuStorageService.GetUserdataDb(ctx) if err != nil { diff --git a/cmd/http/main.go b/cmd/http/main.go index 6ddfded..dec4e64 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "path" + "path/filepath" "strconv" "syscall" @@ -36,23 +37,31 @@ func init() { func main() { config.LoadConfig() - var dbDir string + var connStr string var resourceDir string var size uint var database string var engineDebug bool var host string var port uint - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + var err error + flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&database, "db", "gdbm", "database to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + + logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) @@ -69,14 +78,14 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) - rs, err := menuStorageService.GetResource(ctx) + menuStorageService := storage.NewMenuStorageService(resourceDir) + err = menuStorageService.SetConn(connStr) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } - err = menuStorageService.EnsureDbDir() + rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/main.go b/cmd/main.go index 4fd084f..c45615b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "path/filepath" "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" @@ -30,19 +31,28 @@ func init() { func main() { config.LoadConfig() - var dbDir string + var connStr string var size uint var sessionId string var database string var engineDebug bool + var err error + flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&database, "db", "gdbm", "database to be used") - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&connStr, "c", ".", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.Parse() - logg.Infof("start command", "dbdir", dbDir, "outputsize", size) + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + + logg.Infof("start command", "connstr", connStr, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) @@ -58,9 +68,9 @@ func main() { } resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + menuStorageService := storage.NewMenuStorageService(resourceDir) - err := menuStorageService.EnsureDbDir() + err = menuStorageService.SetConn(connStr) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/common/storage.go b/common/storage.go index d37bce3..c8a4ff2 100644 --- a/common/storage.go +++ b/common/storage.go @@ -23,17 +23,19 @@ type StorageServices interface { GetPersister(ctx context.Context) (*persist.Persister, error) GetUserdataDb(ctx context.Context) (db.Db, error) GetResource(ctx context.Context) (resource.Resource, error) - EnsureDbDir() error + SetConn(connStr string) error } type StorageService struct { svc *storage.MenuStorageService } -func NewStorageService(dbDir string) *StorageService { - return &StorageService{ - svc: storage.NewMenuStorageService(dbDir, ""), +func NewStorageService(connStr string) (*StorageService, error) { + svc := &StorageService{ + svc: storage.NewMenuStorageService(""), } + err := svc.SetConn(connStr) + return svc, err } func(ss *StorageService) GetPersister(ctx context.Context) (*persist.Persister, error) { @@ -48,6 +50,6 @@ func(ss *StorageService) GetResource(ctx context.Context) (resource.Resource, er return nil, errors.New("not implemented") } -func(ss *StorageService) EnsureDbDir() error { - return ss.svc.EnsureDbDir() +func(ss *StorageService) SetConn(connStr string) error { + return ss.svc.SetConn(connStr) } diff --git a/devtools/gen/main.go b/devtools/gen/main.go index b9e2aed..448cc16 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path" + "path/filepath" "git.defalsify.org/vise.git/logging" "git.grassecon.net/urdt/ussd/config" @@ -28,23 +29,38 @@ func init() { func main() { config.LoadConfig() - var dbDir string + var connStr string var sessionId string var database string var engineDebug bool + var err error flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&database, "db", "gdbm", "database to be used") - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&connStr, "c", ".", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.Parse() + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + + logg.Infof("start command", "connstr", connStr) + ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + menuStorageService := storage.NewMenuStorageService(resourceDir) + err = menuStorageService.SetConn(connStr) + if err != nil { + fmt.Fprintf(os.Stderr, "connection string error: %v", err) + os.Exit(1) + } store, err := menuStorageService.GetUserdataDb(ctx) if err != nil { diff --git a/devtools/store/main.go b/devtools/store/main.go index 8bd4d16..ed54ba3 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path" + "path/filepath" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -28,23 +29,36 @@ func init() { func main() { config.LoadConfig() - var dbDir string + var connStr string var sessionId string var database string var engineDebug bool + var err error flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&database, "db", "gdbm", "database to be used") - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&connStr, "c", ".state", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.Parse() + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + menuStorageService := storage.NewMenuStorageService(resourceDir) + err = menuStorageService.SetConn(connStr) + if err != nil { + fmt.Fprintf(os.Stderr, "connection string error: %v", err) + os.Exit(1) + } store, err := menuStorageService.GetUserdataDb(ctx) if err != nil { diff --git a/internal/storage/parse.go b/internal/storage/parse.go index 0745a72..d4bc564 100644 --- a/internal/storage/parse.go +++ b/internal/storage/parse.go @@ -1,5 +1,11 @@ package storage +import ( + "fmt" + "net/url" + "path" +) + const ( DBTYPE_MEM = iota DBTYPE_GDBM @@ -11,11 +17,56 @@ type connData struct { str string } -func toConnData(s string) connData { +func (cd *connData) DbType() int { + return cd.typ +} + +func (cd *connData) String() string { + return cd.str +} + +func probePostgres(s string) (string, bool) { + v, err := url.Parse(s) + if err != nil { + return "", false + } + if v.Scheme != "postgres" { + return "", false + } + return s, true +} + +func probeGdbm(s string) (string, bool) { + if !path.IsAbs(s) { + return "", false + } + if path.Ext(s) != ".gdbm" { + return "", false + } + s = path.Clean(s) + return s, true +} + +func toConnData(connStr string) (connData, error) { var o connData - if s == "" { - return o + if connStr == "" { + return o, nil } - return o + + v, ok := probePostgres(connStr) + if ok { + o.typ = DBTYPE_POSTGRES + o.str = v + return o, nil + } + + v, ok = probeGdbm(connStr) + if ok { + o.typ = DBTYPE_GDBM + o.str = v + return o, nil + } + + return o, fmt.Errorf("invalid connection string: %s", connStr) } diff --git a/internal/storage/parse_test.go b/internal/storage/parse_test.go new file mode 100644 index 0000000..d4b8ecb --- /dev/null +++ b/internal/storage/parse_test.go @@ -0,0 +1,37 @@ +package storage + +import ( + "testing" +) + +func TestParseConnStr(t *testing.T) { + svc := NewMenuStorageService("") + err := svc.SetConn("postgres://foo:bar@localhost:5432/baz") + if err != nil { + t.Fatal(err) + } + err = svc.SetConn("/foo/bar/baz.gdbm") + if err != nil { + t.Fatal(err) + } + err = svc.SetConn("foo/bar/baz.gdbm") + if err == nil { + t.Fatalf("expected error") + } + err = svc.SetConn("http://foo/bar") + if err == nil { + t.Fatalf("expected error") + } + err = svc.SetConn("foo/bar/baz.txt") + if err == nil { + t.Fatalf("expected error") + } + err = svc.SetConn("/foo/bar") + if err == nil { + t.Fatalf("expected error") + } + err = svc.SetConn("foo/bar") + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 04e75ce..6a42bac 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "path" "git.defalsify.org/vise.git/db" fsdb "git.defalsify.org/vise.git/db/fs" @@ -24,11 +23,12 @@ type StorageService interface { GetPersister(ctx context.Context) (*persist.Persister, error) GetUserdataDb(ctx context.Context) db.Db GetResource(ctx context.Context) (resource.Resource, error) - EnsureDbDir() error + SetConn(connStr string) error } type MenuStorageService struct { - dbDir string + //dbDir string + conn connData resourceDir string resourceStore db.Db stateStore db.Db @@ -51,36 +51,47 @@ func buildConnStr() string { return connString } -func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService { +func NewMenuStorageService(resourceDir string) *MenuStorageService { return &MenuStorageService{ - dbDir: dbDir, resourceDir: resourceDir, } } -func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) { - database, ok := ctx.Value("Database").(string) - if !ok { - return nil, fmt.Errorf("failed to select the database") +func (ms *MenuStorageService) SetConn(connStr string) error { + o, err := toConnData(connStr) + if err != nil { + return err } + ms.conn = o + return nil +} + +func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) { + var newDb db.Db + var err error +// database, ok := ctx.Value("Database").(string) +// if !ok { +// return nil, fmt.Errorf("failed to select the database") +// } if existingDb != nil { return existingDb, nil } - var newDb db.Db - var err error - if database == "postgres" { + dbTyp := ms.conn.DbType() + if dbTyp == DBTYPE_POSTGRES { newDb = postgres.NewPgDb() - connStr := buildConnStr() - err = newDb.Connect(ctx, connStr) - } else { + } else if dbTyp == DBTYPE_GDBM { + err = ms.ensureDbDir() + if err != nil { + return nil, err + } newDb = gdbmstorage.NewThreadGdbmDb() - storeFile := path.Join(ms.dbDir, fileName) - err = newDb.Connect(ctx, storeFile) + } else { + return nil, fmt.Errorf("unsupported connection string: %s", ms.conn.String()) } - + err = newDb.Connect(ctx, ms.conn.String()) if err != nil { return nil, err } @@ -137,8 +148,8 @@ func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error) return ms.stateStore, nil } -func (ms *MenuStorageService) EnsureDbDir() error { - err := os.MkdirAll(ms.dbDir, 0700) +func (ms *MenuStorageService) ensureDbDir() error { + err := os.MkdirAll(ms.conn.String(), 0700) if err != nil { return fmt.Errorf("state dir create exited with error: %v\n", err) } diff --git a/internal/testutil/TestEngine.go b/internal/testutil/TestEngine.go index 3fcb307..63e9e84 100644 --- a/internal/testutil/TestEngine.go +++ b/internal/testutil/TestEngine.go @@ -41,9 +41,11 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { dbDir := ".test_state" resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + //menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + menuStorageService := storage.NewMenuStorageService(resourceDir) - err := menuStorageService.EnsureDbDir() + //err := menuStorageService.EnsureDbDir() + err := menuStorageService.SetConn(dbDir) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) From c3cbe1cd927df9c94faa73d8a6417a71d18986bb Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 09:41:17 +0000 Subject: [PATCH 17/81] Add connstr to last executable --- cmd/africastalking/main.go | 24 +++++++++++++++++------- cmd/http/main.go | 1 + 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index cb83a3a..e1035e9 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "path" + "path/filepath" "strconv" "syscall" @@ -36,23 +37,32 @@ func init() { func main() { config.LoadConfig() - var dbDir string + var connStr string var resourceDir string var size uint var database string var engineDebug bool var host string var port uint - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + var err error + flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&connStr, "c", ".", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - logg.Infof("start command", "build", build, "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) + if connStr == "." { + connStr, err = filepath.Abs(".state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) + os.Exit(1) + } + } + + logg.Infof("start command", "build", build, "dbdir", connStr, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) @@ -69,14 +79,14 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) - rs, err := menuStorageService.GetResource(ctx) + menuStorageService := storage.NewMenuStorageService(resourceDir) + err = menuStorageService.SetConn(connStr) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } - err = menuStorageService.EnsureDbDir() + rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/http/main.go b/cmd/http/main.go index dec4e64..070cf79 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -47,6 +47,7 @@ func main() { var err error flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") + flag.StringVar(&connStr, "c", ".", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") From dc61d055844fffa71cab6c96893436f14360fa68 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 12:16:28 +0000 Subject: [PATCH 18/81] WIP revert connstr gdbm to dir only, add file as table spec --- internal/storage/storageservice.go | 8 +++++-- .../testutil/{TestEngine.go => engine.go} | 22 +++++++++++-------- internal/testutil/engine_test.go | 15 +++++++++++++ menutraversal_test/menu_traversal_test.go | 11 +++++++--- 4 files changed, 42 insertions(+), 14 deletions(-) rename internal/testutil/{TestEngine.go => engine.go} (82%) create mode 100644 internal/testutil/engine_test.go diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 6a42bac..e14b7a4 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path" "git.defalsify.org/vise.git/db" fsdb "git.defalsify.org/vise.git/db/fs" @@ -66,7 +67,7 @@ func (ms *MenuStorageService) SetConn(connStr string) error { return nil } -func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) { +func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) { var newDb db.Db var err error // database, ok := ctx.Value("Database").(string) @@ -79,6 +80,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D } + connStr := ms.conn.String() dbTyp := ms.conn.DbType() if dbTyp == DBTYPE_POSTGRES { newDb = postgres.NewPgDb() @@ -87,11 +89,13 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D if err != nil { return nil, err } + connStr = path.Join(connStr, section) newDb = gdbmstorage.NewThreadGdbmDb() } else { return nil, fmt.Errorf("unsupported connection string: %s", ms.conn.String()) } - err = newDb.Connect(ctx, ms.conn.String()) + logg.DebugCtxf(ctx, "connecting to db", "conn", connStr) + err = newDb.Connect(ctx, connStr) if err != nil { return nil, err } diff --git a/internal/testutil/TestEngine.go b/internal/testutil/engine.go similarity index 82% rename from internal/testutil/TestEngine.go rename to internal/testutil/engine.go index 63e9e84..4b02e32 100644 --- a/internal/testutil/TestEngine.go +++ b/internal/testutil/engine.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path" + "path/filepath" "time" "git.defalsify.org/vise.git/engine" @@ -27,7 +28,6 @@ var ( func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) - ctx = context.WithValue(ctx, "Database", "gdbm") pfp := path.Join(scriptDir, "pp.csv") var eventChannel = make(chan bool) @@ -39,39 +39,43 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { FlagCount: uint32(128), } - dbDir := ".test_state" + connStr, err := filepath.Abs(".test_state/state.gdbm") + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) + } resourceDir := scriptDir - //menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + //menuStorageService := storage.NewMenuStorageService(connStr, resourceDir) menuStorageService := storage.NewMenuStorageService(resourceDir) //err := menuStorageService.EnsureDbDir() - err := menuStorageService.SetConn(dbDir) + err = menuStorageService.SetConn(connStr) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "conn error: %v", err) os.Exit(1) } rs, err := menuStorageService.GetResource(ctx) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "resource error: %v", err) os.Exit(1) } pe, err := menuStorageService.GetPersister(ctx) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "persister error: %v", err) os.Exit(1) } userDataStore, err := menuStorageService.GetUserdataDb(ctx) if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "userdb error: %v", err) os.Exit(1) } dbResource, ok := rs.(*resource.DbResource) if !ok { - fmt.Fprintf(os.Stderr, err.Error()) + fmt.Fprintf(os.Stderr, "dbresource cast error") os.Exit(1) } diff --git a/internal/testutil/engine_test.go b/internal/testutil/engine_test.go new file mode 100644 index 0000000..f747468 --- /dev/null +++ b/internal/testutil/engine_test.go @@ -0,0 +1,15 @@ +package testutil + +import ( + "testing" +) + +func TestCreateEngine(t *testing.T) { + o, clean, eventC := TestEngine("foo") + defer clean() + defer func() { + <-eventC + close(eventC) + }() + _ = o +} diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index 6b6b3da..52e2273 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -7,6 +7,7 @@ import ( "log" "math/rand" "os" + "path/filepath" "regexp" "testing" @@ -17,7 +18,6 @@ import ( var ( testData = driver.ReadData() - testStore = ".test_state" sessionID string src = rand.NewSource(42) g = rand.New(src) @@ -25,6 +25,11 @@ var ( var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests") +func testStore() string { + v, _ := filepath.Abs(".test_state/state.gdbm") + return v +} + func GenerateSessionId() string { uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g)) v, err := uu.NewV4() @@ -81,8 +86,8 @@ func extractSendAmount(response []byte) string { func TestMain(m *testing.M) { sessionID = GenerateSessionId() defer func() { - if err := os.RemoveAll(testStore); err != nil { - log.Fatalf("Failed to delete state store %s: %v", testStore, err) + if err := os.RemoveAll(testStore()); err != nil { + log.Fatalf("Failed to delete state store %s: %v", testStore(), err) } }() m.Run() From 2992f7ae8e9983c51aeb1328197270db395b5b1a Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 13:19:30 +0000 Subject: [PATCH 19/81] Update executables with new conn str --- cmd/africastalking/main.go | 7 ++++--- cmd/async/main.go | 6 +++--- cmd/http/main.go | 6 +++--- cmd/main.go | 6 +++--- internal/storage/parse.go | 3 --- internal/storage/storageservice.go | 1 - 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index e1035e9..8820215 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -34,6 +34,7 @@ var ( func init() { initializers.LoadEnvVariables() } + func main() { config.LoadConfig() @@ -47,15 +48,15 @@ func main() { var err error flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".", "connection string") + flag.StringVar(&connStr, "c", ".state", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") + if connStr == ".state" { + connStr, err = filepath.Abs(connStr) if err != nil { fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) os.Exit(1) diff --git a/cmd/async/main.go b/cmd/async/main.go index a9a0b1e..8b15031 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -59,15 +59,15 @@ func main() { flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".", "connection string") + flag.StringVar(&connStr, "c", ".state", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") + if connStr == ".state" { + connStr, err = filepath.Abs(connStr) if err != nil { fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) os.Exit(1) diff --git a/cmd/http/main.go b/cmd/http/main.go index 070cf79..f42d4d9 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -47,15 +47,15 @@ func main() { var err error flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".", "connection string") + flag.StringVar(&connStr, "c", ".state", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") + if connStr == ".state" { + connStr, err = filepath.Abs(connStr) if err != nil { fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) os.Exit(1) diff --git a/cmd/main.go b/cmd/main.go index c45615b..2506012 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,13 +39,13 @@ func main() { var err error flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&connStr, "c", ".", "connection string") + flag.StringVar(&connStr, "c", ".state", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") + if connStr == ".state" { + connStr, err = filepath.Abs(connStr) if err != nil { fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) os.Exit(1) diff --git a/internal/storage/parse.go b/internal/storage/parse.go index d4bc564..a97b938 100644 --- a/internal/storage/parse.go +++ b/internal/storage/parse.go @@ -40,9 +40,6 @@ func probeGdbm(s string) (string, bool) { if !path.IsAbs(s) { return "", false } - if path.Ext(s) != ".gdbm" { - return "", false - } s = path.Clean(s) return s, true } diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index e14b7a4..68e6916 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -28,7 +28,6 @@ type StorageService interface { } type MenuStorageService struct { - //dbDir string conn connData resourceDir string resourceStore db.Db From a4d6cef9c0fa3e077b6691d4ced7085a7e7f52c0 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 13:21:31 +0000 Subject: [PATCH 20/81] Remove commented code --- internal/storage/storageservice.go | 4 ---- internal/testutil/engine.go | 2 -- 2 files changed, 6 deletions(-) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 68e6916..c9c138f 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -69,10 +69,6 @@ func (ms *MenuStorageService) SetConn(connStr string) error { func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) { var newDb db.Db var err error -// database, ok := ctx.Value("Database").(string) -// if !ok { -// return nil, fmt.Errorf("failed to select the database") -// } if existingDb != nil { return existingDb, nil diff --git a/internal/testutil/engine.go b/internal/testutil/engine.go index 4b02e32..4234773 100644 --- a/internal/testutil/engine.go +++ b/internal/testutil/engine.go @@ -45,10 +45,8 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { os.Exit(1) } resourceDir := scriptDir - //menuStorageService := storage.NewMenuStorageService(connStr, resourceDir) menuStorageService := storage.NewMenuStorageService(resourceDir) - //err := menuStorageService.EnsureDbDir() err = menuStorageService.SetConn(connStr) if err != nil { fmt.Fprintf(os.Stderr, "conn error: %v", err) From f61e65f4fe38380978fc7961a86ece26c50c2eac Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 13:24:14 +0000 Subject: [PATCH 21/81] Rename accountservice testservice source file --- .../testservice/{TestAccountService.go => accountservice.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/testutil/testservice/{TestAccountService.go => accountservice.go} (100%) diff --git a/internal/testutil/testservice/TestAccountService.go b/internal/testutil/testservice/accountservice.go similarity index 100% rename from internal/testutil/testservice/TestAccountService.go rename to internal/testutil/testservice/accountservice.go From 450dfa02cc016b525d743a864530581e0372acda Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 20:36:18 +0000 Subject: [PATCH 22/81] Refactor to use conndata as menustorageservice conn arg --- cmd/africastalking/main.go | 16 ++++++++-------- cmd/async/main.go | 20 ++++++++++---------- cmd/http/main.go | 24 ++++++++++-------------- cmd/main.go | 23 +++++++++-------------- common/storage.go | 12 ++++++------ config/config.go | 15 +++++++++++++-- devtools/gen/main.go | 26 +++++++++++--------------- devtools/store/main.go | 16 ++++++++-------- internal/storage/parse.go | 10 +++++----- internal/storage/parse_test.go | 25 ++++++++----------------- internal/storage/storageservice.go | 29 ++++------------------------- internal/testutil/engine.go | 7 ++++++- 12 files changed, 98 insertions(+), 125 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 8820215..6035fc6 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -8,7 +8,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "strconv" "syscall" @@ -55,12 +54,13 @@ func main() { flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == ".state" { - connStr, err = filepath.Abs(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } logg.Infof("start command", "build", build, "dbdir", connStr, "resourcedir", resourceDir, "outputsize", size) @@ -81,7 +81,7 @@ func main() { } menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) + menuStorageService = menuStorageService.WithConn(connData) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/async/main.go b/cmd/async/main.go index 8b15031..17ea2a1 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -7,7 +7,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "syscall" "git.defalsify.org/vise.git/engine" @@ -47,8 +46,8 @@ func (p *asyncRequestParser) GetInput(r any) ([]byte, error) { func main() { config.LoadConfig() - var sessionId string var connStr string + var sessionId string var resourceDir string var size uint var database string @@ -59,19 +58,20 @@ func main() { flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".state", "connection string") + flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == ".state" { - connStr, err = filepath.Abs(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) @@ -92,7 +92,7 @@ func main() { } menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) + menuStorageService = menuStorageService.WithConn(connData) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/http/main.go b/cmd/http/main.go index f42d4d9..69a9c90 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -8,7 +8,6 @@ import ( "os" "os/signal" "path" - "path/filepath" "strconv" "syscall" @@ -47,19 +46,20 @@ func main() { var err error flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".state", "connection string") + flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() - if connStr == ".state" { - connStr, err = filepath.Abs(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size) @@ -80,12 +80,8 @@ func main() { } menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } - + menuStorageService = menuStorageService.WithConn(connData) + rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) diff --git a/cmd/main.go b/cmd/main.go index 2506012..a475dbe 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path" - "path/filepath" "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" @@ -39,17 +38,18 @@ func main() { var err error flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&connStr, "c", ".state", "connection string") + flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.Parse() - if connStr == ".state" { - connStr, err = filepath.Abs(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } logg.Infof("start command", "connstr", connStr, "outputsize", size) @@ -69,12 +69,7 @@ func main() { resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(resourceDir) - - err = menuStorageService.SetConn(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } + menuStorageService = menuStorageService.WithConn(connData) rs, err := menuStorageService.GetResource(ctx) if err != nil { diff --git a/common/storage.go b/common/storage.go index c8a4ff2..6ab5211 100644 --- a/common/storage.go +++ b/common/storage.go @@ -23,19 +23,19 @@ type StorageServices interface { GetPersister(ctx context.Context) (*persist.Persister, error) GetUserdataDb(ctx context.Context) (db.Db, error) GetResource(ctx context.Context) (resource.Resource, error) - SetConn(connStr string) error + SetConn(conn storage.ConnData) } type StorageService struct { svc *storage.MenuStorageService } -func NewStorageService(connStr string) (*StorageService, error) { +func NewStorageService(conn storage.ConnData) (*StorageService, error) { svc := &StorageService{ svc: storage.NewMenuStorageService(""), } - err := svc.SetConn(connStr) - return svc, err + svc.SetConn(conn) + return svc, nil } func(ss *StorageService) GetPersister(ctx context.Context) (*persist.Persister, error) { @@ -50,6 +50,6 @@ func(ss *StorageService) GetResource(ctx context.Context) (resource.Resource, er return nil, errors.New("not implemented") } -func(ss *StorageService) SetConn(connStr string) error { - return ss.svc.SetConn(connStr) +func(ss *StorageService) SetConn(conn storage.ConnData) { + ss.svc = ss.svc.WithConn(conn) } diff --git a/config/config.go b/config/config.go index 3a8e8ed..6f2b225 100644 --- a/config/config.go +++ b/config/config.go @@ -34,6 +34,7 @@ var ( VoucherTransfersURL string VoucherDataURL string CheckAliasURL string + DbConn string ) func setBase() error { @@ -43,14 +44,20 @@ func setBase() error { dataURLBase = initializers.GetEnv("DATA_URL_BASE", "http://localhost:5006") BearerToken = initializers.GetEnv("BEARER_TOKEN", "") - _, err = url.JoinPath(custodialURLBase, "/foo") + _, err = url.Parse(custodialURLBase) if err != nil { return err } - _, err = url.JoinPath(dataURLBase, "/bar") + _, err = url.Parse(dataURLBase) if err != nil { return err } + + return nil +} + +func setConn() error { + DbConn = initializers.GetEnv("DB_CONN", "") return nil } @@ -60,6 +67,10 @@ func LoadConfig() error { if err != nil { return err } + err = setConn() + if err != nil { + return err + } CreateAccountURL, _ = url.JoinPath(custodialURLBase, createAccountPath) TrackStatusURL, _ = url.JoinPath(custodialURLBase, trackStatusPath) BalanceURL, _ = url.JoinPath(custodialURLBase, balancePathPrefix) diff --git a/devtools/gen/main.go b/devtools/gen/main.go index 448cc16..272fd9c 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "path" - "path/filepath" "git.defalsify.org/vise.git/logging" "git.grassecon.net/urdt/ussd/config" @@ -36,19 +35,20 @@ func main() { var err error flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&connStr, "c", ".", "connection string") + flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } - logg.Infof("start command", "connstr", connStr) + logg.Infof("start command", "conn", connData) ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) @@ -56,12 +56,8 @@ func main() { resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) - if err != nil { - fmt.Fprintf(os.Stderr, "connection string error: %v", err) - os.Exit(1) - } - + menuStorageService = menuStorageService.WithConn(connData) + store, err := menuStorageService.GetUserdataDb(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) diff --git a/devtools/store/main.go b/devtools/store/main.go index ed54ba3..da988ad 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "path" - "path/filepath" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -40,12 +39,13 @@ func main() { flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.Parse() - if connStr == "." { - connStr, err = filepath.Abs(".state/state.gdbm") - if err != nil { - fmt.Fprintf(os.Stderr, "auto connstr generate error: %v", err) - os.Exit(1) - } + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) } ctx := context.Background() @@ -54,7 +54,7 @@ func main() { resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) + menuStorageService = menuStorageService.WithConn(connData) if err != nil { fmt.Fprintf(os.Stderr, "connection string error: %v", err) os.Exit(1) diff --git a/internal/storage/parse.go b/internal/storage/parse.go index a97b938..bb25627 100644 --- a/internal/storage/parse.go +++ b/internal/storage/parse.go @@ -12,16 +12,16 @@ const ( DBTYPE_POSTGRES ) -type connData struct { +type ConnData struct { typ int str string } -func (cd *connData) DbType() int { +func (cd *ConnData) DbType() int { return cd.typ } -func (cd *connData) String() string { +func (cd *ConnData) String() string { return cd.str } @@ -44,8 +44,8 @@ func probeGdbm(s string) (string, bool) { return s, true } -func toConnData(connStr string) (connData, error) { - var o connData +func ToConnData(connStr string) (ConnData, error) { + var o ConnData if connStr == "" { return o, nil diff --git a/internal/storage/parse_test.go b/internal/storage/parse_test.go index d4b8ecb..e18e57c 100644 --- a/internal/storage/parse_test.go +++ b/internal/storage/parse_test.go @@ -5,32 +5,23 @@ import ( ) func TestParseConnStr(t *testing.T) { - svc := NewMenuStorageService("") - err := svc.SetConn("postgres://foo:bar@localhost:5432/baz") + _, err := ToConnData("postgres://foo:bar@localhost:5432/baz") if err != nil { t.Fatal(err) } - err = svc.SetConn("/foo/bar/baz.gdbm") + _, err = ToConnData("/foo/bar") if err != nil { t.Fatal(err) } - err = svc.SetConn("foo/bar/baz.gdbm") + _, err = ToConnData("/foo/bar/") + if err != nil { + t.Fatal(err) + } + _, err = ToConnData("foo/bar") if err == nil { t.Fatalf("expected error") } - err = svc.SetConn("http://foo/bar") - if err == nil { - t.Fatalf("expected error") - } - err = svc.SetConn("foo/bar/baz.txt") - if err == nil { - t.Fatalf("expected error") - } - err = svc.SetConn("/foo/bar") - if err == nil { - t.Fatalf("expected error") - } - err = svc.SetConn("foo/bar") + _, err = ToConnData("http://foo/bar") if err == nil { t.Fatalf("expected error") } diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index c9c138f..e3429bc 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -12,7 +12,6 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" - "git.grassecon.net/urdt/ussd/initializers" gdbmstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm" ) @@ -28,42 +27,22 @@ type StorageService interface { } type MenuStorageService struct { - conn connData + conn ConnData resourceDir string resourceStore db.Db stateStore db.Db userDataStore db.Db } -func buildConnStr() string { - host := initializers.GetEnv("DB_HOST", "localhost") - user := initializers.GetEnv("DB_USER", "postgres") - password := initializers.GetEnv("DB_PASSWORD", "") - dbName := initializers.GetEnv("DB_NAME", "") - port := initializers.GetEnv("DB_PORT", "5432") - - connString := fmt.Sprintf( - "postgres://%s:%s@%s:%s/%s", - user, password, host, port, dbName, - ) - logg.Debugf("pg conn string", "conn", connString) - - return connString -} - func NewMenuStorageService(resourceDir string) *MenuStorageService { return &MenuStorageService{ resourceDir: resourceDir, } } -func (ms *MenuStorageService) SetConn(connStr string) error { - o, err := toConnData(connStr) - if err != nil { - return err - } - ms.conn = o - return nil +func (ms *MenuStorageService) WithConn(conn ConnData) *MenuStorageService { + ms.conn = conn + return ms } func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) { diff --git a/internal/testutil/engine.go b/internal/testutil/engine.go index 4234773..3d804ae 100644 --- a/internal/testutil/engine.go +++ b/internal/testutil/engine.go @@ -44,10 +44,15 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) } + conn, err := storage.ToConnData(connStr) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr parse err: %v", err) + os.Exit(1) + } resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(resourceDir) - err = menuStorageService.SetConn(connStr) + menuStorageService.WithConn(conn) if err != nil { fmt.Fprintf(os.Stderr, "conn error: %v", err) os.Exit(1) From 89c21847b959d304c364f657249dee2c01c4b281 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 20:52:49 +0000 Subject: [PATCH 23/81] Improve error message --- internal/storage/storageservice.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index e3429bc..24d5eff 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -66,7 +66,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D connStr = path.Join(connStr, section) newDb = gdbmstorage.NewThreadGdbmDb() } else { - return nil, fmt.Errorf("unsupported connection string: %s", ms.conn.String()) + return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String()) } logg.DebugCtxf(ctx, "connecting to db", "conn", connStr) err = newDb.Connect(ctx, connStr) From 3a9f3fa373c7feb4295ad3e52896d1ba7389269e Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 20:54:51 +0000 Subject: [PATCH 24/81] Update env example --- .env.example | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index c636fa8..0a43945 100644 --- a/.env.example +++ b/.env.example @@ -6,13 +6,8 @@ HOST=127.0.0.1 AT_ENDPOINT=/ussd/africastalking #PostgreSQL -DB_HOST=localhost -DB_USER=postgres -DB_PASSWORD=strongpass -DB_NAME=urdt_ussd -DB_PORT=5432 -DB_SSLMODE=disable -DB_TIMEZONE=Africa/Nairobi +DB_CONN=postgres://postgres:strongpass@localhost:5432/urdt_ussd +#DB_TIMEZONE=Africa/Nairobi #External API Calls CUSTODIAL_URL_BASE=http://localhost:5003 From cc9760125aa9e47f1939fc90a6b50b0a2e8e1376 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 21:02:08 +0000 Subject: [PATCH 25/81] Remove unnecessary connection chain step --- .env.example | 1 + cmd/africastalking/main.go | 3 +-- cmd/async/main.go | 3 +-- cmd/http/main.go | 3 +-- cmd/main.go | 3 +-- common/storage.go | 7 +------ devtools/gen/main.go | 3 +-- devtools/store/main.go | 7 +------ internal/storage/storageservice.go | 8 ++------ internal/testutil/engine.go | 8 +------- 10 files changed, 11 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index 0a43945..dae37bc 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,7 @@ AT_ENDPOINT=/ussd/africastalking #PostgreSQL DB_CONN=postgres://postgres:strongpass@localhost:5432/urdt_ussd #DB_TIMEZONE=Africa/Nairobi +#DB_SCHEMA=vise #External API Calls CUSTODIAL_URL_BASE=http://localhost:5003 diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 6035fc6..0d2b8ec 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -80,8 +80,7 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/async/main.go b/cmd/async/main.go index 17ea2a1..65e681d 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -91,8 +91,7 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/cmd/http/main.go b/cmd/http/main.go index 69a9c90..2f043e6 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -79,8 +79,7 @@ func main() { cfg.EngineDebug = true } - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) rs, err := menuStorageService.GetResource(ctx) if err != nil { diff --git a/cmd/main.go b/cmd/main.go index a475dbe..ad4535b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -68,8 +68,7 @@ func main() { } resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) rs, err := menuStorageService.GetResource(ctx) if err != nil { diff --git a/common/storage.go b/common/storage.go index 6ab5211..c7e478b 100644 --- a/common/storage.go +++ b/common/storage.go @@ -32,9 +32,8 @@ type StorageService struct { func NewStorageService(conn storage.ConnData) (*StorageService, error) { svc := &StorageService{ - svc: storage.NewMenuStorageService(""), + svc: storage.NewMenuStorageService(conn, ""), } - svc.SetConn(conn) return svc, nil } @@ -49,7 +48,3 @@ func(ss *StorageService) GetUserdataDb(ctx context.Context) (db.Db, error) { func(ss *StorageService) GetResource(ctx context.Context) (resource.Resource, error) { return nil, errors.New("not implemented") } - -func(ss *StorageService) SetConn(conn storage.ConnData) { - ss.svc = ss.svc.WithConn(conn) -} diff --git a/devtools/gen/main.go b/devtools/gen/main.go index 272fd9c..749f340 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -55,8 +55,7 @@ func main() { ctx = context.WithValue(ctx, "Database", database) resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) store, err := menuStorageService.GetUserdataDb(ctx) if err != nil { diff --git a/devtools/store/main.go b/devtools/store/main.go index da988ad..f7f76c6 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -53,12 +53,7 @@ func main() { ctx = context.WithValue(ctx, "Database", database) resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(resourceDir) - menuStorageService = menuStorageService.WithConn(connData) - if err != nil { - fmt.Fprintf(os.Stderr, "connection string error: %v", err) - os.Exit(1) - } + menuStorageService := storage.NewMenuStorageService(connData, resourceDir) store, err := menuStorageService.GetUserdataDb(ctx) if err != nil { diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 24d5eff..9148d7c 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -34,17 +34,13 @@ type MenuStorageService struct { userDataStore db.Db } -func NewMenuStorageService(resourceDir string) *MenuStorageService { +func NewMenuStorageService(conn ConnData, resourceDir string) *MenuStorageService { return &MenuStorageService{ + conn: conn, resourceDir: resourceDir, } } -func (ms *MenuStorageService) WithConn(conn ConnData) *MenuStorageService { - ms.conn = conn - return ms -} - func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string) (db.Db, error) { var newDb db.Db var err error diff --git a/internal/testutil/engine.go b/internal/testutil/engine.go index 3d804ae..2372ce9 100644 --- a/internal/testutil/engine.go +++ b/internal/testutil/engine.go @@ -50,13 +50,7 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { os.Exit(1) } resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(resourceDir) - - menuStorageService.WithConn(conn) - if err != nil { - fmt.Fprintf(os.Stderr, "conn error: %v", err) - os.Exit(1) - } + menuStorageService := storage.NewMenuStorageService(conn, resourceDir) rs, err := menuStorageService.GetResource(ctx) if err != nil { From 51b6fc0dde431b6489e5ceb26a6e0cc2c66c6b51 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 4 Jan 2025 21:13:39 +0000 Subject: [PATCH 26/81] Remove unused methods in storage interfaces, improve logs --- cmd/africastalking/main.go | 4 ++-- cmd/async/main.go | 2 +- cmd/http/main.go | 2 +- cmd/main.go | 2 +- common/storage.go | 1 - devtools/store/main.go | 2 ++ internal/storage/storageservice.go | 1 - 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 0d2b8ec..40c3609 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -47,7 +47,7 @@ func main() { var err error flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.StringVar(&connStr, "c", ".state", "connection string") + flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -63,7 +63,7 @@ func main() { os.Exit(1) } - logg.Infof("start command", "build", build, "dbdir", connStr, "resourcedir", resourceDir, "outputsize", size) + logg.Infof("start command", "build", build, "conn", connData, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) diff --git a/cmd/async/main.go b/cmd/async/main.go index 65e681d..b0c7caa 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -74,7 +74,7 @@ func main() { os.Exit(1) } - logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) + logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) diff --git a/cmd/http/main.go b/cmd/http/main.go index 2f043e6..d744afc 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -62,7 +62,7 @@ func main() { os.Exit(1) } - logg.Infof("start command", "connstr", connStr, "resourcedir", resourceDir, "outputsize", size) + logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) diff --git a/cmd/main.go b/cmd/main.go index ad4535b..ac680fc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -52,7 +52,7 @@ func main() { os.Exit(1) } - logg.Infof("start command", "connstr", connStr, "outputsize", size) + logg.Infof("start command", "conn", connData, "outputsize", size) ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) diff --git a/common/storage.go b/common/storage.go index c7e478b..2960578 100644 --- a/common/storage.go +++ b/common/storage.go @@ -23,7 +23,6 @@ type StorageServices interface { GetPersister(ctx context.Context) (*persist.Persister, error) GetUserdataDb(ctx context.Context) (db.Db, error) GetResource(ctx context.Context) (resource.Resource, error) - SetConn(conn storage.ConnData) } type StorageService struct { diff --git a/devtools/store/main.go b/devtools/store/main.go index f7f76c6..6915f74 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -48,6 +48,8 @@ func main() { os.Exit(1) } + logg.Infof("start command", "conn", connData) + ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 9148d7c..83ce051 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -23,7 +23,6 @@ type StorageService interface { GetPersister(ctx context.Context) (*persist.Persister, error) GetUserdataDb(ctx context.Context) db.Db GetResource(ctx context.Context) (resource.Resource, error) - SetConn(connStr string) error } type MenuStorageService struct { From bb4037e73f6c1c3f4b8a65e21298b5c6d399c2ad Mon Sep 17 00:00:00 2001 From: lash Date: Sun, 5 Jan 2025 21:25:09 +0000 Subject: [PATCH 27/81] Add languages env example --- .env.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.example b/.env.example index c636fa8..f60c1aa 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,7 @@ DB_TIMEZONE=Africa/Nairobi CUSTODIAL_URL_BASE=http://localhost:5003 BEARER_TOKEN=eyJeSIsInRcCI6IkpXVCJ.yJwdWJsaWNLZXkiOiIwrrrrrr DATA_URL_BASE=http://localhost:5006 + +#Language +DEFAULT_LANGUAGE=eng +LANGUAGES=eng, swa From 974af6b2a75a90bd6c05ed66d9632edd5c1ceae1 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 08:50:53 +0300 Subject: [PATCH 28/81] pass context as an argument --- cmd/africastalking/main.go | 4 +-- cmd/async/main.go | 6 ++-- internal/handlers/single.go | 30 +++++++++---------- internal/http/parse.go | 5 ++-- internal/http/server_test.go | 3 +- .../mocks/httpmocks/requestparsermock.go | 4 ++- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 0019239..dfcaca1 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -121,9 +121,7 @@ func main() { } defer stateStore.Close() - rp := &at.ATRequestParser{ - Context: ctx, - } + rp := &at.ATRequestParser{} bsh := handlers.NewBaseSessionHandler(cfg, rs, stateStore, userdataStore, rp, hl) sh := httpserver.NewATSessionHandler(bsh) diff --git a/cmd/async/main.go b/cmd/async/main.go index 9cd04b3..bf23d9f 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -21,8 +21,8 @@ import ( ) var ( - logg = logging.NewVanilla() - scriptDir = path.Join("services", "registration") + logg = logging.NewVanilla() + scriptDir = path.Join("services", "registration") menuSeparator = ": " ) @@ -35,7 +35,7 @@ type asyncRequestParser struct { input []byte } -func (p *asyncRequestParser) GetSessionId(r any) (string, error) { +func (p *asyncRequestParser) GetSessionId(ctx context.Context, r any) (string, error) { return p.sessionId, nil } diff --git a/internal/handlers/single.go b/internal/handlers/single.go index 6929617..1b11a64 100644 --- a/internal/handlers/single.go +++ b/internal/handlers/single.go @@ -6,9 +6,9 @@ import ( "io" "git.defalsify.org/vise.git/engine" - "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/logging" + "git.defalsify.org/vise.git/persist" + "git.defalsify.org/vise.git/resource" "git.grassecon.net/urdt/ussd/internal/storage" ) @@ -20,33 +20,33 @@ var ( var ( ErrInvalidRequest = errors.New("invalid request for context") ErrSessionMissing = errors.New("missing session") - ErrInvalidInput = errors.New("invalid input") - ErrStorage = errors.New("storage retrieval fail") - ErrEngineType = errors.New("incompatible engine") - ErrEngineInit = errors.New("engine init fail") - ErrEngineExec = errors.New("engine exec fail") + ErrInvalidInput = errors.New("invalid input") + ErrStorage = errors.New("storage retrieval fail") + ErrEngineType = errors.New("incompatible engine") + ErrEngineInit = errors.New("engine init fail") + ErrEngineExec = errors.New("engine exec fail") ) type RequestSession struct { - Ctx context.Context - Config engine.Config - Engine engine.Engine - Input []byte - Storage *storage.Storage - Writer io.Writer + Ctx context.Context + Config engine.Config + Engine engine.Engine + Input []byte + Storage *storage.Storage + Writer io.Writer Continue bool } // TODO: seems like can remove this. type RequestParser interface { - GetSessionId(rq any) (string, error) + GetSessionId(context context.Context, rq any) (string, error) GetInput(rq any) ([]byte, error) } type RequestHandler interface { GetConfig() engine.Config GetRequestParser() RequestParser - GetEngine(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine + GetEngine(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine Process(rs RequestSession) (RequestSession, error) Output(rs RequestSession) (RequestSession, error) Reset(rs RequestSession) (RequestSession, error) diff --git a/internal/http/parse.go b/internal/http/parse.go index ec8e00b..b4e784d 100644 --- a/internal/http/parse.go +++ b/internal/http/parse.go @@ -1,6 +1,7 @@ package http import ( + "context" "io/ioutil" "net/http" @@ -10,7 +11,7 @@ import ( type DefaultRequestParser struct { } -func (rp *DefaultRequestParser) GetSessionId(rq any) (string, error) { +func (rp *DefaultRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) { rqv, ok := rq.(*http.Request) if !ok { return "", handlers.ErrInvalidRequest @@ -34,5 +35,3 @@ func (rp *DefaultRequestParser) GetInput(rq any) ([]byte, error) { } return v, nil } - - diff --git a/internal/http/server_test.go b/internal/http/server_test.go index a46f98e..23afd5d 100644 --- a/internal/http/server_test.go +++ b/internal/http/server_test.go @@ -2,6 +2,7 @@ package http import ( "bytes" + "context" "errors" "net/http" "net/http/httptest" @@ -161,7 +162,7 @@ func TestDefaultRequestParser_GetSessionId(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - id, err := parser.GetSessionId(tt.request) + id, err := parser.GetSessionId(context.Background(),tt.request) if id != tt.expectedID { t.Errorf("Expected session ID %s, got %s", tt.expectedID, id) diff --git a/internal/testutil/mocks/httpmocks/requestparsermock.go b/internal/testutil/mocks/httpmocks/requestparsermock.go index 54b16bf..3c19e12 100644 --- a/internal/testutil/mocks/httpmocks/requestparsermock.go +++ b/internal/testutil/mocks/httpmocks/requestparsermock.go @@ -1,12 +1,14 @@ package httpmocks +import "context" + // MockRequestParser implements the handlers.RequestParser interface for testing type MockRequestParser struct { GetSessionIdFunc func(any) (string, error) GetInputFunc func(any) ([]byte, error) } -func (m *MockRequestParser) GetSessionId(rq any) (string, error) { +func (m *MockRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) { return m.GetSessionIdFunc(rq) } From c69d3896f1a932af9be30390fbbea514817893b9 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 08:52:53 +0300 Subject: [PATCH 29/81] pass context as an argument,rename context keys --- internal/handlers/ussd/menuhandler.go | 4 ++-- internal/http/at/parse.go | 8 +++----- internal/http/at/server.go | 6 +++--- internal/http/server.go | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index 095d77b..dfdbd02 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -28,7 +28,7 @@ import ( ) var ( - logg = logging.NewVanilla().WithDomain("ussdmenuhandler").WithContextKey("session-id") + logg = logging.NewVanilla().WithDomain("ussdmenuhandler").WithContextKey("SessionId") scriptDir = path.Join("services", "registration") translationDir = path.Join(scriptDir, "locale") ) @@ -124,7 +124,7 @@ func (h *Handlers) Init(ctx context.Context, sym string, input []byte) (resource sessionId, ok := ctx.Value("SessionId").(string) if ok { - context.WithValue(ctx, "session-id", sessionId) + ctx = context.WithValue(ctx, "SessionId", sessionId) } flag_admin_privilege, _ := h.flagManager.GetFlag("flag_admin_privilege") diff --git a/internal/http/at/parse.go b/internal/http/at/parse.go index d2696ed..5f27d50 100644 --- a/internal/http/at/parse.go +++ b/internal/http/at/parse.go @@ -15,16 +15,14 @@ import ( ) type ATRequestParser struct { - Context context.Context } -func (arp *ATRequestParser) GetSessionId(rq any) (string, error) { +func (arp *ATRequestParser) GetSessionId(ctx context.Context, rq any) (string, error) { rqv, ok := rq.(*http.Request) if !ok { logg.Warnf("got an invalid request", "req", rq) return "", handlers.ErrInvalidRequest } - // Capture body (if any) for logging body, err := io.ReadAll(rqv.Body) if err != nil { @@ -43,9 +41,9 @@ func (arp *ATRequestParser) GetSessionId(rq any) (string, error) { decodedStr := string(logBytes) sessionId, err := extractATSessionId(decodedStr) if err != nil { - context.WithValue(arp.Context, "at-session-id", sessionId) + ctx = context.WithValue(ctx, "AT-SessionId", sessionId) } - logg.Debugf("Received request:", decodedStr) + logg.DebugCtxf(ctx, "Received request:", decodedStr) } if err := rqv.ParseForm(); err != nil { diff --git a/internal/http/at/server.go b/internal/http/at/server.go index 705ff76..92f76bc 100644 --- a/internal/http/at/server.go +++ b/internal/http/at/server.go @@ -10,7 +10,7 @@ import ( ) var ( - logg = logging.NewVanilla().WithDomain("atserver") + logg = logging.NewVanilla().WithDomain("atserver").WithContextKey("AT-SessionId") ) type ATSessionHandler struct { @@ -34,7 +34,7 @@ func (ash *ATSessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) rp := ash.GetRequestParser() cfg := ash.GetConfig() - cfg.SessionId, err = rp.GetSessionId(req) + cfg.SessionId, err = rp.GetSessionId(req.Context(), req) if err != nil { logg.ErrorCtxf(rqs.Ctx, "", "header processing error", err) ash.WriteError(w, 400, err) @@ -48,7 +48,7 @@ func (ash *ATSessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) return } - rqs, err = ash.Process(rqs) + rqs, err = ash.Process(rqs) switch err { case nil: // set code to 200 if no err code = 200 diff --git a/internal/http/server.go b/internal/http/server.go index 9cadfa3..92855b9 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -10,7 +10,7 @@ import ( ) var ( - logg = logging.NewVanilla().WithDomain("httpserver") + logg = logging.NewVanilla().WithDomain("httpserver").WithContextKey("at-session-id") ) type SessionHandler struct { @@ -46,7 +46,7 @@ func (f *SessionHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { rp := f.GetRequestParser() cfg := f.GetConfig() - cfg.SessionId, err = rp.GetSessionId(req) + cfg.SessionId, err = rp.GetSessionId(req.Context(), req) if err != nil { logg.ErrorCtxf(rqs.Ctx, "", "header processing error", err) f.WriteError(w, 400, err) From 362eb209efd81973045abf9deaa1b9e7370c6848 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 09:54:28 +0300 Subject: [PATCH 30/81] add SessionId to context key --- internal/http/at/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/at/server.go b/internal/http/at/server.go index 92f76bc..3399dd5 100644 --- a/internal/http/at/server.go +++ b/internal/http/at/server.go @@ -10,7 +10,7 @@ import ( ) var ( - logg = logging.NewVanilla().WithDomain("atserver").WithContextKey("AT-SessionId") + logg = logging.NewVanilla().WithDomain("atserver").WithContextKey("SessionId").WithContextKey("AT-SessionId") ) type ATSessionHandler struct { From 44570e20ef5fbde511d79f9b4a3f694478f14dcd Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 09:59:47 +0300 Subject: [PATCH 31/81] remove unused context key :- at-session-id --- internal/http/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/http/server.go b/internal/http/server.go index 92855b9..0a2533e 100644 --- a/internal/http/server.go +++ b/internal/http/server.go @@ -10,7 +10,7 @@ import ( ) var ( - logg = logging.NewVanilla().WithDomain("httpserver").WithContextKey("at-session-id") + logg = logging.NewVanilla().WithDomain("httpserver") ) type SessionHandler struct { From 2024cc96e2123f4721ace79fd0987123ba3100c6 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 6 Jan 2025 07:22:58 +0000 Subject: [PATCH 32/81] Bring up-to-date with refactor --- internal/ssh/keystore.go | 3 ++- internal/ssh/ssh.go | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/ssh/keystore.go b/internal/ssh/keystore.go index b80727b..206d684 100644 --- a/internal/ssh/keystore.go +++ b/internal/ssh/keystore.go @@ -11,6 +11,7 @@ import ( "git.defalsify.org/vise.git/db" "git.grassecon.net/urdt/ussd/internal/storage" + dbstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm" ) type SshKeyStore struct { @@ -20,7 +21,7 @@ type SshKeyStore struct { func NewSshKeyStore(ctx context.Context, dbDir string) (*SshKeyStore, error) { keyStore := &SshKeyStore{} keyStoreFile := path.Join(dbDir, "ssh_authorized_keys.gdbm") - keyStore.store = storage.NewThreadGdbmDb() + keyStore.store = dbstorage.NewThreadGdbmDb() err := keyStore.store.Connect(ctx, keyStoreFile) if err != nil { return nil, err diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go index 394f55f..1831ebe 100644 --- a/internal/ssh/ssh.go +++ b/internal/ssh/ssh.go @@ -19,6 +19,7 @@ import ( "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" + "git.grassecon.net/urdt/ussd/remote" ) var ( @@ -174,7 +175,7 @@ func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) { return nil, nil, err } - lhs, err := handlers.NewLocalHandlerService(s.FlagFile, true, dbResource, s.Cfg, rs) + lhs, err := handlers.NewLocalHandlerService(ctx, s.FlagFile, true, dbResource, s.Cfg, rs) lhs.SetDataStore(&userdatastore) lhs.SetPersister(pe) lhs.Cfg.SessionId = sessionId @@ -183,7 +184,9 @@ func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) { return nil, nil, err } - hl, err := lhs.GetHandler() + // TODO: clear up why pointer here and by-value other cmds + accountService := &remote.AccountService{} + hl, err := lhs.GetHandler(accountService) if err != nil { return nil, nil, err } From d39740a09abd7dc8a7ba092d946cfcebcb7e0c9f Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 6 Jan 2025 07:41:24 +0000 Subject: [PATCH 33/81] Edit ssh cli help text --- cmd/ssh/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index 0227616..0b22216 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -37,8 +37,8 @@ func main() { flag.BoolVar(&engineDebug, "engine-debug", false, "use engine debug output") flag.BoolVar(&stateDebug, "state-debug", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") - flag.StringVar(&host, "h", "127.0.0.1", "http host") - flag.UintVar(&port, "p", 7122, "http port") + flag.StringVar(&host, "h", "127.0.0.1", "socket host") + flag.UintVar(&port, "p", 7122, "socket port") flag.Parse() sshKeyFile := flag.Arg(0) From 4836162f40bcd652b9658d786e0482b1df299d05 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Mon, 6 Jan 2025 10:51:20 +0300 Subject: [PATCH 34/81] ci: add ssh build --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3a5da7d..827dcee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ WORKDIR /build RUN echo "Building on $BUILDPLATFORM, building for $TARGETPLATFORM" RUN go mod download RUN go build -tags logtrace -o ussd-africastalking -ldflags="-X main.build=${BUILD} -s -w" cmd/africastalking/main.go +RUN go build -tags logtrace -o ussd-ssh -ldflags="-X main.build=${BUILD} -s -w" cmd/ssh/main.go FROM debian:bookworm-slim @@ -30,6 +31,7 @@ RUN apt-get clean && rm -rf /var/lib/apt/lists/* WORKDIR /service COPY --from=build /build/ussd-africastalking . +COPY --from=build /build/ussd-ssh . COPY --from=build /build/LICENSE . COPY --from=build /build/README.md . COPY --from=build /build/services ./services From a312ea5b84465af2d7b7d5ecb4a196f3133f27c2 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Mon, 6 Jan 2025 11:09:51 +0300 Subject: [PATCH 35/81] feat: inject build string in ssh binary, expose default ssh port --- Dockerfile | 1 + cmd/ssh/main.go | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 827dcee..d68733c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,5 +39,6 @@ COPY --from=build /build/.env.example . RUN mv .env.example .env EXPOSE 7123 +EXPOSE 7122 CMD ["./ussd-africastalking"] \ No newline at end of file diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index 0227616..ae046a5 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -4,9 +4,9 @@ import ( "context" "flag" "fmt" - "path" "os" "os/signal" + "path" "sync" "syscall" @@ -18,10 +18,12 @@ import ( ) var ( - wg sync.WaitGroup - keyStore db.Db + wg sync.WaitGroup + keyStore db.Db logg = logging.NewVanilla() scriptDir = path.Join("services", "registration") + + build = "dev" ) func main() { @@ -76,7 +78,7 @@ func main() { fmt.Fprintf(os.Stderr, "keystore file open error: %v", err) os.Exit(1) } - defer func () { + defer func() { logg.TraceCtxf(ctx, "shutdown auth key store reached") err = authKeyStore.Close() if err != nil { @@ -90,14 +92,14 @@ func main() { signal.Notify(cterm, os.Interrupt, syscall.SIGTERM) runner := &ssh.SshRunner{ - Cfg: cfg, - Debug: engineDebug, - FlagFile: pfp, - DbDir: dbDir, + Cfg: cfg, + Debug: engineDebug, + FlagFile: pfp, + DbDir: dbDir, ResourceDir: resourceDir, - SrvKeyFile: sshKeyFile, - Host: host, - Port: port, + SrvKeyFile: sshKeyFile, + Host: host, + Port: port, } go func() { select { @@ -109,7 +111,7 @@ func main() { if err != nil { logg.ErrorCtxf(ctx, "runner stop error", "err", err) } - + }() runner.Run(ctx, authKeyStore) } From 52fd1eced24fc499e5b9bf04151df7a78973f62f Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 6 Jan 2025 08:11:37 +0000 Subject: [PATCH 36/81] Enable env, config, db in ssh --- cmd/main.go | 3 ++- cmd/ssh/main.go | 28 +++++++++++++++++++++++----- internal/ssh/ssh.go | 35 +++++++++++++++-------------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ac680fc..01a352c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -35,8 +35,10 @@ func main() { var sessionId string var database string var engineDebug bool + var resourceDir string var err error + flag.StringVar(&resourceDir, "resourcedir", scriptDir, "resource dir") flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&connStr, "c", "", "connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") @@ -67,7 +69,6 @@ func main() { MenuSeparator: menuSeparator, } - resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(connData, resourceDir) rs, err := menuStorageService.GetResource(ctx) diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index 0b22216..58f2b0b 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -14,7 +14,10 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/ssh" + "git.grassecon.net/urdt/ussd/internal/storage" ) var ( @@ -24,7 +27,14 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + initializers.LoadEnvVariables() +} + func main() { + config.LoadConfig() + + var connStr string var dbDir string var resourceDir string var size uint @@ -32,17 +42,25 @@ func main() { var stateDebug bool var host string var port uint - flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") + flag.StringVar(&connStr, "c", "", "connection string") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") - flag.BoolVar(&engineDebug, "engine-debug", false, "use engine debug output") - flag.BoolVar(&stateDebug, "state-debug", false, "use engine debug output") + flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", "127.0.0.1", "socket host") flag.UintVar(&port, "p", 7122, "socket port") flag.Parse() + if connStr != "" { + connStr = config.DbConn + } + connData, err := storage.ToConnData(config.DbConn) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) + } + sshKeyFile := flag.Arg(0) - _, err := os.Stat(sshKeyFile) + _, err = os.Stat(sshKeyFile) if err != nil { fmt.Fprintf(os.Stderr, "cannot open ssh server private key file: %v\n", err) os.Exit(1) @@ -93,7 +111,7 @@ func main() { Cfg: cfg, Debug: engineDebug, FlagFile: pfp, - DbDir: dbDir, + Conn: connData, ResourceDir: resourceDir, SrvKeyFile: sshKeyFile, Host: host, diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go index 1831ebe..c3ec725 100644 --- a/internal/ssh/ssh.go +++ b/internal/ssh/ssh.go @@ -71,6 +71,20 @@ func(a *auther) Get(k []byte) (string, error) { return v, nil } +type SshRunner struct { + Ctx context.Context + Cfg engine.Config + FlagFile string + Conn storage.ConnData + ResourceDir string + Debug bool + SrvKeyFile string + Host string + Port uint + wg sync.WaitGroup + lst net.Listener +} + func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChannel, en engine.Engine) error { if ch == nil { return errors.New("nil channel") @@ -128,32 +142,13 @@ func(s *SshRunner) serve(ctx context.Context, sessionId string, ch ssh.NewChanne return nil } -type SshRunner struct { - Ctx context.Context - Cfg engine.Config - FlagFile string - DbDir string - ResourceDir string - Debug bool - SrvKeyFile string - Host string - Port uint - wg sync.WaitGroup - lst net.Listener -} - func(s *SshRunner) Stop() error { return s.lst.Close() } func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) { ctx := s.Ctx - menuStorageService := storage.NewMenuStorageService(s.DbDir, s.ResourceDir) - - err := menuStorageService.EnsureDbDir() - if err != nil { - return nil, nil, err - } + menuStorageService := storage.NewMenuStorageService(s.Conn, s.ResourceDir) rs, err := menuStorageService.GetResource(ctx) if err != nil { From 824d39908ba46ec25d181997f7bf7759c325f281 Mon Sep 17 00:00:00 2001 From: Mohammed Sohail Date: Mon, 6 Jan 2025 11:19:36 +0300 Subject: [PATCH 37/81] ci: fix missing ssh dir --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index a118f64..2c2b83b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ /** !/cmd/africastalking +!/cmd/ssh !/common !/config !/initializers From 599815c343e958234b88a2075670e368ad89a597 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 6 Jan 2025 09:00:41 +0000 Subject: [PATCH 38/81] Fix remaining conflict in cmd cli --- cmd/main.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8547994..4bf134b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -88,22 +88,12 @@ func main() { MenuSeparator: menuSeparator, } -<<<<<<< HEAD menuStorageService := storage.NewMenuStorageService(connData, resourceDir) -======= - resourceDir := scriptDir - menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir) + if gettextDir != "" { menuStorageService = menuStorageService.WithGettext(gettextDir, langs.Langs()) } - err = menuStorageService.EnsureDbDir() - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - os.Exit(1) - } ->>>>>>> master - rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) From 3ee15497a5bc080f424feb0008933bb53143c8de Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Mon, 6 Jan 2025 14:50:39 +0300 Subject: [PATCH 39/81] specify the base directory for loading the .env file --- initializers/load.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/initializers/load.go b/initializers/load.go index 4ea5980..4cbeb0e 100644 --- a/initializers/load.go +++ b/initializers/load.go @@ -3,24 +3,26 @@ package initializers import ( "log" "os" + "path" "strconv" "github.com/joho/godotenv" ) -func LoadEnvVariables() { - err := godotenv.Load() +func LoadEnvVariables(baseDir string) { + envDir := path.Join(baseDir, ".env") + err := godotenv.Load(envDir) if err != nil { - log.Fatal("Error loading .env file") + log.Fatal("Error loading .env file", err) } } // Helper to get environment variables with a default fallback func GetEnv(key, defaultVal string) string { - if value, exists := os.LookupEnv(key); exists { - return value + if value, exists := os.LookupEnv(key); exists { + return value } - return defaultVal + return defaultVal } // Helper to safely convert environment variables to uint From 79de0a9092f9fa7f9e5f610203a0574353d1b9a9 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Mon, 6 Jan 2025 14:54:04 +0300 Subject: [PATCH 40/81] pass the base directory to load the .env file --- cmd/africastalking/main.go | 4 +++- cmd/async/main.go | 4 +++- cmd/http/main.go | 8 +++++--- cmd/main.go | 4 +++- devtools/gen/main.go | 12 ++++++------ devtools/store/main.go | 13 +++++++------ 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index dfcaca1..4ca8400 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -14,6 +14,7 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -26,13 +27,14 @@ import ( var ( logg = logging.NewVanilla().WithDomain("AfricasTalking").WithContextKey("at-session-id") + baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") build = "dev" menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } func main() { config.LoadConfig() diff --git a/cmd/async/main.go b/cmd/async/main.go index bf23d9f..51b9e40 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -12,6 +12,7 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -22,12 +23,13 @@ import ( var ( logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } type asyncRequestParser struct { diff --git a/cmd/http/main.go b/cmd/http/main.go index 6ddfded..46dbe91 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -14,6 +14,7 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -24,13 +25,14 @@ import ( ) var ( - logg = logging.NewVanilla() - scriptDir = path.Join("services", "registration") + logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() + scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } func main() { diff --git a/cmd/main.go b/cmd/main.go index 4fd084f..fc6f147 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,16 +15,18 @@ import ( "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" + testdataloader "github.com/peteole/testdata-loader" ) var ( logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } func main() { diff --git a/devtools/gen/main.go b/devtools/gen/main.go index b9e2aed..f54afb7 100644 --- a/devtools/gen/main.go +++ b/devtools/gen/main.go @@ -9,22 +9,23 @@ import ( "path" "git.defalsify.org/vise.git/logging" - "git.grassecon.net/urdt/ussd/config" - "git.grassecon.net/urdt/ussd/internal/storage" - "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/common" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/storage" + testdataloader "github.com/peteole/testdata-loader" ) var ( logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } - func main() { config.LoadConfig() @@ -75,5 +76,4 @@ func main() { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) } - } diff --git a/devtools/store/main.go b/devtools/store/main.go index 8bd4d16..9f3e196 100644 --- a/devtools/store/main.go +++ b/devtools/store/main.go @@ -7,24 +7,25 @@ import ( "os" "path" - "git.grassecon.net/urdt/ussd/config" - "git.grassecon.net/urdt/ussd/initializers" - "git.grassecon.net/urdt/ussd/internal/storage" - "git.grassecon.net/urdt/ussd/debug" "git.defalsify.org/vise.git/db" "git.defalsify.org/vise.git/logging" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/debug" + "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/storage" + testdataloader "github.com/peteole/testdata-loader" ) var ( logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariables(baseDir) } - func main() { config.LoadConfig() From c12e867ac37e9d352a2c924622b79fb34885d41a Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Mon, 6 Jan 2025 15:06:25 +0300 Subject: [PATCH 41/81] add a db flag to specify the database of choice --- internal/testutil/TestEngine.go | 21 ++++++++++++++++----- menutraversal_test/menu_traversal_test.go | 8 +++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/internal/testutil/TestEngine.go b/internal/testutil/TestEngine.go index 3fcb307..40a744f 100644 --- a/internal/testutil/TestEngine.go +++ b/internal/testutil/TestEngine.go @@ -10,24 +10,35 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/internal/testutil/testservice" "git.grassecon.net/urdt/ussd/internal/testutil/testtag" - testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/remote" + testdataloader "github.com/peteole/testdata-loader" ) var ( - baseDir = testdataloader.GetBasePath() - logg = logging.NewVanilla() - scriptDir = path.Join(baseDir, "services", "registration") + baseDir = testdataloader.GetBasePath() + logg = logging.NewVanilla() + scriptDir = path.Join(baseDir, "services", "registration") + selectedDatabase = "" ) +func init() { + initializers.LoadEnvVariables(baseDir) +} + +// SetDatabase updates the database used by TestEngine +func SetDatabase(dbType string) { + selectedDatabase = dbType +} + func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) - ctx = context.WithValue(ctx, "Database", "gdbm") + ctx = context.WithValue(ctx, "Database", selectedDatabase) pfp := path.Join(scriptDir, "pp.csv") var eventChannel = make(chan bool) diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index 6b6b3da..8cfe710 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -24,6 +24,7 @@ var ( ) var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests") +var database = flag.String("db", "gdbm", "Specify the database (gdbm or postgres)") func GenerateSessionId() string { uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g)) @@ -79,12 +80,18 @@ func extractSendAmount(response []byte) string { } func TestMain(m *testing.M) { + // Parse the flags + flag.Parse() + sessionID = GenerateSessionId() defer func() { if err := os.RemoveAll(testStore); err != nil { log.Fatalf("Failed to delete state store %s: %v", testStore, err) } }() + + // Set the selected database + testutil.SetDatabase(*database) m.Run() } @@ -121,7 +128,6 @@ func TestAccountCreationSuccessful(t *testing.T) { } } <-eventChannel - } func TestAccountRegistrationRejectTerms(t *testing.T) { From 68e4c9af036e2c38499f9e7b2c7ec1a77dec1e24 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 21:00:34 +0300 Subject: [PATCH 42/81] add key for incorrect pin attempts --- common/db.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/db.go b/common/db.go index a5cf1c1..5e2fc4c 100644 --- a/common/db.go +++ b/common/db.go @@ -7,7 +7,7 @@ import ( "git.defalsify.org/vise.git/logging" ) -// DataType is a subprefix value used in association with vise/db.DATATYPE_USERDATA. +// DataType is a subprefix value used in association with vise/db.DATATYPE_USERDATA. // // All keys are used only within the context of a single account. Unless otherwise specified, the user context is the session id. // @@ -55,6 +55,8 @@ const ( DATA_ACTIVE_DECIMAL // EVM address of the currently active voucher DATA_ACTIVE_ADDRESS + //Holds count of the number of incorrect PIN attempts + DATA_INCORRECT_PIN_ATTEMPTS ) const ( From b37f2a0a110cc5e2c50f76eeac854144d61363c3 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 21:06:54 +0300 Subject: [PATCH 43/81] add flag for when an account has been blocked --- services/registration/pp.csv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/registration/pp.csv b/services/registration/pp.csv index 26a8833..aa1eb05 100644 --- a/services/registration/pp.csv +++ b/services/registration/pp.csv @@ -28,3 +28,5 @@ flag,flag_gender_set,34,this is set when the gender of the profile is set flag,flag_location_set,35,this is set when the location of the profile is set flag,flag_offerings_set,36,this is set when the offerings of the profile is set flag,flag_back_set,37,this is set when it is a back navigation +flag,flag_account_blocked,38,this is set when an account has been blocked after the allowed incorrect PIN attempts have been exceeded + From 90df0eefc39dc62ee6c32b96e1de16705a4a2442 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 22:53:59 +0300 Subject: [PATCH 44/81] add value for allowed number of PIN attempts --- common/pin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/pin.go b/common/pin.go index 6db9d15..d42fb58 100644 --- a/common/pin.go +++ b/common/pin.go @@ -8,7 +8,8 @@ import ( // Define the regex pattern as a constant const ( - pinPattern = `^\d{4}$` + pinPattern = `^\d{4}$` + AllowedPINAttempts = uint8(3) ) // checks whether the given input is a 4 digit number From 562bd4fa241b9759cf344a5a2cad85e65bf1935e Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Mon, 6 Jan 2025 22:54:31 +0300 Subject: [PATCH 45/81] check for incorrect pin --- internal/handlers/ussd/menuhandler.go | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index dfdbd02..70fd305 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -739,6 +739,7 @@ func (h *Handlers) Authorize(ctx context.Context, sym string, input []byte) (res res.FlagReset = append(res.FlagReset, flag_account_authorized) } } else { + h.countIncorrectPINAttempts(ctx, sessionId) res.FlagSet = append(res.FlagSet, flag_incorrect_pin) res.FlagReset = append(res.FlagReset, flag_account_authorized) return res, nil @@ -752,6 +753,24 @@ func (h *Handlers) Authorize(ctx context.Context, sym string, input []byte) (res // ResetIncorrectPin resets the incorrect pin flag after a new PIN attempt. func (h *Handlers) ResetIncorrectPin(ctx context.Context, sym string, input []byte) (resource.Result, error) { var res resource.Result + store := h.userdataStore + + sessionId, ok := ctx.Value("SessionId").(string) + if !ok { + return res, fmt.Errorf("missing session") + } + currentWrongPinAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) + if err != nil { + if !db.IsNotFound(err) { + return res, err + } + } + pinAttemptsValue, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) + remainingPINAttempts := common.AllowedPINAttempts - uint8(pinAttemptsValue) + if remainingPINAttempts < common.AllowedPINAttempts { + res.Content = string(remainingPINAttempts) + } + flag_incorrect_pin, _ := h.flagManager.GetFlag("flag_incorrect_pin") res.FlagReset = append(res.FlagReset, flag_incorrect_pin) return res, nil @@ -2075,3 +2094,28 @@ func (h *Handlers) UpdateAllProfileItems(ctx context.Context, sym string, input } return res, nil } + +func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId string) { + var pinAttemptsCount uint8 + store := h.userdataStore + + currentWrongPinAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) + if err != nil { + if db.IsNotFound(err) { + //First time Wrong PIN attempt: initialize with a count of 1 + pinAttemptsCount = 1 + err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(pinAttemptsCount)))) + if err != nil { + logg.ErrorCtxf(ctx, "failed to write incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", currentWrongPinAttempts, "error", err) + } + return + } + } + pinAttemptsValue, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) + pinAttemptsCount = uint8(pinAttemptsValue) + 1 + + err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(pinAttemptsCount)))) + if err != nil { + logg.ErrorCtxf(ctx, "failed to write incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", pinAttemptsCount, "error", err) + } +} From f83f53904668883a4f71b2dc78b282c7a287e861 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Tue, 7 Jan 2025 10:48:59 +0300 Subject: [PATCH 46/81] add node to show remaining pin attempts --- services/registration/incorrect_pin | 2 +- services/registration/incorrect_pin_swa | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/registration/incorrect_pin b/services/registration/incorrect_pin index 7fcf610..0e34cd1 100644 --- a/services/registration/incorrect_pin +++ b/services/registration/incorrect_pin @@ -1 +1 @@ -Incorrect PIN \ No newline at end of file +Incorrect PIN.You have: {{.reset_incorrect}} remaining attempt(s). \ No newline at end of file diff --git a/services/registration/incorrect_pin_swa b/services/registration/incorrect_pin_swa index 34a0b28..8a8b913 100644 --- a/services/registration/incorrect_pin_swa +++ b/services/registration/incorrect_pin_swa @@ -1 +1 @@ -PIN ulioeka sio sahihi \ No newline at end of file +PIN ulioeka sio sahihi,una majaribio: {{.reset_incorrect}} yaliyobaki \ No newline at end of file From 11d30583a432088503388ba35cc479fc719fe2c9 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Tue, 7 Jan 2025 10:50:30 +0300 Subject: [PATCH 47/81] map content of reset_incorrect and catch blocked account --- services/registration/incorrect_pin.vis | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/registration/incorrect_pin.vis b/services/registration/incorrect_pin.vis index 844f3d6..167364a 100644 --- a/services/registration/incorrect_pin.vis +++ b/services/registration/incorrect_pin.vis @@ -1,5 +1,7 @@ LOAD reset_incorrect 0 RELOAD reset_incorrect +MAP reset_incorrect +CATCH blocked_account flag_account_blocked 1 MOUT retry 1 MOUT quit 9 HALT From 6c5873da6fe3ed6795f964ebefad89d055e110ad Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Tue, 7 Jan 2025 12:15:15 +0300 Subject: [PATCH 48/81] trim any leading whitespace in the input --- internal/http/at/parse.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/http/at/parse.go b/internal/http/at/parse.go index 5f27d50..76e84e7 100644 --- a/internal/http/at/parse.go +++ b/internal/http/at/parse.go @@ -81,7 +81,8 @@ func (arp *ATRequestParser) GetInput(rq any) ([]byte, error) { return nil, fmt.Errorf("no input found") } - return []byte(parts[len(parts)-1]), nil + trimmedInput := strings.TrimSpace(parts[len(parts)-1]) + return []byte(trimmedInput), nil } func parseQueryParams(query string) map[string]string { From ee442daefa0c04db6b91183d37589cdf74b76795 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Tue, 7 Jan 2025 14:03:53 +0300 Subject: [PATCH 49/81] add blocked account node --- services/registration/blocked_account | 1 + services/registration/blocked_account.vis | 3 +++ services/registration/blocked_account_swa | 1 + 3 files changed, 5 insertions(+) create mode 100644 services/registration/blocked_account create mode 100644 services/registration/blocked_account.vis create mode 100644 services/registration/blocked_account_swa diff --git a/services/registration/blocked_account b/services/registration/blocked_account new file mode 100644 index 0000000..9a34267 --- /dev/null +++ b/services/registration/blocked_account @@ -0,0 +1 @@ +Your account has been locked.For help on how to unblock your account,contact support at: 0757628885 diff --git a/services/registration/blocked_account.vis b/services/registration/blocked_account.vis new file mode 100644 index 0000000..d5e6b2d --- /dev/null +++ b/services/registration/blocked_account.vis @@ -0,0 +1,3 @@ +MOUT quit 9 +HALT +INCMP quit 9 diff --git a/services/registration/blocked_account_swa b/services/registration/blocked_account_swa new file mode 100644 index 0000000..c00243b --- /dev/null +++ b/services/registration/blocked_account_swa @@ -0,0 +1 @@ +Akaunti yako imefungwa.Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885 \ No newline at end of file From c15958a1ad8d98cc684363bc181bd4627b5684a4 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Tue, 7 Jan 2025 14:32:44 +0300 Subject: [PATCH 50/81] reset incorrect pin attempts on correct entry --- internal/handlers/ussd/menuhandler.go | 54 ++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index 70fd305..5582265 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -734,12 +734,23 @@ func (h *Handlers) Authorize(ctx context.Context, sym string, input []byte) (res if h.st.MatchFlag(flag_account_authorized, false) { res.FlagReset = append(res.FlagReset, flag_incorrect_pin) res.FlagSet = append(res.FlagSet, flag_allow_update, flag_account_authorized) + err := h.resetIncorrectPINAttempts(ctx, sessionId) + if err != nil { + return res, err + } } else { res.FlagSet = append(res.FlagSet, flag_allow_update) res.FlagReset = append(res.FlagReset, flag_account_authorized) + err := h.resetIncorrectPINAttempts(ctx, sessionId) + if err != nil { + return res, err + } } } else { - h.countIncorrectPINAttempts(ctx, sessionId) + err := h.countIncorrectPINAttempts(ctx, sessionId) + if err != nil { + return res, err + } res.FlagSet = append(res.FlagSet, flag_incorrect_pin) res.FlagReset = append(res.FlagReset, flag_account_authorized) return res, nil @@ -755,10 +766,16 @@ func (h *Handlers) ResetIncorrectPin(ctx context.Context, sym string, input []by var res resource.Result store := h.userdataStore + flag_incorrect_pin, _ := h.flagManager.GetFlag("flag_incorrect_pin") + flag_account_blocked, _ := h.flagManager.GetFlag("flag_account_blocked") + sessionId, ok := ctx.Value("SessionId").(string) if !ok { return res, fmt.Errorf("missing session") } + + res.FlagReset = append(res.FlagReset, flag_incorrect_pin) + currentWrongPinAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) if err != nil { if !db.IsNotFound(err) { @@ -767,12 +784,14 @@ func (h *Handlers) ResetIncorrectPin(ctx context.Context, sym string, input []by } pinAttemptsValue, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) remainingPINAttempts := common.AllowedPINAttempts - uint8(pinAttemptsValue) + if remainingPINAttempts == 0 { + res.FlagSet = append(res.FlagSet, flag_account_blocked) + return res, nil + } if remainingPINAttempts < common.AllowedPINAttempts { - res.Content = string(remainingPINAttempts) + res.Content = strconv.Itoa(int(remainingPINAttempts)) } - flag_incorrect_pin, _ := h.flagManager.GetFlag("flag_incorrect_pin") - res.FlagReset = append(res.FlagReset, flag_incorrect_pin) return res, nil } @@ -2095,7 +2114,7 @@ func (h *Handlers) UpdateAllProfileItems(ctx context.Context, sym string, input return res, nil } -func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId string) { +func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId string) error { var pinAttemptsCount uint8 store := h.userdataStore @@ -2107,8 +2126,9 @@ func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId stri err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(pinAttemptsCount)))) if err != nil { logg.ErrorCtxf(ctx, "failed to write incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", currentWrongPinAttempts, "error", err) + return err } - return + return nil } } pinAttemptsValue, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) @@ -2117,5 +2137,27 @@ func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId stri err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(pinAttemptsCount)))) if err != nil { logg.ErrorCtxf(ctx, "failed to write incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", pinAttemptsCount, "error", err) + return err } + return nil +} + +func (h *Handlers) resetIncorrectPINAttempts(ctx context.Context, sessionId string) error { + store := h.userdataStore + currentWrongPinAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) + if err != nil { + if db.IsNotFound(err) { + return nil + } + return err + } + remainingPINAttempts, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) + if remainingPINAttempts <= uint64(common.AllowedPINAttempts) { + err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(string("0"))) + if err != nil { + logg.ErrorCtxf(ctx, "failed to reset incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", common.AllowedPINAttempts, "error", err) + return err + } + } + return nil } From 64fba916700508d8f3ab6c1ba29ca4b8d1c5c618 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Tue, 7 Jan 2025 14:38:44 +0300 Subject: [PATCH 51/81] catch blocked account --- services/registration/root.vis | 1 + 1 file changed, 1 insertion(+) diff --git a/services/registration/root.vis b/services/registration/root.vis index 02ef9e9..102e6e5 100644 --- a/services/registration/root.vis +++ b/services/registration/root.vis @@ -1,3 +1,4 @@ +CATCH blocked_account flag_account_blocked 1 CATCH select_language flag_language_set 0 CATCH terms flag_account_created 0 LOAD check_account_status 0 From 1fa830f2866d9b39b96d4ea593b7aad99653bab4 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 7 Jan 2025 13:51:26 +0000 Subject: [PATCH 52/81] Add auth conn string to ssh, use connstr for execs --- cmd/africastalking/main.go | 2 +- cmd/async/main.go | 2 +- cmd/http/main.go | 2 +- cmd/main.go | 2 +- cmd/ssh/main.go | 19 ++++++++++++++----- internal/ssh/ssh.go | 1 + 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index e0d05f6..72d3944 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -63,7 +63,7 @@ func main() { if connStr != "" { connStr = config.DbConn } - connData, err := storage.ToConnData(config.DbConn) + connData, err := storage.ToConnData(connStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) diff --git a/cmd/async/main.go b/cmd/async/main.go index e63c469..dc293e6 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -74,7 +74,7 @@ func main() { if connStr != "" { connStr = config.DbConn } - connData, err := storage.ToConnData(config.DbConn) + connData, err := storage.ToConnData(connStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) diff --git a/cmd/http/main.go b/cmd/http/main.go index c61d68d..8e65232 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -62,7 +62,7 @@ func main() { if connStr != "" { connStr = config.DbConn } - connData, err := storage.ToConnData(config.DbConn) + connData, err := storage.ToConnData(connStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) diff --git a/cmd/main.go b/cmd/main.go index 4bf134b..3939c9d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,7 +55,7 @@ func main() { if connStr != "" { connStr = config.DbConn } - connData, err := storage.ToConnData(config.DbConn) + connData, err := storage.ToConnData(connStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index e55d793..51023e5 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -37,7 +37,7 @@ func main() { config.LoadConfig() var connStr string - var dbDir string + var authConnStr string var resourceDir string var size uint var engineDebug bool @@ -45,6 +45,7 @@ func main() { var host string var port uint flag.StringVar(&connStr, "c", "", "connection string") + flag.StringVar(&authConnStr, "authdb", "", "auth connection string") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") @@ -52,14 +53,22 @@ func main() { flag.UintVar(&port, "p", 7122, "socket port") flag.Parse() - if connStr != "" { + if connStr == "" { connStr = config.DbConn } - connData, err := storage.ToConnData(config.DbConn) + if authConnStr == "" { + authConnStr = connStr + } + connData, err := storage.ToConnData(connStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr err: %v", err) os.Exit(1) } + authConnData, err := storage.ToConnData(authConnStr) + if err != nil { + fmt.Fprintf(os.Stderr, "auth connstr err: %v", err) + os.Exit(1) + } sshKeyFile := flag.Arg(0) _, err = os.Stat(sshKeyFile) @@ -75,7 +84,7 @@ func main() { logg.WarnCtxf(ctx, "!!!!! Do not expose to internet and only use with tunnel!") logg.WarnCtxf(ctx, "!!!!! (See ssh -L <...>)") - logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "keyfile", sshKeyFile, "host", host, "port", port) + logg.Infof("start command", "conn", connData, "authconn", authConnData, "resourcedir", resourceDir, "outputsize", size, "keyfile", sshKeyFile, "host", host, "port", port) pfp := path.Join(scriptDir, "pp.csv") @@ -91,7 +100,7 @@ func main() { cfg.EngineDebug = true } - authKeyStore, err := ssh.NewSshKeyStore(ctx, dbDir) + authKeyStore, err := ssh.NewSshKeyStore(ctx, authConnData.String()) if err != nil { fmt.Fprintf(os.Stderr, "keystore file open error: %v", err) os.Exit(1) diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go index c3ec725..4b744a8 100644 --- a/internal/ssh/ssh.go +++ b/internal/ssh/ssh.go @@ -41,6 +41,7 @@ func NewAuther(ctx context.Context, keyStore *SshKeyStore) *auther { } func(a *auther) Check(conn ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + logg.TraceCtxf(a.Ctx, "looking for publickey", "pubkey", fmt.Sprintf("%x", pubKey)) va, err := a.keyStore.Get(a.Ctx, pubKey) if err != nil { return nil, err From 07b061a68b4d034a607cc38ab9c217f0c3c2eee5 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 09:26:53 +0300 Subject: [PATCH 53/81] remove blocked account templates --- services/registration/blocked_account | 1 - services/registration/blocked_account_swa | 1 - 2 files changed, 2 deletions(-) delete mode 100644 services/registration/blocked_account delete mode 100644 services/registration/blocked_account_swa diff --git a/services/registration/blocked_account b/services/registration/blocked_account deleted file mode 100644 index 9a34267..0000000 --- a/services/registration/blocked_account +++ /dev/null @@ -1 +0,0 @@ -Your account has been locked.For help on how to unblock your account,contact support at: 0757628885 diff --git a/services/registration/blocked_account_swa b/services/registration/blocked_account_swa deleted file mode 100644 index c00243b..0000000 --- a/services/registration/blocked_account_swa +++ /dev/null @@ -1 +0,0 @@ -Akaunti yako imefungwa.Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885 \ No newline at end of file From 50c7ff104616994ce532078483f2b66433a27dfe Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 09:27:45 +0300 Subject: [PATCH 54/81] register handler to show blocked account message --- internal/handlers/handlerservice.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/handlers/handlerservice.go b/internal/handlers/handlerservice.go index 1da28c3..0d49b0c 100644 --- a/internal/handlers/handlerservice.go +++ b/internal/handlers/handlerservice.go @@ -128,6 +128,7 @@ func (ls *LocalHandlerService) GetHandler(accountService remote.AccountServiceIn ls.DbRs.AddLocalFunc("view_statement", ussdHandlers.ViewTransactionStatement) ls.DbRs.AddLocalFunc("update_all_profile_items", ussdHandlers.UpdateAllProfileItems) ls.DbRs.AddLocalFunc("set_back", ussdHandlers.SetBack) + ls.DbRs.AddLocalFunc("show_blocked_account", ussdHandlers.ShowBlockedAccount) return ussdHandlers, nil } From 3004698d5bdeab0cc0def616a97de4f93e407e25 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 09:28:31 +0300 Subject: [PATCH 55/81] add a message for a blocked account --- services/registration/locale/swa/default.po | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/registration/locale/swa/default.po b/services/registration/locale/swa/default.po index 4bf876b..ab238e7 100644 --- a/services/registration/locale/swa/default.po +++ b/services/registration/locale/swa/default.po @@ -10,6 +10,9 @@ msgstr "Asante kwa kutumia huduma ya Sarafu. Kwaheri!" msgid "For more help,please call: 0757628885" msgstr "Kwa usaidizi zaidi,piga: 0757628885" +msgid "Your account has been locked.For help on how to unblock your account,contact support at: 0757628885" +msgstr "Akaunti yako imefungwa.Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885" + msgid "Balance: %s\n" msgstr "Salio: %s\n" From 4fe64a774748f2936ea55bc4bf69f6cfb1736a80 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 09:29:00 +0300 Subject: [PATCH 56/81] show message for a blocked account --- services/registration/blocked_account.vis | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/registration/blocked_account.vis b/services/registration/blocked_account.vis index d5e6b2d..d8adab2 100644 --- a/services/registration/blocked_account.vis +++ b/services/registration/blocked_account.vis @@ -1,3 +1,2 @@ -MOUT quit 9 +LOAD show_blocked_account 0 HALT -INCMP quit 9 From 4d72ae0313ad78052979c0ddcfbe25de94185fd3 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 09:30:51 +0300 Subject: [PATCH 57/81] add handler showing a message for a blocked account --- internal/handlers/ussd/menuhandler.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index 5582265..2fcf92e 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -878,6 +878,16 @@ func (h *Handlers) QuitWithHelp(ctx context.Context, sym string, input []byte) ( return res, nil } +// ShowBlockedAccount displays a message after an account has been blocked and how to reach support. +func (h *Handlers) ShowBlockedAccount(ctx context.Context, sym string, input []byte) (resource.Result, error) { + var res resource.Result + code := codeFromCtx(ctx) + l := gotext.NewLocale(translationDir, code) + l.AddDomain("default") + res.Content = l.Get("Your account has been locked.For help on how to unblock your account,contact support at: 0757628885") + return res, nil +} + // VerifyYob verifies the length of the given input. func (h *Handlers) VerifyYob(ctx context.Context, sym string, input []byte) (resource.Result, error) { var res resource.Result @@ -2114,6 +2124,7 @@ func (h *Handlers) UpdateAllProfileItems(ctx context.Context, sym string, input return res, nil } +// countIncorrectPINAttempts keeps track of the number of incorrect PIN attempts func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId string) error { var pinAttemptsCount uint8 store := h.userdataStore @@ -2142,6 +2153,7 @@ func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId stri return nil } +// resetIncorrectPINAttempts resets the number of incorrect PIN attempts after a correct PIN entry func (h *Handlers) resetIncorrectPINAttempts(ctx context.Context, sessionId string) error { store := h.userdataStore currentWrongPinAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) From 5081b6d4ce993a4da0452668be16ff39e86c8f53 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 8 Jan 2025 06:48:35 +0000 Subject: [PATCH 58/81] Space after comma --- internal/handlers/ussd/menuhandler.go | 2 +- services/registration/locale/swa/default.po | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index dfdbd02..ff0656d 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -835,7 +835,7 @@ func (h *Handlers) QuitWithHelp(ctx context.Context, sym string, input []byte) ( l := gotext.NewLocale(translationDir, code) l.AddDomain("default") - res.Content = l.Get("For more help,please call: 0757628885") + res.Content = l.Get("For more help, please call: 0757628885") res.FlagReset = append(res.FlagReset, flag_account_authorized) return res, nil } diff --git a/services/registration/locale/swa/default.po b/services/registration/locale/swa/default.po index 4bf876b..27e80c4 100644 --- a/services/registration/locale/swa/default.po +++ b/services/registration/locale/swa/default.po @@ -7,8 +7,8 @@ msgstr "Ombi lako limetumwa. %s atapokea %s %s kutoka kwa %s." msgid "Thank you for using Sarafu. Goodbye!" msgstr "Asante kwa kutumia huduma ya Sarafu. Kwaheri!" -msgid "For more help,please call: 0757628885" -msgstr "Kwa usaidizi zaidi,piga: 0757628885" +msgid "For more help, please call: 0757628885" +msgstr "Kwa usaidizi zaidi, piga: 0757628885" msgid "Balance: %s\n" msgstr "Salio: %s\n" From 721f80d0f268e76ae4942de245804a5743886187 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 8 Jan 2025 07:34:15 +0000 Subject: [PATCH 59/81] Repalce missing context --- internal/ssh/ssh.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/ssh/ssh.go b/internal/ssh/ssh.go index 4b744a8..8209187 100644 --- a/internal/ssh/ssh.go +++ b/internal/ssh/ssh.go @@ -204,6 +204,7 @@ func(s *SshRunner) GetEngine(sessionId string) (engine.Engine, func(), error) { // adapted example from crypto/ssh package, NewServerConn doc func(s *SshRunner) Run(ctx context.Context, keyStore *SshKeyStore) { + s.Ctx = ctx running := true // TODO: waitgroup should probably not be global From 46a6d2bc6e2e732d800750cb67901d1887b119bc Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Wed, 8 Jan 2025 10:37:47 +0300 Subject: [PATCH 60/81] create a schema if it does not exist and use it in the connection --- internal/storage/storageservice.go | 32 +++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 04e75ce..d252383 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -14,6 +14,7 @@ import ( "git.defalsify.org/vise.git/resource" "git.grassecon.net/urdt/ussd/initializers" gdbmstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm" + "github.com/jackc/pgx/v5/pgxpool" ) var ( @@ -64,6 +65,11 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D return nil, fmt.Errorf("failed to select the database") } + schema, ok := ctx.Value("Schema").(string) + if !ok { + return nil, fmt.Errorf("failed to select the schema") + } + if existingDb != nil { return existingDb, nil } @@ -72,8 +78,15 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D var err error if database == "postgres" { - newDb = postgres.NewPgDb() connStr := buildConnStr() + + // Ensure the schema exists + err = ensureSchemaExists(ctx, connStr, schema) + if err != nil { + return nil, fmt.Errorf("failed to ensure schema exists: %w", err) + } + + newDb = postgres.NewPgDb().WithSchema(schema) err = newDb.Connect(ctx, connStr) } else { newDb = gdbmstorage.NewThreadGdbmDb() @@ -88,6 +101,23 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D return newDb, nil } +// ensureSchemaExists creates a new schema if it does not exist +func ensureSchemaExists(ctx context.Context, connStr, schema string) error { + conn, err := pgxpool.New(ctx, connStr) + if err != nil { + return fmt.Errorf("failed to connect to the database: %w", err) + } + defer conn.Close() + + query := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", schema) + _, err = conn.Exec(ctx, query) + if err != nil { + return fmt.Errorf("failed to create schema: %w", err) + } + + return nil +} + func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) { stateStore, err := ms.GetStateStore(ctx) if err != nil { From 81c3378ea690cc12354145167be755ad7ea0b224 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Wed, 8 Jan 2025 10:55:43 +0300 Subject: [PATCH 61/81] use a flag to pass the schema to the context --- cmd/africastalking/main.go | 3 +++ cmd/async/main.go | 3 +++ cmd/http/main.go | 3 +++ cmd/main.go | 3 +++ 4 files changed, 12 insertions(+) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 4ca8400..0e330ae 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -43,12 +43,14 @@ func main() { var resourceDir string var size uint var database string + var dbSchema string var engineDebug bool var host string var port uint flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbSchema, "schema", "public", "database schema to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -59,6 +61,7 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + ctx = context.WithValue(ctx, "Schema", dbSchema) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/async/main.go b/cmd/async/main.go index 51b9e40..91c28c6 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -53,6 +53,7 @@ func main() { var resourceDir string var size uint var database string + var dbSchema string var engineDebug bool var host string var port uint @@ -60,6 +61,7 @@ func main() { flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbSchema, "schema", "public", "database schema to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -70,6 +72,7 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + ctx = context.WithValue(ctx, "Schema", dbSchema) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/http/main.go b/cmd/http/main.go index 46dbe91..69e31a1 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -42,12 +42,14 @@ func main() { var resourceDir string var size uint var database string + var dbSchema string var engineDebug bool var host string var port uint flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbSchema, "schema", "public", "database schema to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -58,6 +60,7 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "Database", database) + ctx = context.WithValue(ctx, "Schema", dbSchema) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/main.go b/cmd/main.go index fc6f147..484b6c1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -36,9 +36,11 @@ func main() { var size uint var sessionId string var database string + var dbSchema string var engineDebug bool flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&database, "db", "gdbm", "database to be used") + flag.StringVar(&dbSchema, "schema", "public", "database schema to be used") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") @@ -49,6 +51,7 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", database) + ctx = context.WithValue(ctx, "Schema", dbSchema) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ From f59c3a53ef743a2adfbc4084dd8d2ae91d8d9e92 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Wed, 8 Jan 2025 10:56:59 +0300 Subject: [PATCH 62/81] allow the BuildConnStr to be accessed by different packages --- internal/storage/storageservice.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index d252383..d333a05 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -36,7 +36,7 @@ type MenuStorageService struct { userDataStore db.Db } -func buildConnStr() string { +func BuildConnStr() string { host := initializers.GetEnv("DB_HOST", "localhost") user := initializers.GetEnv("DB_USER", "postgres") password := initializers.GetEnv("DB_PASSWORD", "") @@ -78,7 +78,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D var err error if database == "postgres" { - connStr := buildConnStr() + connStr := BuildConnStr() // Ensure the schema exists err = ensureSchemaExists(ctx, connStr, schema) From a37f6e6da30f0f21cf07718237169d47d9ce41e0 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Wed, 8 Jan 2025 10:57:58 +0300 Subject: [PATCH 63/81] pass the dbschema in the context --- internal/testutil/TestEngine.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/testutil/TestEngine.go b/internal/testutil/TestEngine.go index 40a744f..25a1c15 100644 --- a/internal/testutil/TestEngine.go +++ b/internal/testutil/TestEngine.go @@ -24,6 +24,7 @@ var ( logg = logging.NewVanilla() scriptDir = path.Join(baseDir, "services", "registration") selectedDatabase = "" + selectedDbSchema = "" ) func init() { @@ -31,14 +32,17 @@ func init() { } // SetDatabase updates the database used by TestEngine -func SetDatabase(dbType string) { +func SetDatabase(dbType string, dbSchema string) { selectedDatabase = dbType + selectedDbSchema = dbSchema } func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) ctx = context.WithValue(ctx, "Database", selectedDatabase) + ctx = context.WithValue(ctx, "Schema", selectedDbSchema) + pfp := path.Join(scriptDir, "pp.csv") var eventChannel = make(chan bool) From ea9cab930e120666715779f0e83f5c22e296ef2a Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Wed, 8 Jan 2025 10:59:22 +0300 Subject: [PATCH 64/81] cleanup the generated test data for the schema --- menutraversal_test/menu_traversal_test.go | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index 8cfe710..d2e353d 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -4,15 +4,18 @@ import ( "bytes" "context" "flag" + "fmt" "log" "math/rand" "os" "regexp" "testing" + "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/internal/testutil" "git.grassecon.net/urdt/ussd/internal/testutil/driver" "github.com/gofrs/uuid" + "github.com/jackc/pgx/v5/pgxpool" ) var ( @@ -25,6 +28,7 @@ var ( var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests") var database = flag.String("db", "gdbm", "Specify the database (gdbm or postgres)") +var dbSchema = flag.String("schema", "test", "Specify the database schema (default test)") func GenerateSessionId() string { uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g)) @@ -91,7 +95,29 @@ func TestMain(m *testing.M) { }() // Set the selected database - testutil.SetDatabase(*database) + testutil.SetDatabase(*database, *dbSchema) + + // Cleanup the schema table after tests + defer func() { + if *database == "postgres" { + ctx := context.Background() + connStr := storage.BuildConnStr() + dbConn, err := pgxpool.New(ctx, connStr) + if err != nil { + log.Fatalf("Failed to connect to database for cleanup: %v", err) + } + defer dbConn.Close() + + query := fmt.Sprintf("DELETE FROM %s.kv_vise;", *dbSchema) + _, execErr := dbConn.Exec(ctx, query) + if execErr != nil { + log.Printf("Failed to cleanup table %s.kv_vise: %v", *dbSchema, execErr) + } else { + log.Printf("Successfully cleaned up table %s.kv_vise", *dbSchema) + } + } + }() + m.Run() } From 7f3294a8a2e4ad188801d867eb793b11b1ea2c4f Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 11:47:41 +0300 Subject: [PATCH 65/81] update tests --- internal/handlers/ussd/menuhandler_test.go | 101 ++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/internal/handlers/ussd/menuhandler_test.go b/internal/handlers/ussd/menuhandler_test.go index 914dffc..5914821 100644 --- a/internal/handlers/ussd/menuhandler_test.go +++ b/internal/handlers/ussd/menuhandler_test.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "path" + "strconv" "strings" "testing" @@ -907,37 +908,79 @@ func TestResetAccountAuthorized(t *testing.T) { } func TestIncorrectPinReset(t *testing.T) { + sessionId := "session123" + ctx, store := InitializeTestStore(t) fm, err := NewFlagManager(flagsPath) + if err != nil { log.Fatal(err) } flag_incorrect_pin, _ := fm.parser.GetFlag("flag_incorrect_pin") + flag_account_blocked, _ := fm.parser.GetFlag("flag_account_blocked") + + ctx = context.WithValue(ctx, "SessionId", sessionId) // Define test cases tests := []struct { name string input []byte + attempts uint8 expectedResult resource.Result }{ { - name: "Test incorrect pin reset", + name: "Test when incorrect PIN attempts is 2", input: []byte(""), expectedResult: resource.Result{ FlagReset: []uint32{flag_incorrect_pin}, + Content: "1", //Expected remaining PIN attempts }, + attempts: 2, + }, + { + name: "Test incorrect pin reset when incorrect PIN attempts is 1", + input: []byte(""), + expectedResult: resource.Result{ + FlagReset: []uint32{flag_incorrect_pin}, + Content: "2", //Expected remaining PIN attempts + }, + attempts: 1, + }, + { + name: "Test incorrect pin reset when incorrect PIN attempts is 1", + input: []byte(""), + expectedResult: resource.Result{ + FlagReset: []uint32{flag_incorrect_pin}, + Content: "2", //Expected remaining PIN attempts + }, + attempts: 1, + }, + { + name: "Test incorrect pin reset when incorrect PIN attempts is 3(account expected to be blocked)", + input: []byte(""), + expectedResult: resource.Result{ + FlagReset: []uint32{flag_incorrect_pin}, + FlagSet: []uint32{flag_account_blocked}, + }, + attempts: 3, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + + if err := store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(tt.attempts)))); err != nil { + t.Fatal(err) + } + // Create the Handlers instance with the mock flag manager h := &Handlers{ - flagManager: fm.parser, + flagManager: fm.parser, + userdataStore: store, } // Call the method - res, err := h.ResetIncorrectPin(context.Background(), "reset_incorrect_pin", tt.input) + res, err := h.ResetIncorrectPin(ctx, "reset_incorrect_pin", tt.input) if err != nil { t.Error(err) } @@ -2190,3 +2233,55 @@ func TestGetVoucherDetails(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expectedResult, res) } + +func TestCountIncorrectPINAttempts(t *testing.T) { + ctx, store := InitializeTestStore(t) + sessionId := "session123" + ctx = context.WithValue(ctx, "SessionId", sessionId) + attempts := uint8(2) + + h := &Handlers{ + userdataStore: store, + } + err := store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(strconv.Itoa(int(attempts)))) + if err != nil { + t.Logf(err.Error()) + } + err = h.countIncorrectPINAttempts(ctx, sessionId) + if err != nil { + t.Logf(err.Error()) + } + + attemptsAfterCount, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) + if err != nil { + t.Logf(err.Error()) + } + pinAttemptsValue, _ := strconv.ParseUint(string(attemptsAfterCount), 0, 64) + pinAttemptsCount := uint8(pinAttemptsValue) + expectedAttempts := attempts + 1 + assert.Equal(t, pinAttemptsCount, expectedAttempts) + +} + +func TestResetIncorrectPINAttempts(t *testing.T) { + ctx, store := InitializeTestStore(t) + sessionId := "session123" + ctx = context.WithValue(ctx, "SessionId", sessionId) + + err := store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(string("2"))) + if err != nil { + t.Logf(err.Error()) + } + + h := &Handlers{ + userdataStore: store, + } + h.resetIncorrectPINAttempts(ctx, sessionId) + incorrectAttempts, err := store.ReadEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS) + + if err != nil { + t.Logf(err.Error()) + } + assert.Equal(t, "0", string(incorrectAttempts)) + +} From 8d477356f38e26112d89e0e70e3e48caa50a991f Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 11:47:55 +0300 Subject: [PATCH 66/81] update tests --- menutraversal_test/group_test.json | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/menutraversal_test/group_test.json b/menutraversal_test/group_test.json index f35beb9..0998c11 100644 --- a/menutraversal_test/group_test.json +++ b/menutraversal_test/group_test.json @@ -54,7 +54,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", @@ -95,7 +95,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", @@ -107,8 +107,7 @@ }, { "input": "0", - "expectedContent": "Balances:\n1:My balance\n2:Community balance\n0:Back" - + "expectedContent": "Balances:\n1:My balance\n2:Community balance\n0:Back" }, { "input": "0", @@ -141,7 +140,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", @@ -153,8 +152,7 @@ }, { "input": "0", - "expectedContent": "Balances:\n1:My balance\n2:Community balance\n0:Back" - + "expectedContent": "Balances:\n1:My balance\n2:Community balance\n0:Back" }, { "input": "0", @@ -195,7 +193,7 @@ }, { "input": "1", - "expectedContent": "Enter your year of birth\n0:Back" + "expectedContent": "Enter your year of birth\n0:Back" }, { "input": "1940", @@ -258,7 +256,6 @@ "input": "0", "expectedContent": "{balance}\n\n1:Send\n2:My Vouchers\n3:My Account\n4:Help\n9:Quit" } - ] }, { @@ -443,10 +440,4 @@ ] } ] -} - - - - - - +} \ No newline at end of file From ed6651697aff2b56d012704fb3e0883888283d65 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 14:13:14 +0300 Subject: [PATCH 67/81] chore : add variable description to AllowedPINAttempts --- common/pin.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/pin.go b/common/pin.go index d42fb58..13f21b3 100644 --- a/common/pin.go +++ b/common/pin.go @@ -6,10 +6,13 @@ import ( "golang.org/x/crypto/bcrypt" ) -// Define the regex pattern as a constant const ( - pinPattern = `^\d{4}$` + // Define the regex pattern as a constant + pinPattern = `^\d{4}$` + + //Allowed incorrect PIN attempts AllowedPINAttempts = uint8(3) + ) // checks whether the given input is a 4 digit number From 477b4cf8f6471e331d9e624bb90a53ef4ad2c9d8 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 14:15:45 +0300 Subject: [PATCH 68/81] chore : rename remainingPINAttempts to currentWrongPinAttemptsCount --- internal/handlers/ussd/menuhandler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index a06c274..b76cc7f 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -2163,8 +2163,8 @@ func (h *Handlers) resetIncorrectPINAttempts(ctx context.Context, sessionId stri } return err } - remainingPINAttempts, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) - if remainingPINAttempts <= uint64(common.AllowedPINAttempts) { + currentWrongPinAttemptsCount, _ := strconv.ParseUint(string(currentWrongPinAttempts), 0, 64) + if currentWrongPinAttemptsCount <= uint64(common.AllowedPINAttempts) { err = store.WriteEntry(ctx, sessionId, common.DATA_INCORRECT_PIN_ATTEMPTS, []byte(string("0"))) if err != nil { logg.ErrorCtxf(ctx, "failed to reset incorrect PIN attempts ", "key", common.DATA_INCORRECT_PIN_ATTEMPTS, "value", common.AllowedPINAttempts, "error", err) From 2ff9fed3c5686a70a0dccd9f1f885b47a01f908a Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 14:51:15 +0300 Subject: [PATCH 69/81] chore: rename countIncorrectPINAttempts to incrementIncorrectPINAttempts --- internal/handlers/ussd/menuhandler.go | 8 ++++---- internal/handlers/ussd/menuhandler_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index b76cc7f..f831696 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -747,7 +747,7 @@ func (h *Handlers) Authorize(ctx context.Context, sym string, input []byte) (res } } } else { - err := h.countIncorrectPINAttempts(ctx, sessionId) + err := h.incrementIncorrectPINAttempts(ctx, sessionId) if err != nil { return res, err } @@ -884,7 +884,7 @@ func (h *Handlers) ShowBlockedAccount(ctx context.Context, sym string, input []b code := codeFromCtx(ctx) l := gotext.NewLocale(translationDir, code) l.AddDomain("default") - res.Content = l.Get("Your account has been locked.For help on how to unblock your account,contact support at: 0757628885") + res.Content = l.Get("Your account has been locked.For help on how to unblock your account, contact support at: 0757628885") return res, nil } @@ -2124,8 +2124,8 @@ func (h *Handlers) UpdateAllProfileItems(ctx context.Context, sym string, input return res, nil } -// countIncorrectPINAttempts keeps track of the number of incorrect PIN attempts -func (h *Handlers) countIncorrectPINAttempts(ctx context.Context, sessionId string) error { +// incrementIncorrectPINAttempts keeps track of the number of incorrect PIN attempts +func (h *Handlers) incrementIncorrectPINAttempts(ctx context.Context, sessionId string) error { var pinAttemptsCount uint8 store := h.userdataStore diff --git a/internal/handlers/ussd/menuhandler_test.go b/internal/handlers/ussd/menuhandler_test.go index 5914821..af1380d 100644 --- a/internal/handlers/ussd/menuhandler_test.go +++ b/internal/handlers/ussd/menuhandler_test.go @@ -2247,7 +2247,7 @@ func TestCountIncorrectPINAttempts(t *testing.T) { if err != nil { t.Logf(err.Error()) } - err = h.countIncorrectPINAttempts(ctx, sessionId) + err = h.incrementIncorrectPINAttempts(ctx, sessionId) if err != nil { t.Logf(err.Error()) } From efdb52bccde15e6639fdf1b7175da3b64863de37 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 14:52:40 +0300 Subject: [PATCH 70/81] chore: add space after punctuation --- menutraversal_test/group_test.json | 6 +++--- services/registration/incorrect_pin | 2 +- services/registration/incorrect_pin_swa | 2 +- services/registration/locale/swa/default.po | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/menutraversal_test/group_test.json b/menutraversal_test/group_test.json index 0998c11..0ffb49f 100644 --- a/menutraversal_test/group_test.json +++ b/menutraversal_test/group_test.json @@ -54,7 +54,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN. You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", @@ -95,7 +95,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN. You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", @@ -140,7 +140,7 @@ }, { "input": "1235", - "expectedContent": "Incorrect PIN.You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" + "expectedContent": "Incorrect PIN. You have: 2 remaining attempt(s).\n1:Retry\n9:Quit" }, { "input": "1", diff --git a/services/registration/incorrect_pin b/services/registration/incorrect_pin index 0e34cd1..13a9562 100644 --- a/services/registration/incorrect_pin +++ b/services/registration/incorrect_pin @@ -1 +1 @@ -Incorrect PIN.You have: {{.reset_incorrect}} remaining attempt(s). \ No newline at end of file +Incorrect PIN. You have: {{.reset_incorrect}} remaining attempt(s). \ No newline at end of file diff --git a/services/registration/incorrect_pin_swa b/services/registration/incorrect_pin_swa index 8a8b913..ed22beb 100644 --- a/services/registration/incorrect_pin_swa +++ b/services/registration/incorrect_pin_swa @@ -1 +1 @@ -PIN ulioeka sio sahihi,una majaribio: {{.reset_incorrect}} yaliyobaki \ No newline at end of file +PIN ulioeka sio sahihi, una majaribio: {{.reset_incorrect}} yaliyobaki \ No newline at end of file diff --git a/services/registration/locale/swa/default.po b/services/registration/locale/swa/default.po index 797c703..4effc1e 100644 --- a/services/registration/locale/swa/default.po +++ b/services/registration/locale/swa/default.po @@ -10,7 +10,7 @@ msgstr "Asante kwa kutumia huduma ya Sarafu. Kwaheri!" msgid "For more help, please call: 0757628885" msgstr "Kwa usaidizi zaidi, piga: 0757628885" -msgid "Your account has been locked.For help on how to unblock your account,contact support at: 0757628885" +msgid "Your account has been locked.For help on how to unblock your account, contact support at: 0757628885" msgstr "Akaunti yako imefungwa.Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885" msgid "Balance: %s\n" From b698f081363514907082d9df29520849a37cd8b7 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Wed, 8 Jan 2025 15:27:10 +0300 Subject: [PATCH 71/81] chore: add space after punctuation --- internal/handlers/ussd/menuhandler.go | 2 +- services/registration/locale/swa/default.po | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index f831696..607b812 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -884,7 +884,7 @@ func (h *Handlers) ShowBlockedAccount(ctx context.Context, sym string, input []b code := codeFromCtx(ctx) l := gotext.NewLocale(translationDir, code) l.AddDomain("default") - res.Content = l.Get("Your account has been locked.For help on how to unblock your account, contact support at: 0757628885") + res.Content = l.Get("Your account has been locked. For help on how to unblock your account, contact support at: 0757628885") return res, nil } diff --git a/services/registration/locale/swa/default.po b/services/registration/locale/swa/default.po index 4effc1e..6155063 100644 --- a/services/registration/locale/swa/default.po +++ b/services/registration/locale/swa/default.po @@ -10,8 +10,8 @@ msgstr "Asante kwa kutumia huduma ya Sarafu. Kwaheri!" msgid "For more help, please call: 0757628885" msgstr "Kwa usaidizi zaidi, piga: 0757628885" -msgid "Your account has been locked.For help on how to unblock your account, contact support at: 0757628885" -msgstr "Akaunti yako imefungwa.Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885" +msgid "Your account has been locked. For help on how to unblock your account, contact support at: 0757628885" +msgstr "Akaunti yako imefungwa. Kwa usaidizi wa jinsi ya kufungua akaunti yako, wasiliana na usaidizi kwa: 0757628885" msgid "Balance: %s\n" msgstr "Salio: %s\n" From 5734011f96db3a4abff8560ec4359388cb67fae0 Mon Sep 17 00:00:00 2001 From: konstantinmds Date: Wed, 8 Jan 2025 13:34:08 +0100 Subject: [PATCH 72/81] refactor: rename ussd package to application (#24) - Rename internal/handlers/ussd directory to application - Update all imports and references to use new package name --- .../{ussd => application}/menuhandler.go | 2 +- .../{ussd => application}/menuhandler_test.go | 2 +- internal/handlers/base.go | 38 +++--- internal/handlers/handlerservice.go | 122 +++++++++--------- 4 files changed, 82 insertions(+), 82 deletions(-) rename internal/handlers/{ussd => application}/menuhandler.go (99%) rename internal/handlers/{ussd => application}/menuhandler_test.go (99%) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/application/menuhandler.go similarity index 99% rename from internal/handlers/ussd/menuhandler.go rename to internal/handlers/application/menuhandler.go index 607b812..52b42b3 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/application/menuhandler.go @@ -1,4 +1,4 @@ -package ussd +package application import ( "bytes" diff --git a/internal/handlers/ussd/menuhandler_test.go b/internal/handlers/application/menuhandler_test.go similarity index 99% rename from internal/handlers/ussd/menuhandler_test.go rename to internal/handlers/application/menuhandler_test.go index af1380d..bb6230e 100644 --- a/internal/handlers/ussd/menuhandler_test.go +++ b/internal/handlers/application/menuhandler_test.go @@ -1,4 +1,4 @@ -package ussd +package application import ( "context" diff --git a/internal/handlers/base.go b/internal/handlers/base.go index 755cca4..6c77f49 100644 --- a/internal/handlers/base.go +++ b/internal/handlers/base.go @@ -6,46 +6,46 @@ import ( "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" - "git.grassecon.net/urdt/ussd/internal/handlers/ussd" + "git.grassecon.net/urdt/ussd/internal/handlers/application" "git.grassecon.net/urdt/ussd/internal/storage" ) type BaseSessionHandler struct { cfgTemplate engine.Config - rp RequestParser - rs resource.Resource - hn *ussd.Handlers - provider storage.StorageProvider + rp RequestParser + rs resource.Resource + hn *application.Handlers + provider storage.StorageProvider } -func NewBaseSessionHandler(cfg engine.Config, rs resource.Resource, stateDb db.Db, userdataDb db.Db, rp RequestParser, hn *ussd.Handlers) *BaseSessionHandler { +func NewBaseSessionHandler(cfg engine.Config, rs resource.Resource, stateDb db.Db, userdataDb db.Db, rp RequestParser, hn *application.Handlers) *BaseSessionHandler { return &BaseSessionHandler{ cfgTemplate: cfg, - rs: rs, - hn: hn, - rp: rp, - provider: storage.NewSimpleStorageProvider(stateDb, userdataDb), + rs: rs, + hn: hn, + rp: rp, + provider: storage.NewSimpleStorageProvider(stateDb, userdataDb), } } -func(f* BaseSessionHandler) Shutdown() { +func (f *BaseSessionHandler) Shutdown() { err := f.provider.Close() if err != nil { logg.Errorf("handler shutdown error", "err", err) } } -func(f *BaseSessionHandler) GetEngine(cfg engine.Config, rs resource.Resource, pr *persist.Persister) engine.Engine { +func (f *BaseSessionHandler) GetEngine(cfg engine.Config, rs resource.Resource, pr *persist.Persister) engine.Engine { en := engine.NewEngine(cfg, rs) en = en.WithPersister(pr) return en } -func(f *BaseSessionHandler) Process(rqs RequestSession) (RequestSession, error) { +func (f *BaseSessionHandler) Process(rqs RequestSession) (RequestSession, error) { var r bool var err error var ok bool - + logg.InfoCtxf(rqs.Ctx, "new request", "data", rqs) rqs.Storage, err = f.provider.Get(rqs.Config.SessionId) @@ -84,25 +84,25 @@ func(f *BaseSessionHandler) Process(rqs RequestSession) (RequestSession, error) return rqs, err } - rqs.Continue = r + rqs.Continue = r return rqs, nil } -func(f *BaseSessionHandler) Output(rqs RequestSession) (RequestSession, error) { +func (f *BaseSessionHandler) Output(rqs RequestSession) (RequestSession, error) { var err error _, err = rqs.Engine.Flush(rqs.Ctx, rqs.Writer) return rqs, err } -func(f *BaseSessionHandler) Reset(rqs RequestSession) (RequestSession, error) { +func (f *BaseSessionHandler) Reset(rqs RequestSession) (RequestSession, error) { defer f.provider.Put(rqs.Config.SessionId, rqs.Storage) return rqs, rqs.Engine.Finish() } -func(f *BaseSessionHandler) GetConfig() engine.Config { +func (f *BaseSessionHandler) GetConfig() engine.Config { return f.cfgTemplate } -func(f *BaseSessionHandler) GetRequestParser() RequestParser { +func (f *BaseSessionHandler) GetRequestParser() RequestParser { return f.rp } diff --git a/internal/handlers/handlerservice.go b/internal/handlers/handlerservice.go index 0d49b0c..6fb355b 100644 --- a/internal/handlers/handlerservice.go +++ b/internal/handlers/handlerservice.go @@ -10,13 +10,13 @@ import ( "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" - "git.grassecon.net/urdt/ussd/internal/handlers/ussd" + "git.grassecon.net/urdt/ussd/internal/handlers/application" "git.grassecon.net/urdt/ussd/internal/utils" "git.grassecon.net/urdt/ussd/remote" ) type HandlerService interface { - GetHandler() (*ussd.Handlers, error) + GetHandler() (*application.Handlers, error) } func getParser(fp string, debug bool) (*asm.FlagParser, error) { @@ -64,73 +64,73 @@ func (ls *LocalHandlerService) SetDataStore(db *db.Db) { ls.UserdataStore = db } -func (ls *LocalHandlerService) GetHandler(accountService remote.AccountServiceInterface) (*ussd.Handlers, error) { +func (ls *LocalHandlerService) GetHandler(accountService remote.AccountServiceInterface) (*application.Handlers, error) { replaceSeparatorFunc := func(input string) string { return strings.ReplaceAll(input, ":", ls.Cfg.MenuSeparator) } - ussdHandlers, err := ussd.NewHandlers(ls.Parser, *ls.UserdataStore, ls.AdminStore, accountService, replaceSeparatorFunc) + appHandlers, err := application.NewHandlers(ls.Parser, *ls.UserdataStore, ls.AdminStore, accountService, replaceSeparatorFunc) if err != nil { return nil, err } - ussdHandlers = ussdHandlers.WithPersister(ls.Pe) - ls.DbRs.AddLocalFunc("set_language", ussdHandlers.SetLanguage) - ls.DbRs.AddLocalFunc("create_account", ussdHandlers.CreateAccount) - ls.DbRs.AddLocalFunc("save_temporary_pin", ussdHandlers.SaveTemporaryPin) - ls.DbRs.AddLocalFunc("verify_create_pin", ussdHandlers.VerifyCreatePin) - ls.DbRs.AddLocalFunc("check_identifier", ussdHandlers.CheckIdentifier) - ls.DbRs.AddLocalFunc("check_account_status", ussdHandlers.CheckAccountStatus) - ls.DbRs.AddLocalFunc("authorize_account", ussdHandlers.Authorize) - ls.DbRs.AddLocalFunc("quit", ussdHandlers.Quit) - ls.DbRs.AddLocalFunc("check_balance", ussdHandlers.CheckBalance) - ls.DbRs.AddLocalFunc("validate_recipient", ussdHandlers.ValidateRecipient) - ls.DbRs.AddLocalFunc("transaction_reset", ussdHandlers.TransactionReset) - ls.DbRs.AddLocalFunc("invite_valid_recipient", ussdHandlers.InviteValidRecipient) - ls.DbRs.AddLocalFunc("max_amount", ussdHandlers.MaxAmount) - ls.DbRs.AddLocalFunc("validate_amount", ussdHandlers.ValidateAmount) - ls.DbRs.AddLocalFunc("reset_transaction_amount", ussdHandlers.ResetTransactionAmount) - ls.DbRs.AddLocalFunc("get_recipient", ussdHandlers.GetRecipient) - ls.DbRs.AddLocalFunc("get_sender", ussdHandlers.GetSender) - ls.DbRs.AddLocalFunc("get_amount", ussdHandlers.GetAmount) - ls.DbRs.AddLocalFunc("reset_incorrect", ussdHandlers.ResetIncorrectPin) - ls.DbRs.AddLocalFunc("save_firstname", ussdHandlers.SaveFirstname) - ls.DbRs.AddLocalFunc("save_familyname", ussdHandlers.SaveFamilyname) - ls.DbRs.AddLocalFunc("save_gender", ussdHandlers.SaveGender) - ls.DbRs.AddLocalFunc("save_location", ussdHandlers.SaveLocation) - ls.DbRs.AddLocalFunc("save_yob", ussdHandlers.SaveYob) - ls.DbRs.AddLocalFunc("save_offerings", ussdHandlers.SaveOfferings) - ls.DbRs.AddLocalFunc("reset_account_authorized", ussdHandlers.ResetAccountAuthorized) - ls.DbRs.AddLocalFunc("reset_allow_update", ussdHandlers.ResetAllowUpdate) - ls.DbRs.AddLocalFunc("get_profile_info", ussdHandlers.GetProfileInfo) - ls.DbRs.AddLocalFunc("verify_yob", ussdHandlers.VerifyYob) - ls.DbRs.AddLocalFunc("reset_incorrect_date_format", ussdHandlers.ResetIncorrectYob) - ls.DbRs.AddLocalFunc("initiate_transaction", ussdHandlers.InitiateTransaction) - ls.DbRs.AddLocalFunc("verify_new_pin", ussdHandlers.VerifyNewPin) - ls.DbRs.AddLocalFunc("confirm_pin_change", ussdHandlers.ConfirmPinChange) - ls.DbRs.AddLocalFunc("quit_with_help", ussdHandlers.QuitWithHelp) - ls.DbRs.AddLocalFunc("fetch_community_balance", ussdHandlers.FetchCommunityBalance) - ls.DbRs.AddLocalFunc("set_default_voucher", ussdHandlers.SetDefaultVoucher) - ls.DbRs.AddLocalFunc("check_vouchers", ussdHandlers.CheckVouchers) - ls.DbRs.AddLocalFunc("get_vouchers", ussdHandlers.GetVoucherList) - ls.DbRs.AddLocalFunc("view_voucher", ussdHandlers.ViewVoucher) - ls.DbRs.AddLocalFunc("set_voucher", ussdHandlers.SetVoucher) - ls.DbRs.AddLocalFunc("get_voucher_details", ussdHandlers.GetVoucherDetails) - ls.DbRs.AddLocalFunc("reset_valid_pin", ussdHandlers.ResetValidPin) - ls.DbRs.AddLocalFunc("check_pin_mismatch", ussdHandlers.CheckBlockedNumPinMisMatch) - ls.DbRs.AddLocalFunc("validate_blocked_number", ussdHandlers.ValidateBlockedNumber) - ls.DbRs.AddLocalFunc("retrieve_blocked_number", ussdHandlers.RetrieveBlockedNumber) - ls.DbRs.AddLocalFunc("reset_unregistered_number", ussdHandlers.ResetUnregisteredNumber) - ls.DbRs.AddLocalFunc("reset_others_pin", ussdHandlers.ResetOthersPin) - ls.DbRs.AddLocalFunc("save_others_temporary_pin", ussdHandlers.SaveOthersTemporaryPin) - ls.DbRs.AddLocalFunc("get_current_profile_info", ussdHandlers.GetCurrentProfileInfo) - ls.DbRs.AddLocalFunc("check_transactions", ussdHandlers.CheckTransactions) - ls.DbRs.AddLocalFunc("get_transactions", ussdHandlers.GetTransactionsList) - ls.DbRs.AddLocalFunc("view_statement", ussdHandlers.ViewTransactionStatement) - ls.DbRs.AddLocalFunc("update_all_profile_items", ussdHandlers.UpdateAllProfileItems) - ls.DbRs.AddLocalFunc("set_back", ussdHandlers.SetBack) - ls.DbRs.AddLocalFunc("show_blocked_account", ussdHandlers.ShowBlockedAccount) + appHandlers = appHandlers.WithPersister(ls.Pe) + ls.DbRs.AddLocalFunc("set_language", appHandlers.SetLanguage) + ls.DbRs.AddLocalFunc("create_account", appHandlers.CreateAccount) + ls.DbRs.AddLocalFunc("save_temporary_pin", appHandlers.SaveTemporaryPin) + ls.DbRs.AddLocalFunc("verify_create_pin", appHandlers.VerifyCreatePin) + ls.DbRs.AddLocalFunc("check_identifier", appHandlers.CheckIdentifier) + ls.DbRs.AddLocalFunc("check_account_status", appHandlers.CheckAccountStatus) + ls.DbRs.AddLocalFunc("authorize_account", appHandlers.Authorize) + ls.DbRs.AddLocalFunc("quit", appHandlers.Quit) + ls.DbRs.AddLocalFunc("check_balance", appHandlers.CheckBalance) + ls.DbRs.AddLocalFunc("validate_recipient", appHandlers.ValidateRecipient) + ls.DbRs.AddLocalFunc("transaction_reset", appHandlers.TransactionReset) + ls.DbRs.AddLocalFunc("invite_valid_recipient", appHandlers.InviteValidRecipient) + ls.DbRs.AddLocalFunc("max_amount", appHandlers.MaxAmount) + ls.DbRs.AddLocalFunc("validate_amount", appHandlers.ValidateAmount) + ls.DbRs.AddLocalFunc("reset_transaction_amount", appHandlers.ResetTransactionAmount) + ls.DbRs.AddLocalFunc("get_recipient", appHandlers.GetRecipient) + ls.DbRs.AddLocalFunc("get_sender", appHandlers.GetSender) + ls.DbRs.AddLocalFunc("get_amount", appHandlers.GetAmount) + ls.DbRs.AddLocalFunc("reset_incorrect", appHandlers.ResetIncorrectPin) + ls.DbRs.AddLocalFunc("save_firstname", appHandlers.SaveFirstname) + ls.DbRs.AddLocalFunc("save_familyname", appHandlers.SaveFamilyname) + ls.DbRs.AddLocalFunc("save_gender", appHandlers.SaveGender) + ls.DbRs.AddLocalFunc("save_location", appHandlers.SaveLocation) + ls.DbRs.AddLocalFunc("save_yob", appHandlers.SaveYob) + ls.DbRs.AddLocalFunc("save_offerings", appHandlers.SaveOfferings) + ls.DbRs.AddLocalFunc("reset_account_authorized", appHandlers.ResetAccountAuthorized) + ls.DbRs.AddLocalFunc("reset_allow_update", appHandlers.ResetAllowUpdate) + ls.DbRs.AddLocalFunc("get_profile_info", appHandlers.GetProfileInfo) + ls.DbRs.AddLocalFunc("verify_yob", appHandlers.VerifyYob) + ls.DbRs.AddLocalFunc("reset_incorrect_date_format", appHandlers.ResetIncorrectYob) + ls.DbRs.AddLocalFunc("initiate_transaction", appHandlers.InitiateTransaction) + ls.DbRs.AddLocalFunc("verify_new_pin", appHandlers.VerifyNewPin) + ls.DbRs.AddLocalFunc("confirm_pin_change", appHandlers.ConfirmPinChange) + ls.DbRs.AddLocalFunc("quit_with_help", appHandlers.QuitWithHelp) + ls.DbRs.AddLocalFunc("fetch_community_balance", appHandlers.FetchCommunityBalance) + ls.DbRs.AddLocalFunc("set_default_voucher", appHandlers.SetDefaultVoucher) + ls.DbRs.AddLocalFunc("check_vouchers", appHandlers.CheckVouchers) + ls.DbRs.AddLocalFunc("get_vouchers", appHandlers.GetVoucherList) + ls.DbRs.AddLocalFunc("view_voucher", appHandlers.ViewVoucher) + ls.DbRs.AddLocalFunc("set_voucher", appHandlers.SetVoucher) + ls.DbRs.AddLocalFunc("get_voucher_details", appHandlers.GetVoucherDetails) + ls.DbRs.AddLocalFunc("reset_valid_pin", appHandlers.ResetValidPin) + ls.DbRs.AddLocalFunc("check_pin_mismatch", appHandlers.CheckBlockedNumPinMisMatch) + ls.DbRs.AddLocalFunc("validate_blocked_number", appHandlers.ValidateBlockedNumber) + ls.DbRs.AddLocalFunc("retrieve_blocked_number", appHandlers.RetrieveBlockedNumber) + ls.DbRs.AddLocalFunc("reset_unregistered_number", appHandlers.ResetUnregisteredNumber) + ls.DbRs.AddLocalFunc("reset_others_pin", appHandlers.ResetOthersPin) + ls.DbRs.AddLocalFunc("save_others_temporary_pin", appHandlers.SaveOthersTemporaryPin) + ls.DbRs.AddLocalFunc("get_current_profile_info", appHandlers.GetCurrentProfileInfo) + ls.DbRs.AddLocalFunc("check_transactions", appHandlers.CheckTransactions) + ls.DbRs.AddLocalFunc("get_transactions", appHandlers.GetTransactionsList) + ls.DbRs.AddLocalFunc("view_statement", appHandlers.ViewTransactionStatement) + ls.DbRs.AddLocalFunc("update_all_profile_items", appHandlers.UpdateAllProfileItems) + ls.DbRs.AddLocalFunc("set_back", appHandlers.SetBack) + ls.DbRs.AddLocalFunc("show_blocked_account", appHandlers.ShowBlockedAccount) - return ussdHandlers, nil + return appHandlers, nil } // TODO: enable setting of sessionId on engine init time From df8c9aab0c056dce7a6ca68aa2b8490efa0daf02 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 8 Jan 2025 22:27:19 +0000 Subject: [PATCH 73/81] Rehabilitate tests --- cmd/africastalking/main.go | 8 +------- cmd/async/main.go | 8 +------- cmd/http/main.go | 8 +------- cmd/main.go | 8 +------- devtools/store/generate/main.go | 2 +- initializers/load.go | 10 +++++++--- internal/storage/storageservice.go | 8 -------- internal/testutil/engine.go | 4 +++- menutraversal_test/menu_traversal_test.go | 3 +-- 9 files changed, 16 insertions(+), 43 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 6be7a11..053eab9 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -15,7 +15,6 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" "git.defalsify.org/vise.git/lang" - testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -28,14 +27,13 @@ import ( var ( logg = logging.NewVanilla().WithDomain("AfricasTalking").WithContextKey("at-session-id") - baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") build = "dev" menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables(baseDir) + initializers.LoadEnvVariables() } func main() { @@ -44,8 +42,6 @@ func main() { var connStr string var resourceDir string var size uint - var database string - var dbSchema string var engineDebug bool var host string var port uint @@ -75,8 +71,6 @@ func main() { logg.Infof("start command", "build", build, "conn", connData, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() - ctx = context.WithValue(ctx, "Database", database) - ctx = context.WithValue(ctx, "Schema", dbSchema) ln, err := lang.LanguageFromCode(config.DefaultLanguage) if err != nil { fmt.Fprintf(os.Stderr, "default language set error: %v", err) diff --git a/cmd/async/main.go b/cmd/async/main.go index 59af2e5..0b9a233 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -13,7 +13,6 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" "git.defalsify.org/vise.git/lang" - testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -25,13 +24,12 @@ import ( var ( logg = logging.NewVanilla() - baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables(baseDir) + initializers.LoadEnvVariables() } type asyncRequestParser struct { @@ -54,8 +52,6 @@ func main() { var sessionId string var resourceDir string var size uint - var database string - var dbSchema string var engineDebug bool var host string var port uint @@ -86,8 +82,6 @@ func main() { logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) ctx := context.Background() - ctx = context.WithValue(ctx, "Database", database) - ctx = context.WithValue(ctx, "Schema", dbSchema) ln, err := lang.LanguageFromCode(config.DefaultLanguage) if err != nil { diff --git a/cmd/http/main.go b/cmd/http/main.go index af0aefc..761ee72 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -15,7 +15,6 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" "git.defalsify.org/vise.git/lang" - testdataloader "github.com/peteole/testdata-loader" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" @@ -28,13 +27,12 @@ import ( var ( logg = logging.NewVanilla() - baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables(baseDir) + initializers.LoadEnvVariables() } func main() { @@ -43,8 +41,6 @@ func main() { var connStr string var resourceDir string var size uint - var database string - var dbSchema string var engineDebug bool var host string var port uint @@ -74,8 +70,6 @@ func main() { logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() - ctx = context.WithValue(ctx, "Database", database) - ctx = context.WithValue(ctx, "Schema", dbSchema) ln, err := lang.LanguageFromCode(config.DefaultLanguage) if err != nil { diff --git a/cmd/main.go b/cmd/main.go index 5c89c05..8c06094 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,18 +17,16 @@ import ( "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/remote" - testdataloader "github.com/peteole/testdata-loader" ) var ( logg = logging.NewVanilla() - baseDir = testdataloader.GetBasePath() scriptDir = path.Join("services", "registration") menuSeparator = ": " ) func init() { - initializers.LoadEnvVariables(baseDir) + initializers.LoadEnvVariables() } // TODO: external script automatically generate language handler list from select language vise code OR consider dynamic menu generation script possibility @@ -38,8 +36,6 @@ func main() { var connStr string var size uint var sessionId string - var database string - var dbSchema string var engineDebug bool var resourceDir string var err error @@ -72,8 +68,6 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) - ctx = context.WithValue(ctx, "Database", database) - ctx = context.WithValue(ctx, "Schema", dbSchema) ln, err := lang.LanguageFromCode(config.DefaultLanguage) if err != nil { diff --git a/devtools/store/generate/main.go b/devtools/store/generate/main.go index 58a9808..c421d1a 100644 --- a/devtools/store/generate/main.go +++ b/devtools/store/generate/main.go @@ -23,7 +23,7 @@ var ( ) func init() { - initializers.LoadEnvVariables(baseDir) + initializers.LoadEnvVariables() } func main() { diff --git a/initializers/load.go b/initializers/load.go index 4cbeb0e..fc61746 100644 --- a/initializers/load.go +++ b/initializers/load.go @@ -9,9 +9,13 @@ import ( "github.com/joho/godotenv" ) -func LoadEnvVariables(baseDir string) { - envDir := path.Join(baseDir, ".env") - err := godotenv.Load(envDir) +func LoadEnvVariables() { + LoadEnvVariablesPath(".") +} + +func LoadEnvVariablesPath(dir string) { + fp := path.Join(dir, ".env") + err := godotenv.Load(fp) if err != nil { log.Fatal("Error loading .env file", err) } diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index f2ac273..617c0ef 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -55,14 +55,6 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D connStr := ms.conn.String() dbTyp := ms.conn.DbType() if dbTyp == DBTYPE_POSTGRES { -// // Ensure the schema exists -// err = ensureSchemaExists(ctx, connStr, schema) -// if err != nil { -// return nil, fmt.Errorf("failed to ensure schema exists: %w", err) -// } -// -// newDb = postgres.NewPgDb().WithSchema(schema) - newDb = postgres.NewPgDb() } else if dbTyp == DBTYPE_GDBM { err = ms.ensureDbDir() diff --git a/internal/testutil/engine.go b/internal/testutil/engine.go index a1eefa8..8d5d3e4 100644 --- a/internal/testutil/engine.go +++ b/internal/testutil/engine.go @@ -17,17 +17,19 @@ import ( "git.grassecon.net/urdt/ussd/internal/testutil/testservice" "git.grassecon.net/urdt/ussd/internal/testutil/testtag" "git.grassecon.net/urdt/ussd/remote" + testdataloader "github.com/peteole/testdata-loader" ) var ( logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() scriptDir = path.Join(baseDir, "services", "registration") selectedDatabase = "" selectedDbSchema = "" ) func init() { - initializers.LoadEnvVariables() + initializers.LoadEnvVariablesPath(baseDir) } // SetDatabase updates the database used by TestEngine diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index db4586a..4003641 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -12,7 +12,6 @@ import ( "regexp" "testing" - "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/internal/testutil" "git.grassecon.net/urdt/ussd/internal/testutil/driver" "github.com/gofrs/uuid" @@ -106,7 +105,7 @@ func TestMain(m *testing.M) { defer func() { if *database == "postgres" { ctx := context.Background() - connStr := storage.BuildConnStr() + connStr := "postgres://" //storage.BuildConnStr() dbConn, err := pgxpool.New(ctx, connStr) if err != nil { log.Fatalf("Failed to connect to database for cleanup: %v", err) From b50a51df9b29ad5a0d4df51537b68f15f525a60f Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 9 Jan 2025 07:42:09 +0000 Subject: [PATCH 74/81] Implement postgres schema --- internal/storage/parse.go | 45 ++++++++++++++++++++---------- internal/storage/storageservice.go | 19 ++++++++----- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/internal/storage/parse.go b/internal/storage/parse.go index bb25627..41dac6b 100644 --- a/internal/storage/parse.go +++ b/internal/storage/parse.go @@ -15,6 +15,7 @@ const ( type ConnData struct { typ int str string + domain string } func (cd *ConnData) DbType() int { @@ -25,23 +26,38 @@ func (cd *ConnData) String() string { return cd.str } -func probePostgres(s string) (string, bool) { - v, err := url.Parse(s) - if err != nil { - return "", false - } - if v.Scheme != "postgres" { - return "", false - } - return s, true +func (cd *ConnData) Domain() string { + return cd.domain } -func probeGdbm(s string) (string, bool) { +func (cd *ConnData) Path() string { + v, _ := url.Parse(cd.str) + v.RawQuery = "" + return v.String() +} + +func probePostgres(s string) (string, string, bool) { + domain := "public" + v, err := url.Parse(s) + if err != nil { + return "", "", false + } + if v.Scheme != "postgres" { + return "", "", false + } + vv := v.Query() + if vv.Has("search_path") { + domain = vv.Get("search_path") + } + return s, domain, true +} + +func probeGdbm(s string) (string, string, bool) { if !path.IsAbs(s) { - return "", false + return "", "", false } s = path.Clean(s) - return s, true + return s, "", true } func ToConnData(connStr string) (ConnData, error) { @@ -51,14 +67,15 @@ func ToConnData(connStr string) (ConnData, error) { return o, nil } - v, ok := probePostgres(connStr) + v, domain, ok := probePostgres(connStr) if ok { o.typ = DBTYPE_POSTGRES o.str = v + o.domain = domain return o, nil } - v, ok = probeGdbm(connStr) + v, _, ok = probeGdbm(connStr) if ok { o.typ = DBTYPE_GDBM o.str = v diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 617c0ef..374af74 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -55,7 +55,12 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D connStr := ms.conn.String() dbTyp := ms.conn.DbType() if dbTyp == DBTYPE_POSTGRES { - newDb = postgres.NewPgDb() + // TODO: move to vise + err = ensureSchemaExists(ctx, ms.conn) + if err != nil { + return nil, err + } + newDb = postgres.NewPgDb().WithSchema(ms.conn.Domain()) } else if dbTyp == DBTYPE_GDBM { err = ms.ensureDbDir() if err != nil { @@ -66,7 +71,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D } else { return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String()) } - logg.DebugCtxf(ctx, "connecting to db", "conn", connStr) + logg.DebugCtxf(ctx, "connecting to db", "conn", connStr, "conndata", ms.conn) err = newDb.Connect(ctx, connStr) if err != nil { return nil, err @@ -98,15 +103,15 @@ func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *Men } // ensureSchemaExists creates a new schema if it does not exist -func ensureSchemaExists(ctx context.Context, connStr, schema string) error { - conn, err := pgxpool.New(ctx, connStr) +func ensureSchemaExists(ctx context.Context, conn ConnData) error { + h, err := pgxpool.New(ctx, conn.Path()) if err != nil { return fmt.Errorf("failed to connect to the database: %w", err) } - defer conn.Close() + defer h.Close() - query := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", schema) - _, err = conn.Exec(ctx, query) + query := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", conn.Domain()) + _, err = h.Exec(ctx, query) if err != nil { return fmt.Errorf("failed to create schema: %w", err) } From 3fccfaab618f3f68e37750bae225b6079a03ac2b Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Thu, 9 Jan 2025 13:01:28 +0300 Subject: [PATCH 75/81] Replace the connStr if it is not set --- cmd/africastalking/main.go | 6 +++--- cmd/async/main.go | 7 +++---- cmd/http/main.go | 8 ++++---- cmd/main.go | 8 ++++---- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 053eab9..24812a1 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -12,17 +12,17 @@ import ( "syscall" "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/lang" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/http/at" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" - "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -59,7 +59,7 @@ func main() { flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() - if connStr != "" { + if connStr == "" { connStr = config.DbConn } connData, err := storage.ToConnData(connStr) diff --git a/cmd/async/main.go b/cmd/async/main.go index 0b9a233..27db453 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -10,16 +10,16 @@ import ( "syscall" "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/lang" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" - "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -70,7 +70,7 @@ func main() { flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() - if connStr != "" { + if connStr == "" { connStr = config.DbConn } connData, err := storage.ToConnData(connStr) @@ -115,7 +115,6 @@ func main() { os.Exit(1) } - userdataStore, err := menuStorageService.GetUserdataDb(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) diff --git a/cmd/http/main.go b/cmd/http/main.go index 761ee72..6617ca5 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -12,17 +12,17 @@ import ( "syscall" "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/lang" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/internal/handlers" httpserver "git.grassecon.net/urdt/ussd/internal/http" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/remote" - "git.grassecon.net/urdt/ussd/internal/args" ) var ( @@ -58,7 +58,7 @@ func main() { flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() - if connStr != "" { + if connStr == "" { connStr = config.DbConn } connData, err := storage.ToConnData(connStr) @@ -92,7 +92,7 @@ func main() { } menuStorageService := storage.NewMenuStorageService(connData, resourceDir) - + rs, err := menuStorageService.GetResource(ctx) if err != nil { fmt.Fprintf(os.Stderr, err.Error()) diff --git a/cmd/main.go b/cmd/main.go index 8c06094..d2fe0ba 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,14 +8,14 @@ import ( "path" "git.defalsify.org/vise.git/engine" + "git.defalsify.org/vise.git/lang" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/lang" "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" + "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" - "git.grassecon.net/urdt/ussd/internal/args" "git.grassecon.net/urdt/ussd/remote" ) @@ -51,7 +51,7 @@ func main() { flag.Var(&langs, "language", "add symbol resolution for language") flag.Parse() - if connStr != "" { + if connStr == "" { connStr = config.DbConn } connData, err := storage.ToConnData(connStr) @@ -87,7 +87,7 @@ func main() { } menuStorageService := storage.NewMenuStorageService(connData, resourceDir) - + if gettextDir != "" { menuStorageService = menuStorageService.WithGettext(gettextDir, langs.Langs()) } From f660f6c19aa91e50ef4d1753a454541ff39dec2c Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Thu, 9 Jan 2025 13:04:11 +0300 Subject: [PATCH 76/81] add key to hold selected langauge code --- common/db.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/db.go b/common/db.go index 5e2fc4c..2271716 100644 --- a/common/db.go +++ b/common/db.go @@ -57,6 +57,8 @@ const ( DATA_ACTIVE_ADDRESS //Holds count of the number of incorrect PIN attempts DATA_INCORRECT_PIN_ATTEMPTS + //ISO 639 code for the selected language. + DATA_SELECTED_LANGUAGE_CODE ) const ( From 73eb7654085ef5edb326dd18e6e7b9a18d43b5a3 Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Thu, 9 Jan 2025 13:14:46 +0300 Subject: [PATCH 77/81] persist selected language code --- internal/handlers/ussd/menuhandler.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/internal/handlers/ussd/menuhandler.go b/internal/handlers/ussd/menuhandler.go index 607b812..4b6a713 100644 --- a/internal/handlers/ussd/menuhandler.go +++ b/internal/handlers/ussd/menuhandler.go @@ -161,9 +161,12 @@ func (h *Handlers) SetLanguage(ctx context.Context, sym string, input []byte) (r //Fallback to english instead? code = "eng" } - res.FlagSet = append(res.FlagSet, state.FLAG_LANG) + err := h.persistLanguageCode(ctx, code) + if err != nil { + return res, err + } res.Content = code - + res.FlagSet = append(res.FlagSet, state.FLAG_LANG) languageSetFlag, err := h.flagManager.GetFlag("flag_language_set") if err != nil { logg.ErrorCtxf(ctx, "Error setting the languageSetFlag", "error", err) @@ -2173,3 +2176,18 @@ func (h *Handlers) resetIncorrectPINAttempts(ctx context.Context, sessionId stri } return nil } + +// persistLanguageCode persists the selected ISO 639 language code +func (h *Handlers) persistLanguageCode(ctx context.Context, code string) error { + store := h.userdataStore + sessionId, ok := ctx.Value("SessionId").(string) + if !ok { + return fmt.Errorf("missing session") + } + err := store.WriteEntry(ctx, sessionId, common.DATA_SELECTED_LANGUAGE_CODE, []byte(code)) + if err != nil { + logg.ErrorCtxf(ctx, "failed to persist language code", "key", common.DATA_SELECTED_LANGUAGE_CODE, "value", code, "error", err) + return err + } + return nil +} From 3747f87a7ce18c8ccf53f21ef723ffce1e7f73fc Mon Sep 17 00:00:00 2001 From: Carlosokumu Date: Thu, 9 Jan 2025 15:11:29 +0300 Subject: [PATCH 78/81] test language code saving --- internal/handlers/ussd/menuhandler_test.go | 50 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/internal/handlers/ussd/menuhandler_test.go b/internal/handlers/ussd/menuhandler_test.go index af1380d..29f6948 100644 --- a/internal/handlers/ussd/menuhandler_test.go +++ b/internal/handlers/ussd/menuhandler_test.go @@ -775,6 +775,11 @@ func TestSetLanguage(t *testing.T) { log.Fatal(err) } + sessionId := "session123" + ctx, store := InitializeTestStore(t) + + ctx = context.WithValue(ctx, "SessionId", sessionId) + // Define test cases tests := []struct { name string @@ -807,12 +812,13 @@ func TestSetLanguage(t *testing.T) { // Create the Handlers instance with the mock flag manager h := &Handlers{ - flagManager: fm.parser, - st: mockState, + flagManager: fm.parser, + userdataStore: store, + st: mockState, } // Call the method - res, err := h.SetLanguage(context.Background(), "set_language", nil) + res, err := h.SetLanguage(ctx, "set_language", nil) if err != nil { t.Error(err) } @@ -2285,3 +2291,41 @@ func TestResetIncorrectPINAttempts(t *testing.T) { assert.Equal(t, "0", string(incorrectAttempts)) } + +func TestPersistLanguageCode(t *testing.T) { + ctx, store := InitializeTestStore(t) + + sessionId := "session123" + ctx = context.WithValue(ctx, "SessionId", sessionId) + + h := &Handlers{ + userdataStore: store, + } + tests := []struct { + name string + code string + expectedLanguageCode string + }{ + { + name: "Set Default Language (English)", + code: "eng", + expectedLanguageCode: "eng", + }, + { + name: "Set Swahili Language", + code: "swa", + expectedLanguageCode: "swa", + }, + } + + for _, test := range tests { + err := h.persistLanguageCode(ctx, test.code) + if err != nil { + t.Logf(err.Error()) + } + code, err := store.ReadEntry(ctx, sessionId, common.DATA_SELECTED_LANGUAGE_CODE) + + assert.Equal(t, test.expectedLanguageCode, string(code)) + } + +} From 9a6d8e51589f53ac9480ef3951944f1ca6c44b50 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Fri, 10 Jan 2025 13:41:05 +0300 Subject: [PATCH 79/81] Refactored the code to switch between postgres and gdbm, with db cleanup --- internal/testutil/engine.go | 95 +++++++++++++++++++---- menutraversal_test/menu_traversal_test.go | 43 ++-------- 2 files changed, 87 insertions(+), 51 deletions(-) diff --git a/internal/testutil/engine.go b/internal/testutil/engine.go index 8d5d3e4..5d581ba 100644 --- a/internal/testutil/engine.go +++ b/internal/testutil/engine.go @@ -3,6 +3,8 @@ package testutil import ( "context" "fmt" + "log" + "net/url" "os" "path" "path/filepath" @@ -11,34 +13,90 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/config" "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" "git.grassecon.net/urdt/ussd/internal/testutil/testservice" "git.grassecon.net/urdt/ussd/internal/testutil/testtag" "git.grassecon.net/urdt/ussd/remote" + "github.com/jackc/pgx/v5/pgxpool" testdataloader "github.com/peteole/testdata-loader" ) var ( - logg = logging.NewVanilla() - baseDir = testdataloader.GetBasePath() - scriptDir = path.Join(baseDir, "services", "registration") - selectedDatabase = "" - selectedDbSchema = "" + logg = logging.NewVanilla() + baseDir = testdataloader.GetBasePath() + scriptDir = path.Join(baseDir, "services", "registration") + setDbType string + setConnStr string + setDbSchema string ) func init() { initializers.LoadEnvVariablesPath(baseDir) + config.LoadConfig() } // SetDatabase updates the database used by TestEngine -func SetDatabase(dbType string, dbSchema string) { - selectedDatabase = dbType - selectedDbSchema = dbSchema +func SetDatabase(database, connStr, dbSchema string) { + setDbType = database + setConnStr = connStr + setDbSchema = dbSchema +} + +// CleanDatabase removes all test data from the database +func CleanDatabase() { + if setDbType == "postgres" { + ctx := context.Background() + // Update the connection string with the new search path + updatedConnStr, err := updateSearchPath(setConnStr, setDbSchema) + if err != nil { + log.Fatalf("Failed to update search path: %v", err) + } + + dbConn, err := pgxpool.New(ctx, updatedConnStr) + if err != nil { + log.Fatalf("Failed to connect to database for cleanup: %v", err) + } + defer dbConn.Close() + + query := fmt.Sprintf("DELETE FROM %s.kv_vise;", setDbSchema) + _, execErr := dbConn.Exec(ctx, query) + if execErr != nil { + log.Printf("Failed to cleanup table %s.kv_vise: %v", setDbSchema, execErr) + } else { + log.Printf("Successfully cleaned up table %s.kv_vise", setDbSchema) + } + } else { + setConnStr, _ := filepath.Abs(setConnStr) + if err := os.RemoveAll(setConnStr); err != nil { + log.Fatalf("Failed to delete state store %s: %v", setConnStr, err) + } + } +} + +// updateSearchPath updates the search_path (schema) to be used in the connection +func updateSearchPath(connStr string, newSearchPath string) (string, error) { + u, err := url.Parse(connStr) + if err != nil { + return "", fmt.Errorf("invalid connection string: %w", err) + } + + // Parse the query parameters + q := u.Query() + + // Update or add the search_path parameter + q.Set("search_path", newSearchPath) + + // Rebuild the connection string with updated parameters + u.RawQuery = q.Encode() + + return u.String(), nil } func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { + var err error ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) pfp := path.Join(scriptDir, "pp.csv") @@ -52,16 +110,27 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { FlagCount: uint32(128), } - connStr, err := filepath.Abs(".test_state/state.gdbm") - if err != nil { - fmt.Fprintf(os.Stderr, "connstr err: %v", err) - os.Exit(1) + if setDbType == "postgres" { + setConnStr = config.DbConn + setConnStr, err = updateSearchPath(setConnStr, setDbSchema) + if err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } + } else { + setConnStr, err = filepath.Abs(setConnStr) + if err != nil { + fmt.Fprintf(os.Stderr, "connstr err: %v", err) + os.Exit(1) + } } - conn, err := storage.ToConnData(connStr) + + conn, err := storage.ToConnData(setConnStr) if err != nil { fmt.Fprintf(os.Stderr, "connstr parse err: %v", err) os.Exit(1) } + resourceDir := scriptDir menuStorageService := storage.NewMenuStorageService(conn, resourceDir) diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index 4003641..4aee26e 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -4,18 +4,14 @@ import ( "bytes" "context" "flag" - "fmt" "log" "math/rand" - "os" - "path/filepath" "regexp" "testing" "git.grassecon.net/urdt/ussd/internal/testutil" "git.grassecon.net/urdt/ussd/internal/testutil/driver" "github.com/gofrs/uuid" - "github.com/jackc/pgx/v5/pgxpool" ) var ( @@ -27,13 +23,9 @@ var ( var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests") var database = flag.String("db", "gdbm", "Specify the database (gdbm or postgres)") +var connStr = flag.String("conn", ".test_state", "connection string") var dbSchema = flag.String("schema", "test", "Specify the database schema (default test)") -func testStore() string { - v, _ := filepath.Abs(".test_state/state.gdbm") - return v -} - func GenerateSessionId() string { uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g)) v, err := uu.NewV4() @@ -90,37 +82,12 @@ func extractSendAmount(response []byte) string { func TestMain(m *testing.M) { // Parse the flags flag.Parse() - sessionID = GenerateSessionId() - defer func() { - if err := os.RemoveAll(testStore()); err != nil { - log.Fatalf("Failed to delete state store %s: %v", testStore(), err) - } - }() + // set the db + testutil.SetDatabase(*database, *connStr, *dbSchema) - // Set the selected database - testutil.SetDatabase(*database, *dbSchema) - - // Cleanup the schema table after tests - defer func() { - if *database == "postgres" { - ctx := context.Background() - connStr := "postgres://" //storage.BuildConnStr() - dbConn, err := pgxpool.New(ctx, connStr) - if err != nil { - log.Fatalf("Failed to connect to database for cleanup: %v", err) - } - defer dbConn.Close() - - query := fmt.Sprintf("DELETE FROM %s.kv_vise;", *dbSchema) - _, execErr := dbConn.Exec(ctx, query) - if execErr != nil { - log.Printf("Failed to cleanup table %s.kv_vise: %v", *dbSchema, execErr) - } else { - log.Printf("Successfully cleaned up table %s.kv_vise", *dbSchema) - } - } - }() + // Cleanup the db after tests + defer testutil.CleanDatabase() m.Run() } From a19ace85f87c727a32bde8d244f3913452d34342 Mon Sep 17 00:00:00 2001 From: konstantinmds Date: Fri, 10 Jan 2025 13:41:23 +0100 Subject: [PATCH 80/81] refactor: rename files to snake_case --- internal/handlers/application/{menuhandler.go => menu_handler.go} | 0 .../application/{menuhandler_test.go => menu_handler_test.go} | 0 internal/storage/{storageservice.go => storage_service.go} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename internal/handlers/application/{menuhandler.go => menu_handler.go} (100%) rename internal/handlers/application/{menuhandler_test.go => menu_handler_test.go} (100%) rename internal/storage/{storageservice.go => storage_service.go} (100%) diff --git a/internal/handlers/application/menuhandler.go b/internal/handlers/application/menu_handler.go similarity index 100% rename from internal/handlers/application/menuhandler.go rename to internal/handlers/application/menu_handler.go diff --git a/internal/handlers/application/menuhandler_test.go b/internal/handlers/application/menu_handler_test.go similarity index 100% rename from internal/handlers/application/menuhandler_test.go rename to internal/handlers/application/menu_handler_test.go diff --git a/internal/storage/storageservice.go b/internal/storage/storage_service.go similarity index 100% rename from internal/storage/storageservice.go rename to internal/storage/storage_service.go From 3d35a5de789102cec661e7002a7ae2ce29c4b043 Mon Sep 17 00:00:00 2001 From: konstantinmds Date: Fri, 10 Jan 2025 13:43:49 +0100 Subject: [PATCH 81/81] rename files to snake case --- internal/handlers/{handlerservice.go => handler_service.go} | 0 .../httpmocks/{requesthandlermock.go => request_handler_mock.go} | 0 .../httpmocks/{requestparsermock.go => request_parser_mock.go} | 0 .../testutil/mocks/httpmocks/{writermock.go => writer_mock.go} | 0 internal/testutil/mocks/{servicemock.go => service_mock.go} | 0 .../testservice/{accountservice.go => account_service.go} | 0 models/{accountresponse.go => account_response.go} | 0 models/{balanceresponse.go => balance_response.go} | 0 models/{trackstatusresponse.go => track_status_response.go} | 0 remote/{accountservice.go => account_service.go} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename internal/handlers/{handlerservice.go => handler_service.go} (100%) rename internal/testutil/mocks/httpmocks/{requesthandlermock.go => request_handler_mock.go} (100%) rename internal/testutil/mocks/httpmocks/{requestparsermock.go => request_parser_mock.go} (100%) rename internal/testutil/mocks/httpmocks/{writermock.go => writer_mock.go} (100%) rename internal/testutil/mocks/{servicemock.go => service_mock.go} (100%) rename internal/testutil/testservice/{accountservice.go => account_service.go} (100%) rename models/{accountresponse.go => account_response.go} (100%) rename models/{balanceresponse.go => balance_response.go} (100%) rename models/{trackstatusresponse.go => track_status_response.go} (100%) rename remote/{accountservice.go => account_service.go} (100%) diff --git a/internal/handlers/handlerservice.go b/internal/handlers/handler_service.go similarity index 100% rename from internal/handlers/handlerservice.go rename to internal/handlers/handler_service.go diff --git a/internal/testutil/mocks/httpmocks/requesthandlermock.go b/internal/testutil/mocks/httpmocks/request_handler_mock.go similarity index 100% rename from internal/testutil/mocks/httpmocks/requesthandlermock.go rename to internal/testutil/mocks/httpmocks/request_handler_mock.go diff --git a/internal/testutil/mocks/httpmocks/requestparsermock.go b/internal/testutil/mocks/httpmocks/request_parser_mock.go similarity index 100% rename from internal/testutil/mocks/httpmocks/requestparsermock.go rename to internal/testutil/mocks/httpmocks/request_parser_mock.go diff --git a/internal/testutil/mocks/httpmocks/writermock.go b/internal/testutil/mocks/httpmocks/writer_mock.go similarity index 100% rename from internal/testutil/mocks/httpmocks/writermock.go rename to internal/testutil/mocks/httpmocks/writer_mock.go diff --git a/internal/testutil/mocks/servicemock.go b/internal/testutil/mocks/service_mock.go similarity index 100% rename from internal/testutil/mocks/servicemock.go rename to internal/testutil/mocks/service_mock.go diff --git a/internal/testutil/testservice/accountservice.go b/internal/testutil/testservice/account_service.go similarity index 100% rename from internal/testutil/testservice/accountservice.go rename to internal/testutil/testservice/account_service.go diff --git a/models/accountresponse.go b/models/account_response.go similarity index 100% rename from models/accountresponse.go rename to models/account_response.go diff --git a/models/balanceresponse.go b/models/balance_response.go similarity index 100% rename from models/balanceresponse.go rename to models/balance_response.go diff --git a/models/trackstatusresponse.go b/models/track_status_response.go similarity index 100% rename from models/trackstatusresponse.go rename to models/track_status_response.go diff --git a/remote/accountservice.go b/remote/account_service.go similarity index 100% rename from remote/accountservice.go rename to remote/account_service.go