From 51122d0fc52bcd7b809216e9702c2e1d4924e212 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Tue, 15 Oct 2024 22:31:37 +0300 Subject: [PATCH 01/12] added an env.example file --- .env.example | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3bfd724 --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +#Serve Http +PORT=7123 +HOST=127.0.0.1 + +#PostgreSQL +DB_HOST=localhost +DB_USER=postgres +DB_PASSWORD=strongpass +DB_NAME=urdt_ussd +DB_PORT=5432 +DB_SSLMODE=disable +DB_TIMEZONE=Africa/Nairobi + +#External API Calls +CREATE_ACCOUNT_URL="https://custodial.sarafu.africa/api/account/create" +TRACK_STATUS_URL="https://custodial.sarafu.africa/api/track/" +BALANCE_URL="https://custodial.sarafu.africa/api/account/status/" From 09eac03e10351829f45245ee6316c9b383af84ba Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Tue, 15 Oct 2024 23:41:16 +0300 Subject: [PATCH 02/12] use env variables --- .env.example | 6 +++--- cmd/africastalking/main.go | 13 ++++++++++-- cmd/async/main.go | 12 +++++++++-- cmd/http/main.go | 12 +++++++++-- cmd/main.go | 8 ++++++++ config/config.go | 16 ++++++++++----- go.mod | 2 ++ go.sum | 2 ++ initializers/loadEnvVariables.go | 34 ++++++++++++++++++++++++++++++++ 9 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 initializers/loadEnvVariables.go diff --git a/.env.example b/.env.example index 3bfd724..97d3317 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,6 @@ DB_SSLMODE=disable DB_TIMEZONE=Africa/Nairobi #External API Calls -CREATE_ACCOUNT_URL="https://custodial.sarafu.africa/api/account/create" -TRACK_STATUS_URL="https://custodial.sarafu.africa/api/track/" -BALANCE_URL="https://custodial.sarafu.africa/api/account/status/" +CREATE_ACCOUNT_URL=https://custodial.sarafu.africa/api/account/create +TRACK_STATUS_URL=https://custodial.sarafu.africa/api/track/ +BALANCE_URL=https://custodial.sarafu.africa/api/account/status/ diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index c24c4b1..3f1c723 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -16,6 +16,8 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" httpserver "git.grassecon.net/urdt/ussd/internal/http" "git.grassecon.net/urdt/ussd/internal/storage" @@ -26,6 +28,11 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + fmt.Println("Running init") + initializers.LoadEnvVariables() +} + type atRequestParser struct{} func (arp *atRequestParser) GetSessionId(rq any) (string, error) { @@ -65,6 +72,8 @@ func (arp *atRequestParser) GetInput(rq any) ([]byte, error) { } func main() { + config.LoadConfig() + var dbDir string var resourceDir string var size uint @@ -75,8 +84,8 @@ func main() { flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") - flag.StringVar(&host, "h", "127.0.0.1", "http host") - flag.UintVar(&port, "p", 7123, "http port") + flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") + flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) diff --git a/cmd/async/main.go b/cmd/async/main.go index 09236fd..db58857 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -13,6 +13,8 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" ) @@ -22,6 +24,10 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + initializers.LoadEnvVariables() +} + type asyncRequestParser struct { sessionId string input []byte @@ -36,6 +42,8 @@ func (p *asyncRequestParser) GetInput(r any) ([]byte, error) { } func main() { + config.LoadConfig() + var sessionId string var dbDir string var resourceDir string @@ -48,8 +56,8 @@ func main() { flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") - flag.StringVar(&host, "h", "127.0.0.1", "http host") - flag.UintVar(&port, "p", 7123, "http port") + flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") + flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) diff --git a/cmd/http/main.go b/cmd/http/main.go index 6b868ed..166ab75 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -15,6 +15,8 @@ import ( "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" httpserver "git.grassecon.net/urdt/ussd/internal/http" "git.grassecon.net/urdt/ussd/internal/storage" @@ -25,7 +27,13 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + initializers.LoadEnvVariables() +} + func main() { + config.LoadConfig() + var dbDir string var resourceDir string var size uint @@ -36,8 +44,8 @@ func main() { flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") - flag.StringVar(&host, "h", "127.0.0.1", "http host") - flag.UintVar(&port, "p", 7123, "http port") + flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") + flag.UintVar(&port, "p", initializers.GetEnvUint("PORT", 7123), "http port") flag.Parse() logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) diff --git a/cmd/main.go b/cmd/main.go index 9db5e0a..e42306a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,6 +10,8 @@ import ( "git.defalsify.org/vise.git/engine" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" + "git.grassecon.net/urdt/ussd/config" + "git.grassecon.net/urdt/ussd/initializers" "git.grassecon.net/urdt/ussd/internal/handlers" "git.grassecon.net/urdt/ussd/internal/storage" ) @@ -19,7 +21,13 @@ var ( scriptDir = path.Join("services", "registration") ) +func init() { + initializers.LoadEnvVariables() +} + func main() { + config.LoadConfig() + var dbDir string var size uint var sessionId string diff --git a/config/config.go b/config/config.go index 0571503..dba0da7 100644 --- a/config/config.go +++ b/config/config.go @@ -1,10 +1,16 @@ package config +import "git.grassecon.net/urdt/ussd/initializers" - -const ( - CreateAccountURL = "https://custodial.sarafu.africa/api/account/create" - TrackStatusURL = "https://custodial.sarafu.africa/api/track/" - BalanceURL = "https://custodial.sarafu.africa/api/account/status/" +var ( + CreateAccountURL string + TrackStatusURL string + BalanceURL string ) +// LoadConfig initializes the configuration values after environment variables are loaded. +func LoadConfig() { + CreateAccountURL = initializers.GetEnv("CREATE_ACCOUNT_URL", "https://custodial.sarafu.africa/api/account/create") + TrackStatusURL = initializers.GetEnv("TRACK_STATUS_URL", "https://custodial.sarafu.africa/api/track/") + BalanceURL = initializers.GetEnv("BALANCE_URL", "https://custodial.sarafu.africa/api/account/status/") +} diff --git a/go.mod b/go.mod index c4c5167..1e8f630 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,8 @@ require ( gopkg.in/leonelquinteros/gotext.v1 v1.3.1 ) +require github.com/joho/godotenv v1.5.1 // indirect + require ( github.com/alecthomas/participle/v2 v2.0.0 // indirect github.com/alecthomas/repr v0.2.0 // indirect diff --git a/go.sum b/go.sum index ed5636f..34ee911 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/graygnuorg/go-gdbm v0.0.0-20220711140707-71387d66dce4 h1:U4kkNYryi/qf github.com/graygnuorg/go-gdbm v0.0.0-20220711140707-71387d66dce4/go.mod h1:zpZDgZFzeq9s0MIeB1P50NIEWDFFHSFBohI/NbaTD/Y= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/mattn/kinako v0.0.0-20170717041458-332c0a7e205a h1:0Q3H0YXzMHiciXtRcM+j0jiCe8WKPQHoRgQiRTnfcLY= github.com/mattn/kinako v0.0.0-20170717041458-332c0a7e205a/go.mod h1:CdTTBOYzS5E4mWS1N8NWP6AHI19MP0A2B18n3hLzRMk= github.com/peteole/testdata-loader v0.3.0 h1:8jckE9KcyNHgyv/VPoaljvKZE0Rqr8+dPVYH6rfNr9I= diff --git a/initializers/loadEnvVariables.go b/initializers/loadEnvVariables.go new file mode 100644 index 0000000..4ea5980 --- /dev/null +++ b/initializers/loadEnvVariables.go @@ -0,0 +1,34 @@ +package initializers + +import ( + "log" + "os" + "strconv" + + "github.com/joho/godotenv" +) + +func LoadEnvVariables() { + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file") + } +} + +// Helper to get environment variables with a default fallback +func GetEnv(key, defaultVal string) string { + if value, exists := os.LookupEnv(key); exists { + return value + } + return defaultVal +} + +// Helper to safely convert environment variables to uint +func GetEnvUint(key string, defaultVal uint) uint { + if value, exists := os.LookupEnv(key); exists { + if parsed, err := strconv.Atoi(value); err == nil && parsed >= 0 { + return uint(parsed) + } + } + return defaultVal +} From d8db8df64364ae7b09f12c18fdcf353942cc8f09 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Tue, 15 Oct 2024 23:42:39 +0300 Subject: [PATCH 03/12] use go-vise v0.2.0 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1e8f630..dcd1d13 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.22.6 require ( - git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb + git.defalsify.org/vise.git v0.2.0 github.com/alecthomas/assert/v2 v2.2.2 github.com/peteole/testdata-loader v0.3.0 gopkg.in/leonelquinteros/gotext.v1 v1.3.1 diff --git a/go.sum b/go.sum index 34ee911..0f7275c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb h1:6P4kxihcwMjDKzvUFC6t2zGNb7MDW+l/ACGlSAN1N8Y= git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M= +git.defalsify.org/vise.git v0.2.0 h1:X2ZgiGRq4C+9qOlDMP0b/oE5QHjVQNT4aEFZB88ST0Q= +git.defalsify.org/vise.git v0.2.0/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= From 1aeb18379c4b078a07ecfb307aae7f164ad0eb3a Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Thu, 17 Oct 2024 12:46:20 +0300 Subject: [PATCH 04/12] added a db flag and Database to the context --- cmd/africastalking/main.go | 3 +++ cmd/async/main.go | 3 +++ cmd/http/main.go | 3 +++ cmd/main.go | 3 +++ 4 files changed, 12 insertions(+) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index 3f1c723..56a6c81 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -77,11 +77,13 @@ func main() { var dbDir string var resourceDir string var size uint + var database string var engineDebug bool var host string var port uint flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") + flag.StringVar(&database, "db", "gdbm", "database to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -91,6 +93,7 @@ func main() { logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() + ctx = context.WithValue(ctx, "Database", database) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/async/main.go b/cmd/async/main.go index db58857..e815c96 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -48,12 +48,14 @@ func main() { var dbDir string var resourceDir string var size uint + var database string var engineDebug bool var host string var port uint flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") + flag.StringVar(&database, "db", "gdbm", "database to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -63,6 +65,7 @@ func main() { logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId) ctx := context.Background() + ctx = context.WithValue(ctx, "Database", database) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/http/main.go b/cmd/http/main.go index 166ab75..774eb87 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -37,11 +37,13 @@ func main() { var dbDir string var resourceDir string var size uint + var database string var engineDebug bool var host string var port uint flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.StringVar(&resourceDir, "resourcedir", path.Join("services", "registration"), "resource dir") + flag.StringVar(&database, "db", "gdbm", "database to be used") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", initializers.GetEnv("HOST", "127.0.0.1"), "http host") @@ -51,6 +53,7 @@ func main() { logg.Infof("start command", "dbdir", dbDir, "resourcedir", resourceDir, "outputsize", size) ctx := context.Background() + ctx = context.WithValue(ctx, "Database", database) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ diff --git a/cmd/main.go b/cmd/main.go index e42306a..be2bea3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -31,8 +31,10 @@ func main() { var dbDir string var size uint var sessionId string + var database string var engineDebug bool flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") + flag.StringVar(&database, "db", "gdbm", "database to be used") flag.StringVar(&dbDir, "dbdir", ".state", "database dir to read from") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") @@ -42,6 +44,7 @@ func main() { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) + ctx = context.WithValue(ctx, "Database", database) pfp := path.Join(scriptDir, "pp.csv") cfg := engine.Config{ From 4968cdff37cc8a9ab5e3f20db99fce6c684e25e6 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Thu, 17 Oct 2024 12:47:57 +0300 Subject: [PATCH 05/12] switch to postgres once the flag is set --- internal/storage/postgres.go | 115 +++++++++++++++++++++++++++++ internal/storage/storageservice.go | 53 ++++++++++--- 2 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 internal/storage/postgres.go diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go new file mode 100644 index 0000000..ed4a711 --- /dev/null +++ b/internal/storage/postgres.go @@ -0,0 +1,115 @@ +package storage + +import ( + "context" + + "git.defalsify.org/vise.git/db" + postgres "git.defalsify.org/vise.git/db/postgres" + "git.defalsify.org/vise.git/lang" +) + +var ( + pdbC map[string]chan db.Db +) + +type ThreadPostgresDb struct { + db db.Db + connStr string +} + +func NewThreadPostgresDb() *ThreadPostgresDb { + if pdbC == nil { + pdbC = make(map[string]chan db.Db) + } + return &ThreadPostgresDb{} +} + +func (tpdb *ThreadPostgresDb) Connect(ctx context.Context, connStr string) error { + var ok bool + _, ok = pdbC[connStr] + if ok { + logg.WarnCtxf(ctx, "already registered thread postgres, skipping", "connStr", connStr) + return nil + } + postgresdb := postgres.NewPgDb().WithSchema("public") + err := postgresdb.Connect(ctx, connStr) + if err != nil { + return err + } + pdbC[connStr] = make(chan db.Db, 1) + pdbC[connStr] <- postgresdb + tpdb.connStr = connStr + return nil +} + +func (tpdb *ThreadPostgresDb) reserve() { + if tpdb.db == nil { + tpdb.db = <-pdbC[tpdb.connStr] + } +} + +func (tpdb *ThreadPostgresDb) release() { + if tpdb.db == nil { + return + } + pdbC[tpdb.connStr] <- tpdb.db + tpdb.db = nil +} + +func (tpdb *ThreadPostgresDb) SetPrefix(pfx uint8) { + tpdb.reserve() + tpdb.db.SetPrefix(pfx) +} + +func (tpdb *ThreadPostgresDb) SetSession(sessionId string) { + tpdb.reserve() + tpdb.db.SetSession(sessionId) +} + +func (tpdb *ThreadPostgresDb) SetLanguage(lng *lang.Language) { + tpdb.reserve() + tpdb.db.SetLanguage(lng) +} + +func (tpdb *ThreadPostgresDb) Safe() bool { + tpdb.reserve() + v := tpdb.db.Safe() + tpdb.release() + return v +} + +func (tpdb *ThreadPostgresDb) Prefix() uint8 { + tpdb.reserve() + v := tpdb.db.Prefix() + tpdb.release() + return v +} + +func (tpdb *ThreadPostgresDb) SetLock(typ uint8, locked bool) error { + tpdb.reserve() + err := tpdb.db.SetLock(typ, locked) + tpdb.release() + return err +} + +func (tpdb *ThreadPostgresDb) Put(ctx context.Context, key []byte, val []byte) error { + tpdb.reserve() + err := tpdb.db.Put(ctx, key, val) + tpdb.release() + return err +} + +func (tpdb *ThreadPostgresDb) Get(ctx context.Context, key []byte) ([]byte, error) { + tpdb.reserve() + v, err := tpdb.db.Get(ctx, key) + tpdb.release() + return v, err +} + +func (tpdb *ThreadPostgresDb) Close() error { + tpdb.reserve() + close(pdbC[tpdb.connStr]) + err := tpdb.db.Close() + tpdb.db = nil + return err +} diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 07bccd6..d99e0ca 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -8,14 +8,15 @@ import ( "git.defalsify.org/vise.git/db" fsdb "git.defalsify.org/vise.git/db/fs" + "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" - "git.defalsify.org/vise.git/logging" + "git.grassecon.net/urdt/ussd/initializers" ) var ( logg = logging.NewVanilla().WithDomain("storage") -) +) type StorageService interface { GetPersister(ctx context.Context) (*persist.Persister, error) @@ -24,17 +25,30 @@ type StorageService interface { EnsureDbDir() error } -type MenuStorageService struct{ - dbDir string - resourceDir string +type MenuStorageService struct { + dbDir string + resourceDir string resourceStore db.Db - stateStore db.Db + stateStore db.Db userDataStore db.Db } +func buildConnStr() string { + host := initializers.GetEnv("DB_HOST", "localhost") + user := initializers.GetEnv("DB_USER", "postgres") + password := initializers.GetEnv("DB_PASSWORD", "") + dbName := initializers.GetEnv("DB_NAME", "") + port := initializers.GetEnv("DB_PORT", "5432") + + return fmt.Sprintf( + "postgres://%s:%s@%s:%s/%s", + user, password, host, port, dbName, + ) +} + func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService { return &MenuStorageService{ - dbDir: dbDir, + dbDir: dbDir, resourceDir: resourceDir, } } @@ -52,12 +66,27 @@ func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persis } func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) { - ms.userDataStore = NewThreadGdbmDb() - storeFile := path.Join(ms.dbDir, "userdata.gdbm") - err := ms.userDataStore.Connect(ctx, storeFile) - if err != nil { - return nil, err + database, ok := ctx.Value("Database").(string) + if !ok { + fmt.Println("The database is not set") } + + if database == "postgres" { + ms.userDataStore = NewThreadPostgresDb() + connStr := buildConnStr() + err := ms.userDataStore.Connect(ctx, connStr) + if err != nil { + return nil, err + } + } else { + ms.userDataStore = NewThreadGdbmDb() + storeFile := path.Join(ms.dbDir, "userdata.gdbm") + err := ms.userDataStore.Connect(ctx, storeFile) + if err != nil { + return nil, err + } + } + return ms.userDataStore, nil } From 6727bd3769168bacecbb940930e9373710e2065d Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Thu, 17 Oct 2024 12:55:47 +0300 Subject: [PATCH 06/12] polished code --- cmd/africastalking/main.go | 1 - internal/storage/storageservice.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index b810639..c52094a 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -29,7 +29,6 @@ var ( ) func init() { - fmt.Println("Running init") initializers.LoadEnvVariables() } diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index d99e0ca..ec2b710 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -68,7 +68,7 @@ func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persis func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) { database, ok := ctx.Value("Database").(string) if !ok { - fmt.Println("The database is not set") + return nil, fmt.Errorf("failed to select the database") } if database == "postgres" { From fd8bfae8c73f454e5cbb140d61dd9e2ef03557a2 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Thu, 17 Oct 2024 15:35:35 +0300 Subject: [PATCH 07/12] updated the vise version to include the updated column --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index dcd1d13..3f68e5d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd go 1.22.6 require ( - git.defalsify.org/vise.git v0.2.0 + git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b github.com/alecthomas/assert/v2 v2.2.2 github.com/peteole/testdata-loader v0.3.0 gopkg.in/leonelquinteros/gotext.v1 v1.3.1 diff --git a/go.sum b/go.sum index 0f7275c..9615d3b 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb h1:6P4kxihc git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M= git.defalsify.org/vise.git v0.2.0 h1:X2ZgiGRq4C+9qOlDMP0b/oE5QHjVQNT4aEFZB88ST0Q= git.defalsify.org/vise.git v0.2.0/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M= +git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b h1:dxBplsIlzJHV+5EH+gzB+w08Blt7IJbb2jeRe1OEjLU= +git.defalsify.org/vise.git v0.2.1-0.20241017112704-307fa6fcdc6b/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= From 00a2beae503980693cf6a8d7f458a006671a66a7 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Sat, 19 Oct 2024 13:41:19 +0300 Subject: [PATCH 08/12] rename file --- initializers/{loadEnvVariables.go => load.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename initializers/{loadEnvVariables.go => load.go} (100%) diff --git a/initializers/loadEnvVariables.go b/initializers/load.go similarity index 100% rename from initializers/loadEnvVariables.go rename to initializers/load.go From f13f5996c16c64a06713f7c9cf347edf983635f9 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Sat, 19 Oct 2024 14:51:41 +0300 Subject: [PATCH 09/12] remove thread abstraction from postgres --- internal/storage/postgres.go | 133 ++++++++++------------------- internal/storage/storageservice.go | 2 +- 2 files changed, 47 insertions(+), 88 deletions(-) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index ed4a711..9422a26 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -8,27 +8,18 @@ import ( "git.defalsify.org/vise.git/lang" ) -var ( - pdbC map[string]chan db.Db -) - -type ThreadPostgresDb struct { +type PostgresDb struct { db db.Db connStr string } -func NewThreadPostgresDb() *ThreadPostgresDb { - if pdbC == nil { - pdbC = make(map[string]chan db.Db) - } - return &ThreadPostgresDb{} +func NewPostgresDb() *PostgresDb { + return &PostgresDb{} } -func (tpdb *ThreadPostgresDb) Connect(ctx context.Context, connStr string) error { - var ok bool - _, ok = pdbC[connStr] - if ok { - logg.WarnCtxf(ctx, "already registered thread postgres, skipping", "connStr", connStr) +func (pdb *PostgresDb) Connect(ctx context.Context, connStr string) error { + if pdb.db != nil { + logg.WarnCtxf(ctx, "already connected, skipping", "connStr", connStr) return nil } postgresdb := postgres.NewPgDb().WithSchema("public") @@ -36,80 +27,48 @@ func (tpdb *ThreadPostgresDb) Connect(ctx context.Context, connStr string) error if err != nil { return err } - pdbC[connStr] = make(chan db.Db, 1) - pdbC[connStr] <- postgresdb - tpdb.connStr = connStr + pdb.db = postgresdb + pdb.connStr = connStr return nil } -func (tpdb *ThreadPostgresDb) reserve() { - if tpdb.db == nil { - tpdb.db = <-pdbC[tpdb.connStr] +func (pdb *PostgresDb) SetPrefix(pfx uint8) { + pdb.db.SetPrefix(pfx) +} + +func (pdb *PostgresDb) SetSession(sessionId string) { + pdb.db.SetSession(sessionId) +} + +func (pdb *PostgresDb) SetLanguage(lng *lang.Language) { + pdb.db.SetLanguage(lng) +} + +func (pdb *PostgresDb) Safe() bool { + return pdb.db.Safe() +} + +func (pdb *PostgresDb) Prefix() uint8 { + return pdb.db.Prefix() +} + +func (pdb *PostgresDb) SetLock(typ uint8, locked bool) error { + return pdb.db.SetLock(typ, locked) +} + +func (pdb *PostgresDb) Put(ctx context.Context, key []byte, val []byte) error { + return pdb.db.Put(ctx, key, val) +} + +func (pdb *PostgresDb) Get(ctx context.Context, key []byte) ([]byte, error) { + return pdb.db.Get(ctx, key) +} + +func (pdb *PostgresDb) Close() error { + if pdb.db == nil { + return nil } -} - -func (tpdb *ThreadPostgresDb) release() { - if tpdb.db == nil { - return - } - pdbC[tpdb.connStr] <- tpdb.db - tpdb.db = nil -} - -func (tpdb *ThreadPostgresDb) SetPrefix(pfx uint8) { - tpdb.reserve() - tpdb.db.SetPrefix(pfx) -} - -func (tpdb *ThreadPostgresDb) SetSession(sessionId string) { - tpdb.reserve() - tpdb.db.SetSession(sessionId) -} - -func (tpdb *ThreadPostgresDb) SetLanguage(lng *lang.Language) { - tpdb.reserve() - tpdb.db.SetLanguage(lng) -} - -func (tpdb *ThreadPostgresDb) Safe() bool { - tpdb.reserve() - v := tpdb.db.Safe() - tpdb.release() - return v -} - -func (tpdb *ThreadPostgresDb) Prefix() uint8 { - tpdb.reserve() - v := tpdb.db.Prefix() - tpdb.release() - return v -} - -func (tpdb *ThreadPostgresDb) SetLock(typ uint8, locked bool) error { - tpdb.reserve() - err := tpdb.db.SetLock(typ, locked) - tpdb.release() + err := pdb.db.Close() + pdb.db = nil return err -} - -func (tpdb *ThreadPostgresDb) Put(ctx context.Context, key []byte, val []byte) error { - tpdb.reserve() - err := tpdb.db.Put(ctx, key, val) - tpdb.release() - return err -} - -func (tpdb *ThreadPostgresDb) Get(ctx context.Context, key []byte) ([]byte, error) { - tpdb.reserve() - v, err := tpdb.db.Get(ctx, key) - tpdb.release() - return v, err -} - -func (tpdb *ThreadPostgresDb) Close() error { - tpdb.reserve() - close(pdbC[tpdb.connStr]) - err := tpdb.db.Close() - tpdb.db = nil - return err -} +} \ No newline at end of file diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index ec2b710..01a122c 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -72,7 +72,7 @@ func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) } if database == "postgres" { - ms.userDataStore = NewThreadPostgresDb() + ms.userDataStore = NewPostgresDb() connStr := buildConnStr() err := ms.userDataStore.Connect(ctx, connStr) if err != nil { From a40fc37da4ba015e3505e2f2bd387c16706707a6 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Sat, 19 Oct 2024 15:27:23 +0300 Subject: [PATCH 10/12] implement postgres for the state store --- internal/storage/storageservice.go | 81 ++++++++++++++++++------------ 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index 01a122c..c616019 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -53,40 +53,58 @@ func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService } } -func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) { - ms.stateStore = NewThreadGdbmDb() - storeFile := path.Join(ms.dbDir, "state.gdbm") - err := ms.stateStore.Connect(ctx, storeFile) - if err != nil { - return nil, err - } - pr := persist.NewPersister(ms.stateStore) - logg.TraceCtxf(ctx, "menu storage service", "persist", pr, "store", ms.stateStore) - return pr, nil -} - -func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) { +func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, fileName string) (db.Db, error) { database, ok := ctx.Value("Database").(string) if !ok { return nil, fmt.Errorf("failed to select the database") } - if database == "postgres" { - ms.userDataStore = NewPostgresDb() - connStr := buildConnStr() - err := ms.userDataStore.Connect(ctx, connStr) - if err != nil { - return nil, err - } - } else { - ms.userDataStore = NewThreadGdbmDb() - storeFile := path.Join(ms.dbDir, "userdata.gdbm") - err := ms.userDataStore.Connect(ctx, storeFile) - if err != nil { - return nil, err - } + if existingDb != nil { + return existingDb, nil } + var newDb db.Db + var err error + + if database == "postgres" { + newDb = NewPostgresDb() + connStr := buildConnStr() + err = newDb.Connect(ctx, connStr) + } else { + newDb = NewThreadGdbmDb() + storeFile := path.Join(ms.dbDir, fileName) + err = newDb.Connect(ctx, storeFile) + } + + if err != nil { + return nil, err + } + + return newDb, nil +} + +func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) { + stateStore, err := ms.GetStateStore(ctx) + if err != nil { + return nil, err + } + + pr := persist.NewPersister(stateStore) + logg.TraceCtxf(ctx, "menu storage service", "persist", pr, "store", stateStore) + return pr, nil +} + +func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) { + if ms.userDataStore != nil { + return ms.userDataStore, nil + } + + userDataStore, err := ms.getOrCreateDb(ctx, ms.userDataStore, "userdata.gdbm") + if err != nil { + return nil, err + } + + ms.userDataStore = userDataStore return ms.userDataStore, nil } @@ -102,14 +120,15 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error) { if ms.stateStore != nil { - panic("set up store when already exists") + return ms.stateStore, nil } - ms.stateStore = NewThreadGdbmDb() - storeFile := path.Join(ms.dbDir, "state.gdbm") - err := ms.stateStore.Connect(ctx, storeFile) + + stateStore, err := ms.getOrCreateDb(ctx, ms.stateStore, "state.gdbm") if err != nil { return nil, err } + + ms.stateStore = stateStore return ms.stateStore, nil } From 9f562fe53e58ca9511335e4c8aae15ec58b9ac57 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Sat, 19 Oct 2024 23:06:58 +0300 Subject: [PATCH 11/12] use postgres directly from go-vise --- internal/storage/postgres.go | 74 ------------------------------ internal/storage/storageservice.go | 3 +- 2 files changed, 2 insertions(+), 75 deletions(-) delete mode 100644 internal/storage/postgres.go diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go deleted file mode 100644 index 9422a26..0000000 --- a/internal/storage/postgres.go +++ /dev/null @@ -1,74 +0,0 @@ -package storage - -import ( - "context" - - "git.defalsify.org/vise.git/db" - postgres "git.defalsify.org/vise.git/db/postgres" - "git.defalsify.org/vise.git/lang" -) - -type PostgresDb struct { - db db.Db - connStr string -} - -func NewPostgresDb() *PostgresDb { - return &PostgresDb{} -} - -func (pdb *PostgresDb) Connect(ctx context.Context, connStr string) error { - if pdb.db != nil { - logg.WarnCtxf(ctx, "already connected, skipping", "connStr", connStr) - return nil - } - postgresdb := postgres.NewPgDb().WithSchema("public") - err := postgresdb.Connect(ctx, connStr) - if err != nil { - return err - } - pdb.db = postgresdb - pdb.connStr = connStr - return nil -} - -func (pdb *PostgresDb) SetPrefix(pfx uint8) { - pdb.db.SetPrefix(pfx) -} - -func (pdb *PostgresDb) SetSession(sessionId string) { - pdb.db.SetSession(sessionId) -} - -func (pdb *PostgresDb) SetLanguage(lng *lang.Language) { - pdb.db.SetLanguage(lng) -} - -func (pdb *PostgresDb) Safe() bool { - return pdb.db.Safe() -} - -func (pdb *PostgresDb) Prefix() uint8 { - return pdb.db.Prefix() -} - -func (pdb *PostgresDb) SetLock(typ uint8, locked bool) error { - return pdb.db.SetLock(typ, locked) -} - -func (pdb *PostgresDb) Put(ctx context.Context, key []byte, val []byte) error { - return pdb.db.Put(ctx, key, val) -} - -func (pdb *PostgresDb) Get(ctx context.Context, key []byte) ([]byte, error) { - return pdb.db.Get(ctx, key) -} - -func (pdb *PostgresDb) Close() error { - if pdb.db == nil { - return nil - } - err := pdb.db.Close() - pdb.db = nil - return err -} \ No newline at end of file diff --git a/internal/storage/storageservice.go b/internal/storage/storageservice.go index c616019..9fa1839 100644 --- a/internal/storage/storageservice.go +++ b/internal/storage/storageservice.go @@ -8,6 +8,7 @@ import ( "git.defalsify.org/vise.git/db" fsdb "git.defalsify.org/vise.git/db/fs" + "git.defalsify.org/vise.git/db/postgres" "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/persist" "git.defalsify.org/vise.git/resource" @@ -67,7 +68,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D var err error if database == "postgres" { - newDb = NewPostgresDb() + newDb = postgres.NewPgDb() connStr := buildConnStr() err = newDb.Connect(ctx, connStr) } else { From 415c8074646cc3bacba0e1b112ef4c52a702e771 Mon Sep 17 00:00:00 2001 From: alfred-mk Date: Mon, 21 Oct 2024 09:22:07 +0300 Subject: [PATCH 12/12] include the Database in context --- internal/testutil/TestEngine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testutil/TestEngine.go b/internal/testutil/TestEngine.go index 2432a3f..75ac817 100644 --- a/internal/testutil/TestEngine.go +++ b/internal/testutil/TestEngine.go @@ -25,6 +25,7 @@ var ( func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) + ctx = context.WithValue(ctx, "Database", "gdbm") pfp := path.Join(scriptDir, "pp.csv") var eventChannel = make(chan bool)