Merge remote-tracking branch 'remotes/origin/master' into menu-profile-edit

cmd/africastalking/main.go

@@ -98,21 +98,20 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.MenuStorageService{}
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
-
+	rs, err := menuStorageService.GetResource(ctx)
-	rs, err := menuStorageService.GetResource(scriptDir, ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	err = menuStorageService.EnsureDbDir(dbDir)
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	userdataStore := menuStorageService.GetUserdataDb(dbDir, ctx)
+	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -125,7 +124,7 @@ func main() {
 	}
 
 	lhs, err := handlers.NewLocalHandlerService(pfp, true, dbResource, cfg, rs)
-	lhs.WithDataStore(&userdataStore)
+	lhs.SetDataStore(&userdataStore)
 
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
@@ -138,7 +137,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	stateStore, err := menuStorageService.GetStateStore(dbDir, ctx)
+	stateStore, err := menuStorageService.GetStateStore(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)

cmd/async/main.go

@@ -71,20 +71,20 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.MenuStorageService{}
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
-	rs, err := menuStorageService.GetResource(scriptDir, ctx)
+	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	err = menuStorageService.EnsureDbDir(dbDir)
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	userdataStore := menuStorageService.GetUserdataDb(dbDir, ctx)
+	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -97,7 +97,7 @@ func main() {
 	}
 
 	lhs, err := handlers.NewLocalHandlerService(pfp, true, dbResource, cfg, rs)
-	lhs.WithDataStore(&userdataStore)
+	lhs.SetDataStore(&userdataStore)
 
 	hl, err := lhs.GetHandler()
 	if err != nil {
@@ -105,7 +105,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	stateStore, err := menuStorageService.GetStateStore(dbDir, ctx)
+	stateStore, err := menuStorageService.GetStateStore(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -138,22 +138,26 @@ func main() {
 	for true {
 		rqs, err = sh.Process(rqs)
 		if err != nil {
+			logg.ErrorCtxf(ctx, "error in process: %v", "err", err)
 			fmt.Errorf("error in process: %v", err)
 			os.Exit(1)
 		}
 		rqs, err = sh.Output(rqs)
 		if err != nil {
+			logg.ErrorCtxf(ctx, "error in output: %v", "err", err)
 			fmt.Errorf("error in output: %v", err)
 			os.Exit(1)
 		}
 		rqs, err = sh.Reset(rqs)
 		if err != nil {
+			logg.ErrorCtxf(ctx, "error in reset: %v", "err", err)
 			fmt.Errorf("error in reset: %v", err)
 			os.Exit(1)
 		}
 		fmt.Println("")
 		_, err = fmt.Scanln(&rqs.Input)
 		if err != nil {
+			logg.ErrorCtxf(ctx, "error in input", "err", err)
 			fmt.Errorf("error in input: %v", err)
 			os.Exit(1)
 		}

cmd/http/main.go

@@ -59,20 +59,20 @@ func main() {
 		cfg.EngineDebug = true
 	}
 
-	menuStorageService := storage.MenuStorageService{}
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
-	rs, err := menuStorageService.GetResource(scriptDir, ctx)
+	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	err = menuStorageService.EnsureDbDir(dbDir)
+	err = menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	userdataStore := menuStorageService.GetUserdataDb(dbDir, ctx)
+	userdataStore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -85,7 +85,7 @@ func main() {
 	}
 
 	lhs, err := handlers.NewLocalHandlerService(pfp, true, dbResource, cfg, rs)
-	lhs.WithDataStore(&userdataStore)
+	lhs.SetDataStore(&userdataStore)
 
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
@@ -98,7 +98,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	stateStore, err := menuStorageService.GetStateStore(dbDir, ctx)
+	stateStore, err := menuStorageService.GetStateStore(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)

cmd/main.go

@@ -43,27 +43,28 @@ func main() {
 		FlagCount:  uint32(16),
 	}
 
-	menuStorageService := storage.MenuStorageService{}
+	resourceDir := scriptDir
+	menuStorageService := storage.NewMenuStorageService(dbDir, resourceDir)
 
-	err := menuStorageService.EnsureDbDir(dbDir)
+	err := menuStorageService.EnsureDbDir()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	rs, err := menuStorageService.GetResource(scriptDir, ctx)
+	rs, err := menuStorageService.GetResource(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	pe, err := menuStorageService.GetPersister(dbDir, ctx)
+	pe, err := menuStorageService.GetPersister(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
 	}
 
-	userdatastore := menuStorageService.GetUserdataDb(dbDir, ctx)
+	userdatastore, err := menuStorageService.GetUserdataDb(ctx)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())
 		os.Exit(1)
@@ -76,8 +77,8 @@ func main() {
 	}
 
 	lhs, err := handlers.NewLocalHandlerService(pfp, true, dbResource, cfg, rs)
-	lhs.WithDataStore(&userdatastore)
+	lhs.SetDataStore(&userdatastore)
-	lhs.WithPersister(pe)
+	lhs.SetPersister(pe)
 
 	if err != nil {
 		fmt.Fprintf(os.Stderr, err.Error())

go.mod

@@ -3,7 +3,7 @@ module git.grassecon.net/urdt/ussd
 go 1.22.6
 
 require (
-	git.defalsify.org/vise.git v0.1.0-rc.3.0.20240922152136-7ea16f9137b4
+	git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb
 	github.com/alecthomas/assert/v2 v2.2.2
 	github.com/peteole/testdata-loader v0.3.0
 	gopkg.in/leonelquinteros/gotext.v1 v1.3.1

go.sum

@@ -1,5 +1,5 @@
-git.defalsify.org/vise.git v0.1.0-rc.3.0.20240922152136-7ea16f9137b4 h1:IMVUK9OkZ/QtYZPHgTZ+XUs5VQ4eIewIaTyVSCF/nAY=
+git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb h1:6P4kxihcwMjDKzvUFC6t2zGNb7MDW+l/ACGlSAN1N8Y=
-git.defalsify.org/vise.git v0.1.0-rc.3.0.20240922152136-7ea16f9137b4/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M=
+git.defalsify.org/vise.git v0.1.0-rc.3.0.20240923162317-c20d557a3dbb/go.mod h1:JDguWmcoWBdsnpw7PUjVZAEpdC/ubBmjdUBy3tjP63M=
 github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
 github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
 github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=

internal/handlers/base.go

@@ -46,7 +46,7 @@ func(f *BaseSessionHandler) Process(rqs RequestSession) (RequestSession, error)
 	var err error
 	var ok bool
 	
-	logg.InfoCtxf(rqs.Ctx, "new request",  rqs)
+	logg.InfoCtxf(rqs.Ctx, "new request", "data", rqs)
 
 	rqs.Storage, err = f.provider.Get(rqs.Config.SessionId)
 	if err != nil {

internal/handlers/handlerservice.go

@@ -44,16 +44,16 @@ func NewLocalHandlerService(fp string, debug bool, dbResource *resource.DbResour
 	}, nil
 }
 
-func (localHandlerService *LocalHandlerService) WithPersister(Pe *persist.Persister) {
+func (ls *LocalHandlerService) SetPersister(Pe *persist.Persister) {
-	localHandlerService.Pe = Pe
+	ls.Pe = Pe
 }
 
-func (localHandlerService *LocalHandlerService) WithDataStore(db *db.Db) {
+func (ls *LocalHandlerService) SetDataStore(db *db.Db) {
-	localHandlerService.UserdataStore = db
+	ls.UserdataStore = db
 }
 
-func (localHandlerService *LocalHandlerService) GetHandler() (*ussd.Handlers, error) {
+func (ls *LocalHandlerService) GetHandler() (*ussd.Handlers, error) {
-	ussdHandlers, err := ussd.NewHandlers(localHandlerService.Parser, *localHandlerService.UserdataStore)
+	ussdHandlers, err := ussd.NewHandlers(ls.Parser, *ls.UserdataStore)
 	if err != nil {
 		return nil, err
 	}
@@ -97,8 +97,9 @@ func (localHandlerService *LocalHandlerService) GetHandler() (*ussd.Handlers, er
 	return ussdHandlers, nil
 }
 
-func (localHandlerService *LocalHandlerService) GetEngine() *engine.DefaultEngine {
+// TODO: enable setting of sessionId on engine init time
-	en := engine.NewEngine(localHandlerService.Cfg, localHandlerService.Rs)
+func (ls *LocalHandlerService) GetEngine() *engine.DefaultEngine {
-	en = en.WithPersister(localHandlerService.Pe)
+	en := engine.NewEngine(ls.Cfg, ls.Rs)
+	en = en.WithPersister(ls.Pe)
 	return en
 }

internal/storage/gdbm.go

@@ -0,0 +1,115 @@
+package storage
+
+import (
+	"context"
+
+	"git.defalsify.org/vise.git/db"
+	"git.defalsify.org/vise.git/lang"
+	gdbmdb "git.defalsify.org/vise.git/db/gdbm"
+)
+
+var (
+	dbC map[string]chan db.Db
+)
+
+type ThreadGdbmDb struct {
+	db db.Db
+	connStr string
+}
+
+func NewThreadGdbmDb() *ThreadGdbmDb {
+	if dbC == nil {
+		dbC = make(map[string]chan db.Db)
+	}
+	return &ThreadGdbmDb{}
+}
+
+func(tdb *ThreadGdbmDb) Connect(ctx context.Context, connStr string) error {
+	var ok bool
+	_, ok = dbC[connStr]
+	if ok {
+		logg.WarnCtxf(ctx, "already registered thread gdbm, skipping", "connStr", connStr)
+		return nil
+	}
+	gdb := gdbmdb.NewGdbmDb()
+	err := gdb.Connect(ctx, connStr)
+	if err != nil {
+		return err
+	}
+	dbC[connStr] = make(chan db.Db, 1)
+	dbC[connStr]<- gdb
+	tdb.connStr = connStr
+	return nil
+}
+
+func(tdb *ThreadGdbmDb) reserve() {
+	if tdb.db == nil {
+		tdb.db = <-dbC[tdb.connStr]
+	}
+}
+
+func(tdb *ThreadGdbmDb) release() {
+	if tdb.db == nil {
+		return
+	}
+	dbC[tdb.connStr] <- tdb.db
+	tdb.db = nil
+}
+
+func(tdb *ThreadGdbmDb) SetPrefix(pfx uint8) {
+	tdb.reserve()
+	tdb.db.SetPrefix(pfx)
+}
+
+func(tdb *ThreadGdbmDb) SetSession(sessionId string) {
+	tdb.reserve()
+	tdb.db.SetSession(sessionId)
+}
+
+func(tdb *ThreadGdbmDb) SetLanguage(lng *lang.Language) {
+	tdb.reserve()
+	tdb.db.SetLanguage(lng)
+}
+
+func(tdb *ThreadGdbmDb) Safe() bool {
+	tdb.reserve()
+	v := tdb.db.Safe()
+	tdb.release()
+	return v
+}
+
+func(tdb *ThreadGdbmDb) Prefix() uint8 {
+	tdb.reserve()
+	v := tdb.db.Prefix()
+	tdb.release()
+	return v
+}
+
+func(tdb *ThreadGdbmDb) SetLock(typ uint8, locked bool) error {
+	tdb.reserve()
+	err := tdb.db.SetLock(typ, locked)
+	tdb.release()
+	return err
+}
+
+func(tdb *ThreadGdbmDb) Put(ctx context.Context, key []byte, val []byte) error {
+	tdb.reserve()
+	err := tdb.db.Put(ctx, key, val)
+	tdb.release()
+	return err
+}
+
+func(tdb *ThreadGdbmDb) Get(ctx context.Context, key []byte) ([]byte, error) {
+	tdb.reserve()
+	v, err := tdb.db.Get(ctx, key)
+	tdb.release()
+	return v, err
+}
+
+func(tdb *ThreadGdbmDb) Close() error {
+	tdb.reserve()
+	close(dbC[tdb.connStr])
+	err := tdb.db.Close()
+	tdb.db = nil
+	return err
+}

internal/storage/storage.go

@@ -5,6 +5,10 @@ import (
 	"git.defalsify.org/vise.git/persist"
 )
 
+const (
+	DATATYPE_CUSTOM = 128 
+)
+
 type Storage struct {
 	Persister *persist.Persister
 	UserdataDb db.Db	

internal/storage/storageservice.go

@@ -8,57 +8,96 @@ import (
 
 	"git.defalsify.org/vise.git/db"
 	fsdb "git.defalsify.org/vise.git/db/fs"
-	gdbmdb "git.defalsify.org/vise.git/db/gdbm"
 	"git.defalsify.org/vise.git/persist"
 	"git.defalsify.org/vise.git/resource"
+	"git.defalsify.org/vise.git/logging"
+)
+
+var (
+	logg = logging.NewVanilla().WithDomain("storage")
 )	
 
 type StorageService interface {
-	GetPersister(dbDir string, ctx context.Context) (*persist.Persister, error)
+	GetPersister(ctx context.Context) (*persist.Persister, error)
-	GetUserdataDb(dbDir string, ctx context.Context) db.Db
+	GetUserdataDb(ctx context.Context) db.Db
-	GetResource(resourceDir string, ctx context.Context) (resource.Resource, error)
+	GetResource(ctx context.Context) (resource.Resource, error)
-	EnsureDbDir(dbDir string) error
+	EnsureDbDir() error
 }
 
-type MenuStorageService struct{}
+type MenuStorageService struct{
-
+	dbDir string
-func (menuStorageService *MenuStorageService) GetPersister(dbDir string, ctx context.Context) (*persist.Persister, error) {
+	resourceDir string
-	store := gdbmdb.NewGdbmDb()
+	resourceStore db.Db
-	storeFile := path.Join(dbDir, "state.gdbm")
+	stateStore db.Db
-	store.Connect(ctx, storeFile)
+	userDataStore db.Db
-	pr := persist.NewPersister(store)
-	return pr, nil
 }
 
-func (menuStorageService *MenuStorageService) GetUserdataDb(dbDir string, ctx context.Context) db.Db {
+func NewMenuStorageService(dbDir string, resourceDir string) *MenuStorageService {
-	store := gdbmdb.NewGdbmDb()
+	return &MenuStorageService{
-	storeFile := path.Join(dbDir, "userdata.gdbm")
+		dbDir: dbDir,
-	store.Connect(ctx, storeFile)
+		resourceDir: resourceDir,
-
+	}
-	return store
 }
 
-func (menuStorageService *MenuStorageService) GetResource(resourceDir string, ctx context.Context) (resource.Resource, error) {
+func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) {
-	store := fsdb.NewFsDb()
+	ms.stateStore = NewThreadGdbmDb()
-	err := store.Connect(ctx, resourceDir)
+	storeFile := path.Join(ms.dbDir, "state.gdbm")
+	err := ms.stateStore.Connect(ctx, storeFile)
 	if err != nil {
 		return nil, err
 	}
-	rfs := resource.NewDbResource(store)
+	pr := persist.NewPersister(ms.stateStore)
+	logg.TraceCtxf(ctx, "menu storage service", "persist", pr, "store", ms.stateStore)
+	return pr, nil
+}
+
+func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) {
+	ms.userDataStore = NewThreadGdbmDb()
+	storeFile := path.Join(ms.dbDir, "userdata.gdbm")
+	err := ms.userDataStore.Connect(ctx, storeFile)
+	if err != nil {
+		return nil, err
+	}
+	return ms.userDataStore, nil
+}
+
+func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resource, error) {
+	ms.resourceStore = fsdb.NewFsDb()
+	err := ms.resourceStore.Connect(ctx, ms.resourceDir)
+	if err != nil {
+		return nil, err
+	}
+	rfs := resource.NewDbResource(ms.resourceStore)
 	return rfs, nil
 }
 
-func (menuStorageService *MenuStorageService) GetStateStore(dbDir string, ctx context.Context) (db.Db, error) {
+func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error) {
-	store := gdbmdb.NewGdbmDb()
+	if ms.stateStore != nil {
-	storeFile := path.Join(dbDir, "state.gdbm")
+		panic("set up store when already exists")
-	store.Connect(ctx, storeFile)
+	}
-	return store, nil
+	ms.stateStore = NewThreadGdbmDb()
+	storeFile := path.Join(ms.dbDir, "state.gdbm")
+	err := ms.stateStore.Connect(ctx, storeFile)
+	if err != nil {
+		return nil, err
+	}
+	return ms.stateStore, nil
 }
 
-func (menuStorageService *MenuStorageService) EnsureDbDir(dbDir string) error {
+func (ms *MenuStorageService) EnsureDbDir() error {
-	err := os.MkdirAll(dbDir, 0700)
+	err := os.MkdirAll(ms.dbDir, 0700)
 	if err != nil {
 		return fmt.Errorf("state dir create exited with error: %v\n", err)
 	}
 	return nil
 }
+
+func (ms *MenuStorageService) Close() error {
+	errA := ms.stateStore.Close()
+	errB := ms.userDataStore.Close()
+	errC := ms.resourceStore.Close()
+	if errA != nil || errB != nil || errC != nil {
+		return fmt.Errorf("%v %v %v", errA, errB, errC)
+	}
+	return nil
+}