Merge branch 'master' into lash/stalecache

cmd/africastalking/main.go

@@ -29,10 +29,10 @@ import (
 )
 
 var (
-	logg          = logging.NewVanilla()
-	scriptDir     = path.Join("services", "registration")
-	build         = "dev"
-	menuSeparator = ": "
+	logg      = logging.NewVanilla()
+	scriptDir = path.Join("services", "registration")
+
+	build = "dev"
 )
 
 func init() {
@@ -130,10 +130,9 @@ func main() {
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
-		Root:          "root",
-		OutputSize:    uint32(size),
-		FlagCount:     uint32(128),
-		MenuSeparator: menuSeparator,
+		Root:       "root",
+		OutputSize: uint32(size),
+		FlagCount:  uint32(128),
 	}
 
 	if engineDebug {

cmd/async/main.go

@@ -23,7 +23,6 @@ import (
 var (
 	logg      = logging.NewVanilla()
 	scriptDir = path.Join("services", "registration")
-	menuSeparator = ": "
 )
 
 func init() {
@@ -71,10 +70,9 @@ func main() {
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
-		Root:          "root",
-		OutputSize:    uint32(size),
-		FlagCount:     uint32(128),
-		MenuSeparator: menuSeparator,
+		Root:       "root",
+		OutputSize: uint32(size),
+		FlagCount:  uint32(128),
 	}
 
 	if engineDebug {

cmd/http/main.go

@@ -26,7 +26,6 @@ import (
 var (
 	logg      = logging.NewVanilla()
 	scriptDir = path.Join("services", "registration")
-	menuSeparator = ": "
 )
 
 func init() {
@@ -59,10 +58,9 @@ func main() {
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
-		Root:          "root",
-		OutputSize:    uint32(size),
-		FlagCount:     uint32(128),
-		MenuSeparator: menuSeparator,
+		Root:       "root",
+		OutputSize: uint32(size),
+		FlagCount:  uint32(128),
 	}
 
 	if engineDebug {

cmd/main.go

@@ -18,9 +18,8 @@ import (
 )
 
 var (
-	logg          = logging.NewVanilla()
-	scriptDir     = path.Join("services", "registration")
-	menuSeparator = ": "
+	logg      = logging.NewVanilla()
+	scriptDir = path.Join("services", "registration")
 )
 
 func init() {
@@ -50,11 +49,10 @@ func main() {
 	pfp := path.Join(scriptDir, "pp.csv")
 
 	cfg := engine.Config{
-		Root:          "root",
-		SessionId:     sessionId,
-		OutputSize:    uint32(size),
-		FlagCount:     uint32(128),
-		MenuSeparator: menuSeparator,
+		Root:       "root",
+		SessionId:  sessionId,
+		OutputSize: uint32(size),
+		FlagCount:  uint32(128),
 	}
 
 	resourceDir := scriptDir

common/transfer_statements.go

@@ -84,18 +84,18 @@ func GetTransferData(ctx context.Context, db storage.PrefixDb, publicKey string,
 
 	// Adjust for 0-based indexing
 	i := index - 1
-	transactionType := "Received"
-	party := fmt.Sprintf("From: %s", strings.TrimSpace(senders[i]))
+	transactionType := "received"
+	party := fmt.Sprintf("from: %s", strings.TrimSpace(senders[i]))
 	if strings.TrimSpace(senders[i]) == publicKey {
-		transactionType = "Sent"
-		party = fmt.Sprintf("To: %s", strings.TrimSpace(recipients[i]))
+		transactionType = "sent"
+		party = fmt.Sprintf("to: %s", strings.TrimSpace(recipients[i]))
 	}
 
 	formattedDate := formatDate(strings.TrimSpace(dates[i]))
 
 	// Build the full transaction detail
 	detail := fmt.Sprintf(
-		"%s %s %s\n%s\nContract address: %s\nTxhash: %s\nDate: %s",
+		"%s %s %s\n%s\ncontract address: %s\ntxhash: %s\ndate: %s",
 		transactionType,
 		strings.TrimSpace(values[i]),
 		strings.TrimSpace(syms[i]),

internal/handlers/handlerservice.go

@@ -2,7 +2,6 @@ package handlers
 
 import (
 	"context"
-	"strings"
 
 	"git.defalsify.org/vise.git/asm"
 	"git.defalsify.org/vise.git/db"
@@ -65,11 +64,7 @@ func (ls *LocalHandlerService) SetDataStore(db *db.Db) {
 }
 
 func (ls *LocalHandlerService) GetHandler(accountService remote.AccountServiceInterface) (*ussd.Handlers, error) {
-	replaceSeparatorFunc := func(input string) string {
-		return strings.ReplaceAll(input, ":", ls.Cfg.MenuSeparator)
-	}
-
-	ussdHandlers, err := ussd.NewHandlers(ls.Parser, *ls.UserdataStore, ls.AdminStore, accountService, replaceSeparatorFunc)
+	ussdHandlers, err := ussd.NewHandlers(ls.Parser, *ls.UserdataStore, ls.AdminStore, accountService)
 	if err != nil {
 		return nil, err
 	}

internal/handlers/ussd/menuhandler.go

@@ -69,19 +69,18 @@ func (fm *FlagManager) GetFlag(label string) (uint32, error) {
 }
 
 type Handlers struct {
-	pe               *persist.Persister
-	st               *state.State
-	ca               cache.Memory
-	userdataStore    common.DataStore
-	adminstore       *utils.AdminStore
-	flagManager      *asm.FlagParser
-	accountService   remote.AccountServiceInterface
-	prefixDb         storage.PrefixDb
-	profile          *models.Profile
-	ReplaceSeparatorFunc func(string) string
+	pe             *persist.Persister
+	st             *state.State
+	ca             cache.Memory
+	userdataStore  common.DataStore
+	adminstore     *utils.AdminStore
+	flagManager    *asm.FlagParser
+	accountService remote.AccountServiceInterface
+	prefixDb       storage.PrefixDb
+	profile        *models.Profile
 }
 
-func NewHandlers(appFlags *asm.FlagParser, userdataStore db.Db, adminstore *utils.AdminStore, accountService remote.AccountServiceInterface, replaceSeparatorFunc func(string) string) (*Handlers, error) {
+func NewHandlers(appFlags *asm.FlagParser, userdataStore db.Db, adminstore *utils.AdminStore, accountService remote.AccountServiceInterface) (*Handlers, error) {
 	if userdataStore == nil {
 		return nil, fmt.Errorf("cannot create handler with nil userdata store")
 	}
@@ -94,13 +93,12 @@ func NewHandlers(appFlags *asm.FlagParser, userdataStore db.Db, adminstore *util
 	prefixDb := storage.NewSubPrefixDb(userdataStore, prefix)
 
 	h := &Handlers{
-		userdataStore:    userDb,
-		flagManager:      appFlags,
-		adminstore:       adminstore,
-		accountService:   accountService,
-		prefixDb:         prefixDb,
-		profile:          &models.Profile{Max: 6},
-		ReplaceSeparatorFunc: replaceSeparatorFunc,
+		userdataStore:  userDb,
+		flagManager:    appFlags,
+		adminstore:     adminstore,
+		accountService: accountService,
+		prefixDb:       prefixDb,
+		profile:        &models.Profile{Max: 6},
 	}
 	return h, nil
 }
@@ -1685,9 +1683,7 @@ func (h *Handlers) GetVoucherList(ctx context.Context, sym string, input []byte)
 		return res, err
 	}
 
-	formattedData := h.ReplaceSeparatorFunc(string(voucherData))
-
-	res.Content = string(formattedData)
+	res.Content = string(voucherData)
 
 	return res, nil
 }
@@ -1850,14 +1846,13 @@ func (h *Handlers) CheckTransactions(ctx context.Context, sym string, input []by
 	return res, nil
 }
 
-// GetTransactionsList reads the list of transactions from the db and formats them
+// GetTransactionsList fetches the list of transactions and formats them
 func (h *Handlers) GetTransactionsList(ctx context.Context, sym string, input []byte) (resource.Result, error) {
 	var res resource.Result
 	sessionId, ok := ctx.Value("SessionId").(string)
 	if !ok {
 		return res, fmt.Errorf("missing session")
 	}
-
 	store := h.userdataStore
 	publicKey, err := store.ReadEntry(ctx, sessionId, common.DATA_PUBLIC_KEY)
 	if err != nil {
@@ -1900,14 +1895,12 @@ func (h *Handlers) GetTransactionsList(ctx context.Context, sym string, input []
 		value := strings.TrimSpace(values[i])
 		date := strings.Split(strings.TrimSpace(dates[i]), " ")[0]
 
-		status := "Received"
+		status := "received"
 		if sender == string(publicKey) {
-			status = "Sent"
+			status = "sent"
 		}
 
-		// Use the ReplaceSeparator function for the menu separator
-		transactionLine := fmt.Sprintf("%d%s%s %s %s %s", i+1, h.ReplaceSeparatorFunc(":"), status, value, sym, date)
-		formattedTransactions = append(formattedTransactions, transactionLine)
+		formattedTransactions = append(formattedTransactions, fmt.Sprintf("%d:%s %s %s %s", i+1, status, value, sym, date))
 	}
 
 	res.Content = strings.Join(formattedTransactions, "\n")

internal/handlers/ussd/menuhandler_test.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"path"
-	"strings"
 	"testing"
 
 	"git.defalsify.org/vise.git/lang"
@@ -33,11 +32,6 @@ var (
 	flagsPath = path.Join(baseDir, "services", "registration", "pp.csv")
 )
 
-// mockReplaceSeparator function
-var mockReplaceSeparator = func(input string) string {
-	return strings.ReplaceAll(input, ":", ": ")
-}
-
 // InitializeTestStore sets up and returns an in-memory database and store.
 func InitializeTestStore(t *testing.T) (context.Context, *common.UserDataStore) {
 	ctx := context.Background()
@@ -73,15 +67,12 @@ func TestNewHandlers(t *testing.T) {
 	_, store := InitializeTestStore(t)
 
 	fm, err := NewFlagManager(flagsPath)
-	if err != nil {
-		log.Fatal(err)
-	}
-
 	accountService := testservice.TestAccountService{}
-
-	// Test case for valid UserDataStore
+	if err != nil {
+		t.Logf(err.Error())
+	}
 	t.Run("Valid UserDataStore", func(t *testing.T) {
-		handlers, err := NewHandlers(fm.parser, store, nil, &accountService, mockReplaceSeparator)
+		handlers, err := NewHandlers(fm.parser, store, nil, &accountService)
 		if err != nil {
 			t.Fatalf("expected no error, got %v", err)
 		}
@@ -91,30 +82,19 @@ func TestNewHandlers(t *testing.T) {
 		if handlers.userdataStore == nil {
 			t.Fatal("expected userdataStore to be set in handlers")
 		}
-		if handlers.ReplaceSeparatorFunc == nil {
-			t.Fatal("expected ReplaceSeparatorFunc to be set in handlers")
-		}
-
-		// Test ReplaceSeparatorFunc functionality
-		input := "1:Menu item"
-		expectedOutput := "1: Menu item"
-		if handlers.ReplaceSeparatorFunc(input) != expectedOutput {
-			t.Fatalf("ReplaceSeparatorFunc function did not return expected output: got %v, want %v", handlers.ReplaceSeparatorFunc(input), expectedOutput)
-		}
 	})
 
-	// Test case for nil UserDataStore
+	// Test case for nil userdataStore
 	t.Run("Nil UserDataStore", func(t *testing.T) {
-		handlers, err := NewHandlers(fm.parser, nil, nil, &accountService, mockReplaceSeparator)
+		handlers, err := NewHandlers(fm.parser, nil, nil, &accountService)
 		if err == nil {
 			t.Fatal("expected an error, got none")
 		}
 		if handlers != nil {
 			t.Fatal("expected handlers to be nil")
 		}
-		expectedError := "cannot create handler with nil userdata store"
-		if err.Error() != expectedError {
-			t.Fatalf("expected error '%s', got '%v'", expectedError, err)
+		if err.Error() != "cannot create handler with nil userdata store" {
+			t.Fatalf("expected specific error, got %v", err)
 		}
 	})
 }
@@ -2002,31 +1982,26 @@ func TestCheckVouchers(t *testing.T) {
 
 func TestGetVoucherList(t *testing.T) {
 	sessionId := "session123"
-
 	ctx := context.WithValue(context.Background(), "SessionId", sessionId)
 
 	spdb := InitializeTestSubPrefixDb(t, ctx)
 
-	// Initialize Handlers
 	h := &Handlers{
-		prefixDb:             spdb,
-		ReplaceSeparatorFunc: mockReplaceSeparator,
+		prefixDb: spdb,
 	}
 
-	mockSyms := []byte("1:SRF\n2:MILO")
+	expectedSym := []byte("1:SRF\n2:MILO")
 
 	// Put voucher sym data from the store
-	err := spdb.Put(ctx, common.ToBytes(common.DATA_VOUCHER_SYMBOLS), mockSyms)
+	err := spdb.Put(ctx, common.ToBytes(common.DATA_VOUCHER_SYMBOLS), expectedSym)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	expectedSyms := []byte("1: SRF\n2: MILO")
-
 	res, err := h.GetVoucherList(ctx, "", []byte(""))
 
 	assert.NoError(t, err)
-	assert.Equal(t, res.Content, string(expectedSyms))
+	assert.Equal(t, res.Content, string(expectedSym))
 }
 
 func TestViewVoucher(t *testing.T) {

internal/storage/gdbm.go

@@ -12,6 +12,7 @@ var (
 	dbC map[string]chan db.Db
 )
 
+
 type ThreadGdbmDb struct {
 	db db.Db
 	connStr string

internal/storage/sub_prefix_db.go

@@ -6,6 +6,11 @@ import (
 	"git.defalsify.org/vise.git/db"
 )
 
+const (
+	DATATYPE_USERSUB = 64
+	SUBPREFIX_TIME = uint16(1)
+)
+
 // PrefixDb interface abstracts the database operations.
 type PrefixDb interface {
 	Get(ctx context.Context, key []byte) ([]byte, error)
@@ -26,8 +31,12 @@ func NewSubPrefixDb(store db.Db, pfx []byte) *SubPrefixDb {
 	}
 }
 
-func (s *SubPrefixDb) toKey(k []byte) []byte {
-	return append(s.pfx, k...)
+func(s *SubPrefixDb) SetSession(sessionId string) {
+	s.store.SetSession(sessionId)	
+}
+
+func(s *SubPrefixDb) toKey(k []byte) []byte {
+        return append(s.pfx, k...)
 }
 
 func (s *SubPrefixDb) Get(ctx context.Context, key []byte) ([]byte, error) {

internal/storage/timed.go

@@ -0,0 +1,109 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"time"
+	"encoding/binary"
+
+	"git.defalsify.org/vise.git/db"
+)
+
+type TimedDb struct {
+	db.Db
+	tdb *SubPrefixDb
+	ttl time.Duration
+	parentPfx uint8
+	parentSession []byte
+	matchPfx map[uint8][][]byte
+}
+
+func NewTimedDb(db db.Db, ttl time.Duration) *TimedDb {
+	var b [2]byte
+	binary.BigEndian.PutUint16(b[:], SUBPREFIX_TIME)
+	sdb := NewSubPrefixDb(db, b[:])
+	return &TimedDb{
+		Db: db,
+		tdb: sdb,
+		ttl: ttl,
+	}
+}
+
+func(tib *TimedDb) WithMatch(pfx uint8, keyPart []byte) *TimedDb {
+	if tib.matchPfx == nil {
+		tib.matchPfx = make(map[uint8][][]byte)
+	}
+	tib.matchPfx[pfx] = append(tib.matchPfx[pfx], keyPart)
+	return tib
+}
+
+func(tib *TimedDb) checkPrefix(pfx uint8, key []byte) bool {
+	var v []byte
+	if tib.matchPfx == nil {
+		return true
+	}
+	for _, v = range(tib.matchPfx[pfx]) {
+		l := len(v)
+		if l > len(key) {
+			continue
+		}
+		if bytes.Equal(v, key[:l]) {
+			return true	
+		}
+	}
+	return false
+}
+
+func(tib *TimedDb) SetPrefix(pfx uint8) {
+	tib.Db.SetPrefix(pfx)
+	tib.parentPfx = pfx
+}
+
+func(tib *TimedDb) SetSession(session string) {
+	tib.Db.SetSession(session)
+	tib.parentSession = []byte(session)
+}
+
+func(tib *TimedDb) Put(ctx context.Context, key []byte, val []byte) error {
+	t := time.Now()	
+	b, err := t.MarshalBinary()
+	if err != nil {
+		return err
+	}
+	err = tib.Db.Put(ctx, key, val)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		tib.parentPfx = 0
+		tib.parentSession = nil
+	}()
+	if tib.checkPrefix(tib.parentPfx, key) {
+		tib.tdb.SetSession("")
+		k := db.ToSessionKey(tib.parentPfx, []byte(tib.parentSession), key)
+		k = append([]byte{tib.parentPfx}, k...)
+		err = tib.tdb.Put(ctx, k, b)
+		if err != nil {
+			logg.ErrorCtxf(ctx, "failed to update timestamp of record", err)
+		}
+	}
+	return nil
+}
+
+func(tib *TimedDb) Stale(ctx context.Context, pfx uint8, sessionId string, key []byte) bool {
+	tib.tdb.SetSession("")
+	b := db.ToSessionKey(pfx, []byte(sessionId), key)
+	b = append([]byte{pfx}, b...)
+	v, err := tib.tdb.Get(ctx, b)
+	if err != nil {
+		logg.WarnCtxf(ctx, "no time entry", "key", key, "b", b)
+		return false
+	}
+	t_now := time.Now()	
+	t_then := time.Time{}
+	err = t_then.UnmarshalBinary(v)
+	if err != nil {
+		return false
+	}
+	return t_now.After(t_then.Add(tib.ttl))
+}

internal/storage/timed_test.go

@@ -0,0 +1,125 @@
+package storage
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"git.defalsify.org/vise.git/db"
+	memdb "git.defalsify.org/vise.git/db/mem"
+)
+
+func TestStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	k := []byte("foo")
+	err = tdb.Put(ctx, k, []byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "", k) {
+		t.Fatal("expected not stale")
+	}
+	time.Sleep(time.Millisecond)
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "", k) {
+		t.Fatal("expected stale")
+	}
+}
+
+func TestFilteredStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k := []byte("foo")
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb = tdb.WithMatch(db.DATATYPE_STATE, []byte("fo"))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("inky")
+	err = tdb.Put(ctx, k, []byte("bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_STATE)
+	tdb.SetSession("inky")
+	err = tdb.Put(ctx, k, []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetSession("blinky")
+	err = tdb.Put(ctx, k, []byte("clyde"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "blinky", k) {
+		t.Fatal("expected not stale")
+	}
+	time.Sleep(time.Millisecond)
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "inky", k) {
+		t.Fatal("expected not stale")
+	}
+	if !tdb.Stale(ctx, db.DATATYPE_STATE, "inky", k) {
+		t.Fatal("expected stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_STATE, "blinky", k) {
+		t.Fatal("expected not stale")
+	}
+}
+
+func TestFilteredSameKeypartStaleDb(t *testing.T) {
+	ctx := context.Background()
+	mdb := memdb.NewMemDb()
+	err := mdb.Connect(ctx, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tdb := NewTimedDb(mdb, time.Duration(time.Millisecond))
+	tdb = tdb.WithMatch(db.DATATYPE_USERDATA, []byte("ba"))
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("bar"), []byte("inky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("baz"), []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	tdb.SetPrefix(db.DATATYPE_USERDATA)
+	tdb.SetSession("xyzzy")
+	err = tdb.Put(ctx, []byte("foo"), []byte("blinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	time.Sleep(time.Millisecond)
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("bar")) {
+		t.Fatal("expected stale")
+	}
+	if !tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("baz")) {
+	t.Fatal("expected stale")
+	}
+	if tdb.Stale(ctx, db.DATATYPE_USERDATA, "xyzzy", []byte("foo")) {
+		t.Fatal("expected not stale")
+	}
+}