urdt/ussd
5
0
Fork 1
Code Issues 62 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

postgres-switch-for-tests #255

Merged
lash merged 15 commits from postgres-switch-for-tests into master 2025-01-10 12:07:07 +01:00
Conversation 0 Commits 15 Files Changed 10 +190 -73
10 changed files with 190 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/africastalking/main.go
View File

@ -12,17 +12,17 @@ import (
"syscall"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/lang"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/lang"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/args"
"git.grassecon.net/urdt/ussd/internal/handlers"
"git.grassecon.net/urdt/ussd/internal/http/at"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/remote"
"git.grassecon.net/urdt/ussd/internal/args"
)
var (
@ -42,7 +42,6 @@ func main() {
var connStr string
var resourceDir string
var size uint
var database string
var engineDebug bool
var host string
var port uint
@ -60,7 +59,7 @@ func main() {
flag.Var(&langs, "language", "add symbol resolution for language")
flag.Parse()
if connStr != "" {
if connStr == "" {
connStr = config.DbConn
}
connData, err := storage.ToConnData(connStr)
@ -72,7 +71,6 @@ func main() {
logg.Infof("start command", "build", build, "conn", connData, "resourcedir", resourceDir, "outputsize", size)
ctx := context.Background()
ctx = context.WithValue(ctx, "Database", database)
ln, err := lang.LanguageFromCode(config.DefaultLanguage)
if err != nil {
fmt.Fprintf(os.Stderr, "default language set error: %v", err)

9
cmd/async/main.go
View File

@ -10,16 +10,16 @@ import (
"syscall"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/lang"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/lang"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/args"
"git.grassecon.net/urdt/ussd/internal/handlers"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/remote"
"git.grassecon.net/urdt/ussd/internal/args"
)
var (
@ -52,7 +52,6 @@ func main() {
var sessionId string
var resourceDir string
var size uint
var database string
var engineDebug bool
var host string
var port uint
@ -71,7 +70,7 @@ func main() {
flag.Var(&langs, "language", "add symbol resolution for language")
flag.Parse()
if connStr != "" {
if connStr == "" {
connStr = config.DbConn
}
connData, err := storage.ToConnData(connStr)
@ -83,7 +82,6 @@ func main() {
logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size, "sessionId", sessionId)
ctx := context.Background()
ctx = context.WithValue(ctx, "Database", database)
ln, err := lang.LanguageFromCode(config.DefaultLanguage)
if err != nil {
@ -117,7 +115,6 @@ func main() {
os.Exit(1)
}
userdataStore, err := menuStorageService.GetUserdataDb(ctx)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())

8
cmd/http/main.go
View File

@ -12,17 +12,17 @@ import (
"syscall"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/lang"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/lang"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/args"
"git.grassecon.net/urdt/ussd/internal/handlers"
httpserver "git.grassecon.net/urdt/ussd/internal/http"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/remote"
"git.grassecon.net/urdt/ussd/internal/args"
)
var (
@ -41,7 +41,6 @@ func main() {
var connStr string
var resourceDir string
var size uint
var database string
var engineDebug bool
var host string
var port uint
@ -59,7 +58,7 @@ func main() {
flag.Var(&langs, "language", "add symbol resolution for language")
flag.Parse()
if connStr != "" {
if connStr == "" {
connStr = config.DbConn
}
connData, err := storage.ToConnData(connStr)
@ -71,7 +70,6 @@ func main() {
logg.Infof("start command", "conn", connData, "resourcedir", resourceDir, "outputsize", size)
ctx := context.Background()
ctx = context.WithValue(ctx, "Database", database)
ln, err := lang.LanguageFromCode(config.DefaultLanguage)
if err != nil {

8
cmd/main.go
View File

@ -8,14 +8,14 @@ import (
"path"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/lang"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/lang"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/args"
"git.grassecon.net/urdt/ussd/internal/handlers"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/internal/args"
"git.grassecon.net/urdt/ussd/remote"
)
@ -36,7 +36,6 @@ func main() {
var connStr string
var size uint
var sessionId string
var database string
var engineDebug bool
var resourceDir string
var err error
@ -52,7 +51,7 @@ func main() {
flag.Var(&langs, "language", "add symbol resolution for language")
flag.Parse()
if connStr != "" {
if connStr == "" {
connStr = config.DbConn
}
connData, err := storage.ToConnData(connStr)
@ -69,7 +68,6 @@ func main() {
ctx := context.Background()
ctx = context.WithValue(ctx, "SessionId", sessionId)
ctx = context.WithValue(ctx, "Database", database)
ln, err := lang.LanguageFromCode(config.DefaultLanguage)
if err != nil {

10
devtools/store/generate/main.go
View File

@ -9,14 +9,16 @@ import (
"path"
"git.defalsify.org/vise.git/logging"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/common"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/storage"
testdataloader "github.com/peteole/testdata-loader"
)
var (
logg = logging.NewVanilla()
baseDir = testdataloader.GetBasePath()
scriptDir = path.Join("services", "registration")
)
@ -24,7 +26,6 @@ func init() {
initializers.LoadEnvVariables()
}
func main() {
config.LoadConfig()
@ -86,5 +87,4 @@ func main() {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
}

10
initializers/load.go
View File

@ -3,15 +3,21 @@ package initializers
import (
"log"
"os"
"path"
"strconv"
"github.com/joho/godotenv"
)
func LoadEnvVariables() {
err := godotenv.Load()
LoadEnvVariablesPath(".")
}
func LoadEnvVariablesPath(dir string) {
fp := path.Join(dir, ".env")
err := godotenv.Load(fp)
if err != nil {
log.Fatal("Error loading .env file")
log.Fatal("Error loading .env file", err)
}
}

45
internal/storage/parse.go
View File

@ -15,6 +15,7 @@ const (
type ConnData struct {
typ int
str string
domain string
}
func (cd *ConnData) DbType() int {
@ -25,23 +26,38 @@ func (cd *ConnData) String() string {
return cd.str
}
func probePostgres(s string) (string, bool) {
v, err := url.Parse(s)
if err != nil {
return "", false
}
if v.Scheme != "postgres" {
return "", false
}
return s, true
func (cd *ConnData) Domain() string {
return cd.domain
}
func probeGdbm(s string) (string, bool) {
func (cd *ConnData) Path() string {
v, _ := url.Parse(cd.str)
v.RawQuery = ""
return v.String()
}
func probePostgres(s string) (string, string, bool) {
domain := "public"
v, err := url.Parse(s)
if err != nil {
return "", "", false
}
if v.Scheme != "postgres" {
return "", "", false
}
vv := v.Query()
if vv.Has("search_path") {
domain = vv.Get("search_path")
}
return s, domain, true
}
func probeGdbm(s string) (string, string, bool) {
if !path.IsAbs(s) {
return "", false
return "", "", false
}
s = path.Clean(s)
return s, true
return s, "", true
}
func ToConnData(connStr string) (ConnData, error) {
@ -51,14 +67,15 @@ func ToConnData(connStr string) (ConnData, error) {
return o, nil
}
v, ok := probePostgres(connStr)
v, domain, ok := probePostgres(connStr)
if ok {
o.typ = DBTYPE_POSTGRES
o.str = v
o.domain = domain
return o, nil
}
v, ok = probeGdbm(connStr)
v, _, ok = probeGdbm(connStr)
if ok {
o.typ = DBTYPE_GDBM
o.str = v

27
internal/storage/storageservice.go
View File

@ -14,6 +14,7 @@ import (
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
gdbmstorage "git.grassecon.net/urdt/ussd/internal/storage/db/gdbm"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
@ -54,7 +55,12 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D
connStr := ms.conn.String()
dbTyp := ms.conn.DbType()
if dbTyp == DBTYPE_POSTGRES {
newDb = postgres.NewPgDb()
// TODO: move to vise
err = ensureSchemaExists(ctx, ms.conn)
if err != nil {
return nil, err
}
newDb = postgres.NewPgDb().WithSchema(ms.conn.Domain())
} else if dbTyp == DBTYPE_GDBM {
err = ms.ensureDbDir()
if err != nil {
@ -65,7 +71,7 @@ func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.D
} else {
return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String())
}
logg.DebugCtxf(ctx, "connecting to db", "conn", connStr)
logg.DebugCtxf(ctx, "connecting to db", "conn", connStr, "conndata", ms.conn)
err = newDb.Connect(ctx, connStr)
if err != nil {
return nil, err
@ -96,6 +102,23 @@ func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *Men
return ms
}
// ensureSchemaExists creates a new schema if it does not exist
func ensureSchemaExists(ctx context.Context, conn ConnData) error {
h, err := pgxpool.New(ctx, conn.Path())
if err != nil {
lash marked this conversation as resolved
lash commented 2025-01-08 12:33:50 +01:00
Review
Copy Link

If we are using connection string, then this needs to be url parsed to get the query string in order to determine the schema.

If we are using connection string, then this needs to be url parsed to get the query string in order to determine the schema.
return fmt.Errorf("failed to connect to the database: %w", err)
}
defer h.Close()
query := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", conn.Domain())
_, err = h.Exec(ctx, query)
if err != nil {
return fmt.Errorf("failed to create schema: %w", err)
}
return nil
}
func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) {
stateStore, err := ms.GetStateStore(ctx)
if err != nil {

90
internal/testutil/engine.go
View File

@ -3,6 +3,8 @@ package testutil
import (
"context"
"fmt"
"log"
"net/url"
"os"
"path"
"path/filepath"
@ -11,21 +13,90 @@ import (
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/resource"
"git.grassecon.net/urdt/ussd/config"
"git.grassecon.net/urdt/ussd/initializers"
"git.grassecon.net/urdt/ussd/internal/handlers"
"git.grassecon.net/urdt/ussd/internal/storage"
"git.grassecon.net/urdt/ussd/internal/testutil/testservice"
"git.grassecon.net/urdt/ussd/internal/testutil/testtag"
testdataloader "github.com/peteole/testdata-loader"
"git.grassecon.net/urdt/ussd/remote"
"github.com/jackc/pgx/v5/pgxpool"
testdataloader "github.com/peteole/testdata-loader"
)
var (
baseDir = testdataloader.GetBasePath()
logg = logging.NewVanilla()
baseDir = testdataloader.GetBasePath()
scriptDir = path.Join(baseDir, "services", "registration")
setDbType string
setConnStr string
setDbSchema string
)
func init() {
initializers.LoadEnvVariablesPath(baseDir)
config.LoadConfig()
}
// SetDatabase updates the database used by TestEngine
func SetDatabase(database, connStr, dbSchema string) {
setDbType = database
setConnStr = connStr
setDbSchema = dbSchema
}
// CleanDatabase removes all test data from the database
func CleanDatabase() {
if setDbType == "postgres" {
ctx := context.Background()
// Update the connection string with the new search path
updatedConnStr, err := updateSearchPath(setConnStr, setDbSchema)
if err != nil {
log.Fatalf("Failed to update search path: %v", err)
}
dbConn, err := pgxpool.New(ctx, updatedConnStr)
if err != nil {
log.Fatalf("Failed to connect to database for cleanup: %v", err)
}
defer dbConn.Close()
query := fmt.Sprintf("DELETE FROM %s.kv_vise;", setDbSchema)
_, execErr := dbConn.Exec(ctx, query)
if execErr != nil {
log.Printf("Failed to cleanup table %s.kv_vise: %v", setDbSchema, execErr)
} else {
log.Printf("Successfully cleaned up table %s.kv_vise", setDbSchema)
}
} else {
setConnStr, _ := filepath.Abs(setConnStr)
if err := os.RemoveAll(setConnStr); err != nil {
log.Fatalf("Failed to delete state store %s: %v", setConnStr, err)
}
}
}
// updateSearchPath updates the search_path (schema) to be used in the connection
func updateSearchPath(connStr string, newSearchPath string) (string, error) {
u, err := url.Parse(connStr)
if err != nil {
return "", fmt.Errorf("invalid connection string: %w", err)
}
// Parse the query parameters
q := u.Query()
// Update or add the search_path parameter
q.Set("search_path", newSearchPath)
// Rebuild the connection string with updated parameters
u.RawQuery = q.Encode()
return u.String(), nil
}
func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
var err error
ctx := context.Background()
ctx = context.WithValue(ctx, "SessionId", sessionId)
pfp := path.Join(scriptDir, "pp.csv")
@ -39,16 +110,27 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) {
FlagCount: uint32(128),
}
connStr, err := filepath.Abs(".test_state/state.gdbm")
if setDbType == "postgres" {
setConnStr = config.DbConn
setConnStr, err = updateSearchPath(setConnStr, setDbSchema)
if err != nil {
fmt.Println("Error:", err)
os.Exit(1)
}
} else {
setConnStr, err = filepath.Abs(setConnStr)
if err != nil {
fmt.Fprintf(os.Stderr, "connstr err: %v", err)
os.Exit(1)
}
conn, err := storage.ToConnData(connStr)
}
conn, err := storage.ToConnData(setConnStr)
if err != nil {
fmt.Fprintf(os.Stderr, "connstr parse err: %v", err)
os.Exit(1)
}
resourceDir := scriptDir
menuStorageService := storage.NewMenuStorageService(conn, resourceDir)

24
menutraversal_test/menu_traversal_test.go
View File

@ -6,8 +6,6 @@ import (
"flag"
"log"
"math/rand"
"os"
"path/filepath"
"regexp"
"testing"
@ -24,11 +22,9 @@ var (
)
var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests")
func testStore() string {
v, _ := filepath.Abs(".test_state/state.gdbm")
return v
}
var database = flag.String("db", "gdbm", "Specify the database (gdbm or postgres)")
var connStr = flag.String("conn", ".test_state", "connection string")
var dbSchema = flag.String("schema", "test", "Specify the database schema (default test)")
func GenerateSessionId() string {
uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g))
@ -84,12 +80,15 @@ func extractSendAmount(response []byte) string {
}
func TestMain(m *testing.M) {
// Parse the flags
flag.Parse()
sessionID = GenerateSessionId()
defer func() {
if err := os.RemoveAll(testStore()); err != nil {
log.Fatalf("Failed to delete state store %s: %v", testStore(), err)
}
}()
// set the db
testutil.SetDatabase(*database, *connStr, *dbSchema)
// Cleanup the db after tests
defer testutil.CleanDatabase()
m.Run()
}
@ -126,7 +125,6 @@ func TestAccountCreationSuccessful(t *testing.T) {
}
}
<-eventChannel
}
func TestAccountRegistrationRejectTerms(t *testing.T) {