From 933943636e3890a33cce4f40c23129af201ea673 Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 20 Jan 2025 10:28:42 +0000 Subject: [PATCH] Implement conns in testutil --- cmd/africastalking/main.go | 8 +- cmd/async/main.go | 8 +- cmd/http/main.go | 8 +- cmd/main.go | 8 +- cmd/ssh/main.go | 8 +- config/args.go | 10 +- config/config.go | 10 +- devtools/admin/main.go | 9 +- devtools/store/dump/main.go | 8 +- internal/cmd/cmd.go | 2 - menutraversal_test/menu_traversal_test.go | 19 ++- testutil/engine.go | 153 ++++++---------------- testutil/fsdb.go | 21 +++ testutil/setup.go | 32 +++++ 14 files changed, 144 insertions(+), 160 deletions(-) create mode 100644 testutil/fsdb.go create mode 100644 testutil/setup.go diff --git a/cmd/africastalking/main.go b/cmd/africastalking/main.go index ef72f5d..da54be8 100644 --- a/cmd/africastalking/main.go +++ b/cmd/africastalking/main.go @@ -46,10 +46,10 @@ func main() { var langs args.LangVar flag.BoolVar(&engineDebug, "d", false, "use engine debug output") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", config.Host(), "http host") flag.UintVar(&port, "p", config.Port(), "http port") diff --git a/cmd/async/main.go b/cmd/async/main.go index 2241c17..85266d0 100644 --- a/cmd/async/main.go +++ b/cmd/async/main.go @@ -55,10 +55,10 @@ func main() { var langs args.LangVar flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") diff --git a/cmd/http/main.go b/cmd/http/main.go index f2e5813..3618bb9 100644 --- a/cmd/http/main.go +++ b/cmd/http/main.go @@ -44,10 +44,10 @@ func main() { var gettextDir string var langs args.LangVar -flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") diff --git a/cmd/main.go b/cmd/main.go index defd00e..ead2edb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,10 +38,10 @@ func main() { var langs args.LangVar flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&gettextDir, "gettext", "", "use gettext translations from given directory") diff --git a/cmd/ssh/main.go b/cmd/ssh/main.go index e375ad1..31d5f65 100644 --- a/cmd/ssh/main.go +++ b/cmd/ssh/main.go @@ -39,10 +39,10 @@ func main() { var host string var port uint //flag.StringVar(&authConnStr, "authdb", "", "auth connection string") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.UintVar(&size, "s", 160, "max size of output") flag.StringVar(&host, "h", config.HostSSH(), "socket host") diff --git a/config/args.go b/config/args.go index 969d221..18305a3 100644 --- a/config/args.go +++ b/config/args.go @@ -5,13 +5,13 @@ import ( ) type Override struct { - DbConn string - StateConn string - ResourceConn string - UserConn string + DbConn *string + StateConn *string + ResourceConn *string + UserConn *string } func Apply(o *Override) error { - viseconfig.ApplyConn(&o.DbConn, &o.StateConn, &o.ResourceConn, &o.UserConn) + viseconfig.ApplyConn(o.DbConn, o.StateConn, o.ResourceConn, o.UserConn) return nil } diff --git a/config/config.go b/config/config.go index 0a1b5c1..397c4c6 100644 --- a/config/config.go +++ b/config/config.go @@ -8,10 +8,15 @@ import ( var ( GetConns = viseconfig.GetConns + EnvPath string ) -func init() { - env.LoadEnvVariables() +func loadEnv() { + if EnvPath == "" { + env.LoadEnvVariables() + } else { + env.LoadEnvVariablesPath(EnvPath) + } } const ( @@ -22,6 +27,7 @@ const ( ) func LoadConfig() error { + loadEnv() err := viseconfig.LoadConfig() if err != nil { return err diff --git a/devtools/admin/main.go b/devtools/admin/main.go index 762360f..7508905 100644 --- a/devtools/admin/main.go +++ b/devtools/admin/main.go @@ -27,11 +27,10 @@ func main() { var override config.Override flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") - + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.Parse() config.Apply(&override) diff --git a/devtools/store/dump/main.go b/devtools/store/dump/main.go index c0cb2bd..dcbc4c7 100644 --- a/devtools/store/dump/main.go +++ b/devtools/store/dump/main.go @@ -40,10 +40,10 @@ func main() { var first bool flag.StringVar(&sessionId, "session-id", "075xx2123", "session id") - flag.StringVar(&override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") - flag.StringVar(&override.ResourceConn, "resource", "?", "resource connection string") - flag.StringVar(&override.UserConn, "userdata", "?", "userdata store connection string") - flag.StringVar(&override.StateConn, "state", "?", "state store connection string") + flag.StringVar(override.DbConn, "c", "?", "default connection string (replaces all unspecified strings)") + flag.StringVar(override.ResourceConn, "resource", "?", "resource connection string") + flag.StringVar(override.UserConn, "userdata", "?", "userdata store connection string") + flag.StringVar(override.StateConn, "state", "?", "state store connection string") flag.BoolVar(&engineDebug, "d", false, "use engine debug output") flag.Parse() diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index fe98f31..d693787 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -22,10 +22,8 @@ type Cmd struct { exec func(ctx context.Context, ss storage.StorageService) error } -//func NewCmd(conn storage.ConnData, sessionId string, flagParser *application.FlagManager) *Cmd { func NewCmd(sessionId string, flagParser *application.FlagManager) *Cmd { return &Cmd{ -// conn: conn, sessionId: sessionId, flagParser: flagParser, } diff --git a/menutraversal_test/menu_traversal_test.go b/menutraversal_test/menu_traversal_test.go index 3b546ae..381a508 100644 --- a/menutraversal_test/menu_traversal_test.go +++ b/menutraversal_test/menu_traversal_test.go @@ -10,11 +10,13 @@ import ( "testing" "github.com/gofrs/uuid" + "git.defalsify.org/vise.git/logging" "git.grassecon.net/grassrootseconomics/visedriver/testutil/driver" "git.grassecon.net/grassrootseconomics/sarafu-vise/testutil" ) var ( + logg = logging.NewVanilla().WithDomain("menutraversaltest") testData = driver.ReadData() sessionID string src = rand.NewSource(42) @@ -22,9 +24,6 @@ var ( ) var groupTestFile = flag.String("test-file", "group_test.json", "The test file to use for running the group tests") -var database = flag.String("db", "gdbm", "Specify the database (gdbm or postgres)") -var connStr = flag.String("conn", ".test_state", "connection string") -var dbSchema = flag.String("schema", "test", "Specify the database schema (default test)") func GenerateSessionId() string { uu := uuid.NewGenWithOptions(uuid.WithRandomReader(g)) @@ -80,12 +79,7 @@ func extractSendAmount(response []byte) string { } func TestMain(m *testing.M) { - // Parse the flags - flag.Parse() sessionID = GenerateSessionId() - // set the db - testutil.SetDatabase(*database, *connStr, *dbSchema) - // Cleanup the db after tests defer testutil.CleanDatabase() @@ -100,7 +94,8 @@ func TestAccountCreationSuccessful(t *testing.T) { for _, session := range sessions { groups := driver.FilterGroupsByName(session.Groups, "account_creation_successful") for _, group := range groups { - for _, step := range group.Steps { + for i, step := range group.Steps { + logg.TraceCtxf(ctx, "executing step", "i", i, "step", step) cont, err := en.Exec(ctx, []byte(step.Input)) if err != nil { t.Fatalf("Test case '%s' failed at input '%s': %v", group.Name, step.Input, err) @@ -142,7 +137,8 @@ func TestAccountRegistrationRejectTerms(t *testing.T) { for _, session := range sessions { groups := driver.FilterGroupsByName(session.Groups, "account_creation_reject_terms") for _, group := range groups { - for _, step := range group.Steps { + for i, step := range group.Steps { + logg.TraceCtxf(ctx, "executing step", "i", i, "step", step) cont, err := en.Exec(ctx, []byte(step.Input)) if err != nil { t.Fatalf("Test case '%s' failed at input '%s': %v", group.Name, step.Input, err) @@ -177,7 +173,8 @@ func TestMainMenuHelp(t *testing.T) { for _, session := range sessions { groups := driver.FilterGroupsByName(session.Groups, "main_menu_help") for _, group := range groups { - for _, step := range group.Steps { + for i, step := range group.Steps { + logg.TraceCtxf(ctx, "executing step", "i", i, "step", step) cont, err := en.Exec(ctx, []byte(step.Input)) if err != nil { t.Fatalf("Test case '%s' failed at input '%s': %v", group.Name, step.Input, err) diff --git a/testutil/engine.go b/testutil/engine.go index 67e5d60..50af445 100644 --- a/testutil/engine.go +++ b/testutil/engine.go @@ -4,20 +4,15 @@ import ( "context" "fmt" "log" - "net/url" "os" "path" "path/filepath" "time" - testdataloader "github.com/peteole/testdata-loader" "github.com/jackc/pgx/v5/pgxpool" "git.defalsify.org/vise.git/engine" - fsdb "git.defalsify.org/vise.git/db/fs" - "git.defalsify.org/vise.git/logging" "git.defalsify.org/vise.git/resource" - "git.grassecon.net/grassrootseconomics/visedriver/env" - "git.grassecon.net/grassrootseconomics/visedriver/config" + "git.grassecon.net/grassrootseconomics/sarafu-vise/config" "git.grassecon.net/grassrootseconomics/sarafu-vise/handlers" "git.grassecon.net/grassrootseconomics/visedriver/storage" "git.grassecon.net/grassrootseconomics/sarafu-api/testutil/testservice" @@ -26,81 +21,58 @@ import ( "git.grassecon.net/grassrootseconomics/sarafu-vise/testutil/testtag" ) -var ( - logg = logging.NewVanilla() - baseDir = testdataloader.GetBasePath() - scriptDir = path.Join(baseDir, "services", "registration") - setDbType string - setConnStr string - setDbSchema string -) - -func init() { - env.LoadEnvVariablesPath(baseDir) - config.LoadConfig() -} - -// SetDatabase updates the database used by TestEngine -func SetDatabase(database, connStr, dbSchema string) { - setDbType = database - setConnStr = connStr - setDbSchema = dbSchema -} - // CleanDatabase removes all test data from the database func CleanDatabase() { - if setDbType == "postgres" { - ctx := context.Background() - // Update the connection string with the new search path - updatedConnStr, err := updateSearchPath(setConnStr, setDbSchema) - if err != nil { - log.Fatalf("Failed to update search path: %v", err) - } + for _, v := range([]int8{ + storage.STORETYPE_STATE, + storage.STORETYPE_USER, + }) { + conn := conns[v] + logg.Infof("cleaning test database", "typ", v, "db", conn) + if conn.DbType() == storage.DBTYPE_POSTGRES { + ctx := context.Background() + // Update the connection string with the new search path + updatedConnStr := conn.String() - dbConn, err := pgxpool.New(ctx, updatedConnStr) - if err != nil { - log.Fatalf("Failed to connect to database for cleanup: %v", err) - } - defer dbConn.Close() + dbConn, err := pgxpool.New(ctx, updatedConnStr) + if err != nil { + log.Fatalf("Failed to connect to database for cleanup: %v", err) + } + defer dbConn.Close() - query := fmt.Sprintf("DELETE FROM %s.kv_vise;", setDbSchema) - _, execErr := dbConn.Exec(ctx, query) - if execErr != nil { - log.Printf("Failed to cleanup table %s.kv_vise: %v", setDbSchema, execErr) + setDbSchema := conn.Domain() + + query := fmt.Sprintf("DELETE FROM %s.kv_vise;", setDbSchema) + _, execErr := dbConn.Exec(ctx, query) + if execErr != nil { + log.Printf("Failed to cleanup table %s.kv_vise: %v", setDbSchema, execErr) + } else { + log.Printf("Successfully cleaned up table %s.kv_vise", setDbSchema) + } + } else if conn.DbType() == storage.DBTYPE_FS || conn.DbType() == storage.DBTYPE_GDBM { + connStr, _ := filepath.Abs(conn.Path()) + if err := os.RemoveAll(connStr); err != nil { + log.Fatalf("Failed to delete state store %v: %v", conn, err) + } } else { - log.Printf("Successfully cleaned up table %s.kv_vise", setDbSchema) - } - } else { - setConnStr, _ := filepath.Abs(setConnStr) - if err := os.RemoveAll(setConnStr); err != nil { - log.Fatalf("Failed to delete state store %s: %v", setConnStr, err) + logg.Errorf("store cleanup not handled") } } } -// updateSearchPath updates the search_path (schema) to be used in the connection -func updateSearchPath(connStr string, newSearchPath string) (string, error) { - u, err := url.Parse(connStr) - if err != nil { - return "", fmt.Errorf("invalid connection string: %w", err) - } - - // Parse the query parameters - q := u.Query() - - // Update or add the search_path parameter - q.Set("search_path", newSearchPath) - - // Rebuild the connection string with updated parameters - u.RawQuery = q.Encode() - - return u.String(), nil -} - func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { - var err error + config.LoadConfig() + err := config.Apply(&override) + if err != nil { + panic(fmt.Errorf("args override fail: %v\n", err)) + } + conns, err = config.GetConns() + if err != nil { + panic(fmt.Errorf("get conns fail: %v\n", err)) + } ctx := context.Background() ctx = context.WithValue(ctx, "SessionId", sessionId) + logg.InfoCtxf(ctx, "loaded engine setup", "conns", conns) pfp := path.Join(scriptDir, "pp.csv") var eventChannel = make(chan bool) @@ -112,48 +84,8 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { FlagCount: uint32(128), } - if setDbType == "postgres" { - conns, err := config.GetConns() - if err != nil { - fmt.Fprintf(os.Stderr, "Getconns error: %v", err) - os.Exit(1) - } - conn := conns[storage.STORETYPE_USER] - setConnStr = conn.String() - setConnStr, err = updateSearchPath(setConnStr, setDbSchema) - if err != nil { - fmt.Fprintf(os.Stderr, "Update search paths Error: %v\n", err) - os.Exit(1) - } - } else { - setConnStr, err = filepath.Abs(setConnStr) - if err != nil { - fmt.Fprintf(os.Stderr, "connstr err: %v", err) - os.Exit(1) - } - } - conn, err := storage.ToConnData(setConnStr) - if err != nil { - fmt.Fprintf(os.Stderr, "connstr parse err: %v", err) - os.Exit(1) - } - conns := storage.NewConns() - conns.Set(conn, storage.STORETYPE_STATE) - conns.Set(conn, storage.STORETYPE_USER) - - conn, err = storage.ToConnData(scriptDir) - if err != nil { - fmt.Fprintf(os.Stderr, "connstr parse err: %v", err) - os.Exit(1) - } - resourceConn := fsdb.NewFsDb() - err = resourceConn.Connect(ctx, scriptDir) - if err != nil { - fmt.Fprintf(os.Stderr, "resource connect err: %v", err) - os.Exit(1) - } menuStorageService := storage.NewMenuStorageService(conns) - menuStorageService = menuStorageService.WithDb(resourceConn, storage.STORETYPE_RESOURCE) + menuStorageService = menuStorageService.WithDb(resourceDb, storage.STORETYPE_RESOURCE) rs, err := menuStorageService.GetResource(ctx) if err != nil { @@ -182,7 +114,6 @@ func TestEngine(sessionId string) (engine.Engine, func(), chan bool) { lhs, err := handlers.NewLocalHandlerService(ctx, pfp, true, dbResource, cfg, rs) lhs.SetDataStore(&userDataStore) lhs.SetPersister(pe) - if err != nil { fmt.Fprintf(os.Stderr, err.Error()) os.Exit(1) diff --git a/testutil/fsdb.go b/testutil/fsdb.go new file mode 100644 index 0000000..03de985 --- /dev/null +++ b/testutil/fsdb.go @@ -0,0 +1,21 @@ +// +build testfsdb + +package testutil + +import ( + "os" + +) + +func init() { + d, err := os.MkdirTemp("", "sarafu-vise-menutraversal-state-") + if err != nil { + panic(err) + } + override.StateConn = &d + d, err = os.MkdirTemp("", "sarafu-vise-menutraversal-user-") + if err != nil { + panic(err) + } + override.UserConn = &d +} diff --git a/testutil/setup.go b/testutil/setup.go new file mode 100644 index 0000000..919641c --- /dev/null +++ b/testutil/setup.go @@ -0,0 +1,32 @@ +package testutil + +import ( + "context" + "path" + + testdataloader "github.com/peteole/testdata-loader" + "git.defalsify.org/vise.git/logging" + fsdb "git.defalsify.org/vise.git/db/fs" + "git.defalsify.org/vise.git/db" + "git.grassecon.net/grassrootseconomics/visedriver/storage" + "git.grassecon.net/grassrootseconomics/sarafu-vise/config" +) + +var ( + logg = logging.NewVanilla().WithDomain("sarafu-vise.testutil").WithContextKey("SessionId") + conns storage.Conns + resourceDb db.Db + baseDir = testdataloader.GetBasePath() + scriptDir = path.Join(baseDir, "services", "registration") + override config.Override +) + +func init() { + ctx := context.Background() + config.EnvPath = baseDir + resourceDb = fsdb.NewFsDb() + err := resourceDb.Connect(ctx, scriptDir) + if err != nil { + panic(err) + } +}