Compare commits

..

1 Commits

Author SHA1 Message Date
16dbdc6c4a config reporter try 1 2025-01-20 10:07:28 +01:00
21 changed files with 324 additions and 617 deletions

View File

@ -5,36 +5,18 @@ import (
"git.defalsify.org/vise.git/logging"
"git.grassecon.net/grassrootseconomics/visedriver/env"
"git.grassecon.net/grassrootseconomics/visedriver/storage"
)
var (
logg = logging.NewVanilla().WithDomain("visedriver-config")
defaultLanguage = "eng"
languages []string
DefaultLanguage string
dbConn string
dbConnMissing bool
dbConnMode storage.DbMode
stateDbConn string
stateDbConnMode storage.DbMode
resourceDbConn string
resourceDbConnMode storage.DbMode
userDbConn string
userDbConnMode storage.DbMode
Languages []string
defaultLanguage = "eng"
languages []string
)
type Override struct {
DbConn string
DbConnMode storage.DbMode
StateConn string
StateConnMode storage.DbMode
ResourceConn string
ResourceConnMode storage.DbMode
UserConn string
UserConnMode storage.DbMode
}
var (
DbConn string
DefaultLanguage string
Languages []string
)
func setLanguage() error {
defaultLanguage = env.GetEnv("DEFAULT_LANGUAGE", defaultLanguage)
@ -55,95 +37,31 @@ func setLanguage() error {
}
func setConn() error {
dbConn = env.GetEnv("DB_CONN", "?")
stateDbConn = env.GetEnv("DB_CONN_STATE", dbConn)
resourceDbConn = env.GetEnv("DB_CONN_RESOURCE", dbConn)
userDbConn = env.GetEnv("DB_CONN_USER", dbConn)
DbConn = env.GetEnv("DB_CONN", "")
return nil
}
func ApplyConn(override *Override) {
if override.DbConn != "?" {
dbConn = override.DbConn
stateDbConn = override.StateConn
resourceDbConn = override.ResourceConn
userDbConn = override.UserConn
}
dbConnMode = override.DbConnMode
if override.StateConn != "?" {
stateDbConn = override.StateConn
}
if override.ResourceConn != "?" {
resourceDbConn = override.ResourceConn
}
if override.UserConn != "?" {
userDbConn = override.UserConn
}
if dbConn == "?" {
dbConn = ""
}
if stateDbConn == "?" {
stateDbConn = dbConn
stateDbConnMode = dbConnMode
}
if resourceDbConn == "?" {
resourceDbConn = dbConn
resourceDbConnMode = dbConnMode
}
if userDbConn == "?" {
userDbConn = dbConn
userDbConnMode = dbConnMode
}
logg.Debugf("conns", "conn", dbConn, "user", userDbConn)
if override.DbConnMode != storage.DBMODE_ANY {
dbConnMode = override.DbConnMode
}
if override.StateConnMode != storage.DBMODE_ANY {
stateDbConnMode = override.StateConnMode
}
if override.ResourceConnMode != storage.DBMODE_ANY {
resourceDbConnMode = override.ResourceConnMode
}
if override.UserConnMode != storage.DBMODE_ANY {
userDbConnMode = override.UserConnMode
}
}
func GetConns() (storage.Conns, error) {
o := storage.NewConns()
c, err := storage.ToConnDataMode(stateDbConn, stateDbConnMode)
if err != nil {
return o, err
}
o.Set(c, storage.STORETYPE_STATE)
c, err = storage.ToConnDataMode(resourceDbConn, resourceDbConnMode)
if err != nil {
return o, err
}
o.Set(c, storage.STORETYPE_RESOURCE)
c, err = storage.ToConnDataMode(userDbConn, userDbConnMode)
if err != nil {
return o, err
}
o.Set(c, storage.STORETYPE_USER)
return o, nil
}
// LoadConfig initializes the configuration values after environment variables are loaded.
func LoadConfig() error {
err := setConn()
if err != nil {
if err := setLanguage(); err != nil {
return err
}
err = setLanguage()
if err != nil {
if err := setConn(); err != nil {
return err
}
DefaultLanguage = defaultLanguage
Languages = languages
// Create and use reporter
logger := logging.NewVanilla().WithDomain("config")
reporter := NewReporter(logger)
reporter.AddValue("CUSTOM_SETTING", func() string {
return env.GetEnv("CUSTOM_SETTING", "default")
}, false)
reporter.Report("INFO")
return nil
}

75
config/reporter.go Normal file
View File

@ -0,0 +1,75 @@
//go:build configreport
package config
import (
"git.defalsify.org/vise.git/logging"
"git.grassecon.net/grassrootseconomics/visedriver/env"
)
// ConfigValue represents a configuration key-value pair
type ConfigValue struct {
Key string
ValueFunc func() string
Sensitive bool
}
// Reporter handles configuration reporting
type Reporter struct {
values []ConfigValue
logger logging.Vanilla
}
func NewReporter(logger logging.Vanilla) *Reporter {
return &Reporter{
values: defaultConfigValues(),
logger: logger,
}
}
func defaultConfigValues() []ConfigValue {
return []ConfigValue{
{"HOST", func() string { return env.GetEnv("HOST", "127.0.0.1") }, false},
{"PORT", func() string { return env.GetEnv("PORT", "7123") }, false},
{"AT_ENDPOINT", func() string { return env.GetEnv("AT_ENDPOINT", "/ussd/africastalking") }, false},
{"DB_CONN", func() string { return env.GetEnv("DB_CONN", "") }, true},
{"DB_TIMEZONE", func() string { return env.GetEnv("DB_TIMEZONE", "") }, false},
{"DB_SCHEMA", func() string { return env.GetEnv("DB_SCHEMA", "") }, false},
{"CUSTODIAL_URL_BASE", func() string { return env.GetEnv("CUSTODIAL_URL_BASE", "") }, false},
{"BEARER_TOKEN", func() string { return env.GetEnv("BEARER_TOKEN", "") }, true},
{"DATA_URL_BASE", func() string { return env.GetEnv("DATA_URL_BASE", "") }, false},
{"DEFAULT_LANGUAGE", func() string { return env.GetEnv("DEFAULT_LANGUAGE", "eng") }, false},
{"LANGUAGES", func() string { return env.GetEnv("LANGUAGES", "eng") }, false},
}
}
func (r *Reporter) AddValue(key string, valueFn func() string, sensitive bool) {
r.values = append(r.values, ConfigValue{
Key: key,
ValueFunc: valueFn,
Sensitive: sensitive,
})
}
// Report outputs all configuration values at the specified log level
func (r *Reporter) Report(level string) {
r.logger.Debugf("Configuration Report:")
for _, cv := range r.values {
value := cv.ValueFunc()
if cv.Sensitive {
value = "****"
}
switch level {
case "DEBUG":
r.logger.Debugf("%s: %s", cv.Key, value)
case "INFO":
r.logger.Infof("%s: %s", cv.Key, value)
case "WARN":
r.logger.Warnf("%s: %s", cv.Key, value)
case "ERROR":
r.logger.Errorf("%s: %s", cv.Key, value)
default:
r.logger.Infof("%s: %s", cv.Key, value)
}
}
}

17
config/reporter_noop.go Normal file
View File

@ -0,0 +1,17 @@
//go:build !configreport
package config
import (
"git.defalsify.org/vise.git/logging"
)
type Reporter struct{}
func NewReporter(logger logging.Vanilla) *Reporter {
return &Reporter{}
}
func (r *Reporter) AddValue(key string, valueFn func() string, sensitive bool) {}
func (r *Reporter) Report(level string) {}

View File

@ -2,9 +2,8 @@ package entry
import (
"context"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/persist"
)
type EntryHandler interface {

2
go.mod
View File

@ -3,7 +3,7 @@ module git.grassecon.net/grassrootseconomics/visedriver
go 1.23.0
require (
git.defalsify.org/vise.git v0.2.3-0.20250204132233-2bffe532f21e
git.defalsify.org/vise.git v0.2.3-0.20250114225117-3b5fc85b650b
github.com/jackc/pgx/v5 v5.7.1
github.com/joho/godotenv v1.5.1
)

4
go.sum
View File

@ -1,5 +1,5 @@
git.defalsify.org/vise.git v0.2.3-0.20250204132233-2bffe532f21e h1:gtB9OdX6x5gQRM3W824dEurXuuf/YPInqgtv2KAp5Zo=
git.defalsify.org/vise.git v0.2.3-0.20250204132233-2bffe532f21e/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
git.defalsify.org/vise.git v0.2.3-0.20250114225117-3b5fc85b650b h1:rwWXMtNSn7aqhb4p1oVZkCA1vC7pVdohwW61QQM8fUs=
git.defalsify.org/vise.git v0.2.3-0.20250114225117-3b5fc85b650b/go.mod h1:jyBMe1qTYUz3mmuoC9JQ/TvFeW0vTanCUcPu3H8p4Ck=
github.com/barbashov/iso639-3 v0.0.0-20211020172741-1f4ffb2d8d1c h1:H9Nm+I7Cg/YVPpEV1RzU3Wq2pjamPc/UtHDgItcb7lE=
github.com/barbashov/iso639-3 v0.0.0-20211020172741-1f4ffb2d8d1c/go.mod h1:rGod7o6KPeJ+hyBpHfhi4v7blx9sf+QsHsA7KAsdN6U=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

View File

@ -1,87 +1,84 @@
package request
import (
"context"
"git.defalsify.org/vise.git/db"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
"git.grassecon.net/grassrootseconomics/visedriver/entry"
"git.grassecon.net/grassrootseconomics/visedriver/errors"
"git.grassecon.net/grassrootseconomics/visedriver/storage"
"git.grassecon.net/grassrootseconomics/visedriver/errors"
"git.grassecon.net/grassrootseconomics/visedriver/entry"
)
type EngineFunc func(engine.Config, resource.Resource, *persist.Persister) engine.Engine
type BaseRequestHandler struct {
cfgTemplate engine.Config
rp RequestParser
rs resource.Resource
hn entry.EntryHandler
provider storage.StorageProvider
engineFunc EngineFunc
rp RequestParser
rs resource.Resource
hn entry.EntryHandler
provider storage.StorageProvider
}
//func NewBaseRequestHandler(cfg engine.Config, rs resource.Resource, stateDb db.Db, userdataDb db.Db, rp request.RequestParser, hn *handlers.Handlers) *BaseRequestHandler {
func NewBaseRequestHandler(cfg engine.Config, rs resource.Resource, stateDb db.Db, userdataDb db.Db, rp RequestParser, hn entry.EntryHandler) *BaseRequestHandler {
h := &BaseRequestHandler{
return &BaseRequestHandler{
cfgTemplate: cfg,
rs: rs,
hn: hn,
rp: rp,
provider: storage.NewSimpleStorageProvider(stateDb, userdataDb),
}
h.engineFunc = h.getDefaultEngine
return h
}
func (f *BaseRequestHandler) WithEngineFunc(fn EngineFunc) *BaseRequestHandler {
f.engineFunc = fn
return f
}
func (f *BaseRequestHandler) Shutdown(ctx context.Context) {
err := f.provider.Close(ctx)
func (f *BaseRequestHandler) Shutdown() {
err := f.provider.Close()
if err != nil {
logg.Errorf("handler shutdown error", "err", err)
}
}
func (f *BaseRequestHandler) GetEngine(cfg engine.Config, rs resource.Resource, pr *persist.Persister) engine.Engine {
return f.engineFunc(cfg, rs, pr)
}
func (f *BaseRequestHandler) getDefaultEngine(cfg engine.Config, rs resource.Resource, pr *persist.Persister) engine.Engine {
en := engine.NewEngine(cfg, rs)
en = en.WithPersister(pr)
en = en.WithFirst(f.hn.Init)
if f.cfgTemplate.EngineDebug {
en = en.WithDebug(nil)
}
return en
}
func (f *BaseRequestHandler) Process(rqs RequestSession) (RequestSession, error) {
func(f *BaseRequestHandler) Process(rqs RequestSession) (RequestSession, error) {
var r bool
var err error
var ok bool
logg.InfoCtxf(rqs.Ctx, "new request", "data", rqs)
rqs.Storage, err = f.provider.Get(rqs.Ctx, rqs.Config.SessionId)
rqs.Storage, err = f.provider.Get(rqs.Config.SessionId)
if err != nil {
logg.ErrorCtxf(rqs.Ctx, "", "storage get error", err)
return rqs, errors.ErrStorage
}
//f.hn = f.hn.WithPersister(rqs.Storage.Persister)
f.hn.SetPersister(rqs.Storage.Persister)
defer func() {
f.hn.Exit()
}()
eni := f.GetEngine(rqs.Config, f.rs, rqs.Storage.Persister)
en, ok := eni.(*engine.DefaultEngine)
if !ok {
perr := f.provider.Put(rqs.Config.SessionId, rqs.Storage)
rqs.Storage = nil
if perr != nil {
logg.ErrorCtxf(rqs.Ctx, "", "storage put error", perr)
}
return rqs, errors.ErrEngineType
}
en = en.WithFirst(f.hn.Init)
if rqs.Config.EngineDebug {
en = en.WithDebug(nil)
}
rqs.Engine = en
rqs.Engine = f.GetEngine(rqs.Config, f.rs, rqs.Storage.Persister)
r, err = rqs.Engine.Exec(rqs.Ctx, rqs.Input)
if err != nil {
perr := f.provider.Put(rqs.Ctx, rqs.Config.SessionId, rqs.Storage)
perr := f.provider.Put(rqs.Config.SessionId, rqs.Storage)
rqs.Storage = nil
if perr != nil {
logg.ErrorCtxf(rqs.Ctx, "", "storage put error", perr)
@ -93,21 +90,21 @@ func (f *BaseRequestHandler) Process(rqs RequestSession) (RequestSession, error)
return rqs, nil
}
func (f *BaseRequestHandler) Output(rqs RequestSession) (RequestSession, error) {
func(f *BaseRequestHandler) Output(rqs RequestSession) (RequestSession, error) {
var err error
_, err = rqs.Engine.Flush(rqs.Ctx, rqs.Writer)
return rqs, err
}
func (f *BaseRequestHandler) Reset(ctx context.Context, rqs RequestSession) (RequestSession, error) {
defer f.provider.Put(ctx, rqs.Config.SessionId, rqs.Storage)
return rqs, rqs.Engine.Finish(ctx)
func(f *BaseRequestHandler) Reset(rqs RequestSession) (RequestSession, error) {
defer f.provider.Put(rqs.Config.SessionId, rqs.Storage)
return rqs, rqs.Engine.Finish()
}
func (f *BaseRequestHandler) GetConfig() engine.Config {
return f.cfgTemplate
}
func (f *BaseRequestHandler) GetRequestParser() RequestParser {
func(f *BaseRequestHandler) GetRequestParser() RequestParser {
return f.rp
}

View File

@ -5,8 +5,8 @@ import (
"strconv"
"git.defalsify.org/vise.git/logging"
"git.grassecon.net/grassrootseconomics/visedriver/errors"
"git.grassecon.net/grassrootseconomics/visedriver/request"
"git.grassecon.net/grassrootseconomics/visedriver/errors"
)
var (
@ -80,7 +80,7 @@ func (hh *HTTPRequestHandler) ServeHTTP(w http.ResponseWriter, req *http.Request
w.WriteHeader(200)
w.Header().Set("Content-Type", "text/plain")
rqs, err = hh.Output(rqs)
rqs, perr = hh.Reset(rqs.Ctx, rqs)
rqs, perr = hh.Reset(rqs)
if err != nil {
hh.WriteError(w, 500, err)
return

View File

@ -10,8 +10,8 @@ import (
"git.defalsify.org/vise.git/engine"
viseerrors "git.grassecon.net/grassrootseconomics/visedriver/errors"
"git.grassecon.net/grassrootseconomics/visedriver/request"
"git.grassecon.net/grassrootseconomics/visedriver/testutil/mocks/httpmocks"
"git.grassecon.net/grassrootseconomics/visedriver/request"
)
// invalidRequestType is a custom type to test invalid request scenarios
@ -88,7 +88,7 @@ func TestRequestHandler_ServeHTTP(t *testing.T) {
OutputFunc: func(rs request.RequestSession) (request.RequestSession, error) {
return rs, tt.outputErr
},
ResetFunc: func(ctx context.Context, rs request.RequestSession) (request.RequestSession, error) {
ResetFunc: func(rs request.RequestSession) (request.RequestSession, error) {
return rs, tt.resetErr
},
GetRequestParserFunc: func() request.RequestParser {
@ -165,7 +165,7 @@ func TestDefaultRequestParser_GetSessionId(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id, err := parser.GetSessionId(context.Background(), tt.request)
id, err := parser.GetSessionId(context.Background(),tt.request)
if id != tt.expectedID {
t.Errorf("Expected session ID %s, got %s", tt.expectedID, id)

View File

@ -4,10 +4,10 @@ import (
"context"
"io"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
"git.grassecon.net/grassrootseconomics/visedriver/storage"
)
@ -16,27 +16,27 @@ var (
)
type RequestSession struct {
Ctx context.Context
Config engine.Config
Engine engine.Engine
Input []byte
Storage *storage.Storage
Writer io.Writer
Ctx context.Context
Config engine.Config
Engine engine.Engine
Input []byte
Storage *storage.Storage
Writer io.Writer
Continue bool
}
// TODO: seems like can remove this.
type RequestParser interface {
GetSessionId(context.Context, any) (string, error)
GetInput(any) ([]byte, error)
GetSessionId(ctx context.Context, rq any) (string, error)
GetInput(rq any) ([]byte, error)
}
type RequestHandler interface {
GetConfig() engine.Config
GetRequestParser() RequestParser
GetEngine(engine.Config, resource.Resource, *persist.Persister) engine.Engine
Process(RequestSession) (RequestSession, error)
Output(RequestSession) (RequestSession, error)
Reset(context.Context, RequestSession) (RequestSession, error)
Shutdown(ctx context.Context)
GetEngine(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine
Process(rs RequestSession) (RequestSession, error)
Output(rs RequestSession) (RequestSession, error)
Reset(rs RequestSession) (RequestSession, error)
Shutdown()
}

View File

@ -1,98 +0,0 @@
package storage
import (
"fmt"
"net/url"
)
type DbMode uint8
const (
DBTYPE_NONE = iota
DBTYPE_MEM
DBTYPE_FS
DBTYPE_GDBM
DBTYPE_POSTGRES
)
const (
DBMODE_ANY DbMode = iota
DBMODE_BINARY
DBMODE_TEXT
)
const (
STORETYPE_STATE = iota
STORETYPE_RESOURCE
STORETYPE_USER
_STORETYPE_MAX
)
var (
DbModeDebug = []string{"ANY", "BIN", "TXT"}
DbTypeDebug = []string{"NONE", "MEM", "FS", "GDBM", "POSTGRES"}
DbStoreDebug = []string{"STATE", "RESOURCE", "USER"}
)
type Conns map[int8]ConnData
func NewConns() Conns {
c := make(Conns)
return c
}
func (c Conns) Set(conn ConnData, typ int8) {
if typ < 0 || typ >= _STORETYPE_MAX {
panic(fmt.Errorf("invalid store type: %d", typ))
}
c[typ] = conn
}
func (c Conns) Have(conn *ConnData) int8 {
for i := range _STORETYPE_MAX {
ii := int8(i)
v, ok := c[ii]
if !ok {
continue
}
if v.Raw() == conn.Raw() {
if v.Mode() == DBMODE_ANY || v.Mode() == conn.Mode() {
return ii
}
}
}
return -1
}
type ConnData struct {
typ int
str string
domain string
mode DbMode
}
func (cd *ConnData) DbType() int {
return cd.typ
}
func (cd ConnData) String() string {
return fmt.Sprintf("conn: %s, mod %s, typ %s", cd.str, DbModeDebug[uint8(cd.mode)], DbTypeDebug[uint8(cd.typ)])
}
func (cd *ConnData) Domain() string {
return cd.domain
}
func (cd *ConnData) Mode() DbMode {
return cd.mode
}
func (cd *ConnData) Path() string {
v, _ := url.Parse(cd.str)
v.RawQuery = ""
return v.String()
}
func (cd *ConnData) Raw() string {
return cd.str
}

View File

@ -18,7 +18,7 @@ var (
)
type ThreadGdbmDb struct {
db db.Db
db db.Db
connStr string
}
@ -29,7 +29,7 @@ func NewThreadGdbmDb() *ThreadGdbmDb {
return &ThreadGdbmDb{}
}
func (tdb *ThreadGdbmDb) Connect(ctx context.Context, connStr string) error {
func(tdb *ThreadGdbmDb) Connect(ctx context.Context, connStr string) error {
var ok bool
_, ok = dbC[connStr]
if ok {
@ -42,18 +42,18 @@ func (tdb *ThreadGdbmDb) Connect(ctx context.Context, connStr string) error {
return err
}
dbC[connStr] = make(chan db.Db, 1)
dbC[connStr] <- gdb
dbC[connStr]<- gdb
tdb.connStr = connStr
return nil
}
func (tdb *ThreadGdbmDb) reserve() {
func(tdb *ThreadGdbmDb) reserve() {
if tdb.db == nil {
tdb.db = <-dbC[tdb.connStr]
}
}
func (tdb *ThreadGdbmDb) release() {
func(tdb *ThreadGdbmDb) release() {
if tdb.db == nil {
return
}
@ -61,87 +61,67 @@ func (tdb *ThreadGdbmDb) release() {
tdb.db = nil
}
func (tdb *ThreadGdbmDb) SetPrefix(pfx uint8) {
func(tdb *ThreadGdbmDb) SetPrefix(pfx uint8) {
tdb.reserve()
tdb.db.SetPrefix(pfx)
}
func (tdb *ThreadGdbmDb) SetSession(sessionId string) {
func(tdb *ThreadGdbmDb) SetSession(sessionId string) {
tdb.reserve()
tdb.db.SetSession(sessionId)
}
func (tdb *ThreadGdbmDb) SetLanguage(lng *lang.Language) {
func(tdb *ThreadGdbmDb) SetLanguage(lng *lang.Language) {
tdb.reserve()
tdb.db.SetLanguage(lng)
}
func (tdb *ThreadGdbmDb) Safe() bool {
func(tdb *ThreadGdbmDb) Safe() bool {
tdb.reserve()
v := tdb.db.Safe()
tdb.release()
return v
}
func (tdb *ThreadGdbmDb) Prefix() uint8 {
func(tdb *ThreadGdbmDb) Prefix() uint8 {
tdb.reserve()
v := tdb.db.Prefix()
tdb.release()
return v
}
func (tdb *ThreadGdbmDb) SetLock(typ uint8, locked bool) error {
func(tdb *ThreadGdbmDb) SetLock(typ uint8, locked bool) error {
tdb.reserve()
err := tdb.db.SetLock(typ, locked)
tdb.release()
return err
}
func (tdb *ThreadGdbmDb) Put(ctx context.Context, key []byte, val []byte) error {
func(tdb *ThreadGdbmDb) Put(ctx context.Context, key []byte, val []byte) error {
tdb.reserve()
err := tdb.db.Put(ctx, key, val)
tdb.release()
return err
}
func (tdb *ThreadGdbmDb) Get(ctx context.Context, key []byte) ([]byte, error) {
func(tdb *ThreadGdbmDb) Get(ctx context.Context, key []byte) ([]byte, error) {
tdb.reserve()
v, err := tdb.db.Get(ctx, key)
tdb.release()
return v, err
}
func (tdb *ThreadGdbmDb) Close(ctx context.Context) error {
func(tdb *ThreadGdbmDb) Close() error {
tdb.reserve()
close(dbC[tdb.connStr])
delete(dbC, tdb.connStr)
err := tdb.db.Close(ctx)
err := tdb.db.Close()
tdb.db = nil
return err
}
func (tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
func(tdb *ThreadGdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
tdb.reserve()
defer tdb.release()
return tdb.db.Dump(ctx, key)
}
func (tdb *ThreadGdbmDb) DecodeKey(ctx context.Context, key []byte) ([]byte, error) {
return tdb.db.DecodeKey(ctx, key)
}
func (tdb *ThreadGdbmDb) Abort(ctx context.Context) {
tdb.db.Abort(ctx)
}
func (tdb *ThreadGdbmDb) Start(ctx context.Context) error {
return tdb.db.Start(ctx)
}
func (tdb *ThreadGdbmDb) Stop(ctx context.Context) error {
return tdb.db.Stop(ctx)
}
func (tdb *ThreadGdbmDb) Connection() string {
return tdb.db.Connection()
}

View File

@ -4,9 +4,40 @@ import (
"fmt"
"net/url"
"path"
"path/filepath"
)
const (
DBTYPE_NONE = iota
DBTYPE_MEM
DBTYPE_FS
DBTYPE_GDBM
DBTYPE_POSTGRES
)
type ConnData struct {
typ int
str string
domain string
}
func (cd *ConnData) DbType() int {
return cd.typ
}
func (cd *ConnData) String() string {
return cd.str
}
func (cd *ConnData) Domain() string {
return cd.domain
}
func (cd *ConnData) Path() string {
v, _ := url.Parse(cd.str)
v.RawQuery = ""
return v.String()
}
func probePostgres(s string) (string, string, bool) {
domain := "public"
v, err := url.Parse(s)
@ -37,18 +68,8 @@ func probeGdbm(s string) (string, string, bool) {
}
func probeFs(s string) (string, string, bool) {
var err error
v, _ := url.Parse(s)
if v.Scheme != "" && v.Scheme != "file://" {
return "", "", false
}
if !path.IsAbs(s) {
s, err = filepath.Abs(s)
if err != nil {
panic(err)
}
return "", "", false
}
s = path.Clean(s)
return s, "", true
@ -61,25 +82,14 @@ func probeMem(s string) (string, string, bool) {
return "", "", true
}
func ToConnDataMode(connStr string, mode DbMode) (ConnData, error) {
o, err := ToConnData(connStr)
if err != nil {
return o, err
}
o.mode = mode
return o, nil
}
func ToConnData(connStr string) (ConnData, error) {
var o ConnData
v, domain, ok := probeMem(connStr)
if ok {
o.typ = DBTYPE_MEM
if connStr == "" {
return o, nil
}
v, domain, ok = probePostgres(connStr)
v, domain, ok := probePostgres(connStr)
if ok {
o.typ = DBTYPE_POSTGRES
o.str = v
@ -101,5 +111,11 @@ func ToConnData(connStr string) (ConnData, error) {
return o, nil
}
v, _, ok = probeMem(connStr)
if ok {
o.typ = DBTYPE_MEM
return o, nil
}
return o, fmt.Errorf("invalid connection string: %s", connStr)
}

View File

@ -5,53 +5,24 @@ import (
)
func TestParseConnStr(t *testing.T) {
v, err := ToConnData("postgres://foo:bar@localhost:5432/baz")
_, err := ToConnData("postgres://foo:bar@localhost:5432/baz")
if err != nil {
t.Fatal(err)
t.Fatal(err)
}
if v.DbType() != DBTYPE_POSTGRES {
t.Fatalf("expected type %v, got %v", DBTYPE_POSTGRES, v.DbType())
}
v, err = ToConnData("gdbm:///foo/bar")
_, err = ToConnData("/foo/bar")
if err != nil {
t.Fatal(err)
t.Fatal(err)
}
if v.DbType() != DBTYPE_GDBM {
t.Fatalf("expected type %v, got %v", DBTYPE_GDBM, v.DbType())
}
v, err = ToConnData("/foo/bar")
_, err = ToConnData("/foo/bar/")
if err != nil {
t.Fatal(err)
t.Fatal(err)
}
if v.DbType() != DBTYPE_FS {
t.Fatalf("expected type %v, got %v", DBTYPE_FS, v.DbType())
}
v, err = ToConnData("/foo/bar/")
if err != nil {
t.Fatal(err)
}
if v.DbType() != DBTYPE_FS {
t.Fatalf("expected type %v, got %v", DBTYPE_FS, v.DbType())
}
v, err = ToConnData("foo/bar")
if err != nil {
t.Fatal(err)
}
if v.DbType() != DBTYPE_FS {
t.Fatalf("expected type %v, got %v", DBTYPE_FS, v.DbType())
}
v, err = ToConnData("")
if err != nil {
t.Fatal(err)
}
if v.DbType() != DBTYPE_MEM {
t.Fatalf("expected type %v, got %v", DBTYPE_MEM, v.DbType())
}
v, err = ToConnData("http://foo/bar")
_, err = ToConnData("foo/bar")
if err == nil {
t.Fatalf("expected error")
}
if v.DbType() != DBTYPE_NONE {
t.Fatalf("expected type %v, got %v", DBTYPE_NONE, v.DbType())
_, err = ToConnData("http://foo/bar")
if err == nil {
t.Fatalf("expected error")
}
}

View File

@ -1,8 +1,6 @@
package storage
import (
"context"
"git.defalsify.org/vise.git/db"
"git.defalsify.org/vise.git/persist"
)
@ -12,18 +10,14 @@ const (
)
type Storage struct {
Persister *persist.Persister
UserdataDb db.Db
}
func (s *Storage) Close(ctx context.Context) error {
return s.UserdataDb.Close(ctx)
Persister *persist.Persister
UserdataDb db.Db
}
type StorageProvider interface {
Get(ctx context.Context, sessionId string) (*Storage, error)
Put(ctx context.Context, sessionId string, storage *Storage) error
Close(ctx context.Context) error
Get(sessionId string) (*Storage, error)
Put(sessionId string, storage *Storage) error
Close() error
}
type SimpleStorageProvider struct {
@ -35,22 +29,20 @@ func NewSimpleStorageProvider(stateStore db.Db, userdataStore db.Db) StorageProv
pe = pe.WithFlush()
return &SimpleStorageProvider{
Storage: &Storage{
Persister: pe,
Persister: pe,
UserdataDb: userdataStore,
},
}
}
func (p *SimpleStorageProvider) Get(ctx context.Context, sessionId string) (*Storage, error) {
p.Storage.UserdataDb.Start(ctx)
func (p *SimpleStorageProvider) Get(sessionId string) (*Storage, error) {
return p.Storage, nil
}
func (p *SimpleStorageProvider) Put(ctx context.Context, sessionId string, storage *Storage) error {
storage.UserdataDb.Stop(ctx)
func (p *SimpleStorageProvider) Put(sessionId string, storage *Storage) error {
return nil
}
func (p *SimpleStorageProvider) Close(ctx context.Context) error {
return p.Storage.Close(ctx)
func (p *SimpleStorageProvider) Close() error {
return p.Storage.UserdataDb.Close()
}

View File

@ -2,21 +2,19 @@ package storage
import (
"context"
"errors"
"fmt"
"os"
"path"
"github.com/jackc/pgx/v5/pgxpool"
"git.defalsify.org/vise.git/db"
fsdb "git.defalsify.org/vise.git/db/fs"
memdb "git.defalsify.org/vise.git/db/mem"
"git.defalsify.org/vise.git/db/postgres"
"git.defalsify.org/vise.git/lang"
"git.defalsify.org/vise.git/logging"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
gdbmstorage "git.grassecon.net/grassrootseconomics/visedriver/storage/db/gdbm"
"github.com/jackc/pgx/v5/pgxpool"
)
var (
@ -29,107 +27,76 @@ type StorageService interface {
GetResource(ctx context.Context) (resource.Resource, error)
}
// TODO: Support individual backend for each store (conndata)
type MenuStorageService struct {
conns Conns
poResource resource.Resource
store map[int8]db.Db
conn ConnData
resourceDir string
poResource resource.Resource
resourceStore db.Db
stateStore db.Db
userDataStore db.Db
}
func NewMenuStorageService(conn Conns) *MenuStorageService {
func NewMenuStorageService(conn ConnData, resourceDir string) *MenuStorageService {
return &MenuStorageService{
conns: conn,
store: make(map[int8]db.Db),
conn: conn,
resourceDir: resourceDir,
}
}
func (ms *MenuStorageService) WithDb(store db.Db, typ int8) *MenuStorageService {
var err error
if ms.store[typ] != nil {
panic(fmt.Errorf("db already set for typ: %d", typ))
}
ms.store[typ] = store
ms.conns[typ], err = ToConnData(store.Connection())
if err != nil {
panic(err)
}
func (ms *MenuStorageService) WithResourceDir(resourceDir string) *MenuStorageService {
ms.resourceDir = resourceDir
return ms
}
func (ms *MenuStorageService) checkDb(ctx context.Context, typ int8) db.Db {
store := ms.store[typ]
if store != nil {
return store
}
connData := ms.conns[typ]
logg.DebugCtxf(ctx, "db check", "conn", connData, "store", DbStoreDebug[typ])
v := ms.conns.Have(&connData)
if v == -1 {
return nil
}
src := ms.store[v]
if src == nil {
return nil
}
ms.store[typ] = ms.store[v]
logg.DebugCtxf(ctx, "found existing db", "typ", typ, "srctyp", v, "store", ms.store[typ], "srcstore", ms.store[v], "conn", connData)
return ms.store[typ]
}
func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, section string, typ int8) (db.Db, error) {
// TODO: allow fsdb, memdb
func (ms *MenuStorageService) getOrCreateDb(ctx context.Context, existingDb db.Db, section string, typ string) (db.Db, error) {
var newDb db.Db
var err error
newDb := ms.checkDb(ctx, typ)
if newDb != nil {
logg.InfoCtxf(ctx, "using existing db", "typ", typ, "db", newDb)
return newDb, nil
if existingDb != nil {
return existingDb, nil
}
connData := ms.conns[typ]
connStr := connData.Raw()
dbTyp := connData.DbType()
connStr := ms.conn.String()
dbTyp := ms.conn.DbType()
if dbTyp == DBTYPE_POSTGRES {
// TODO: move to vise
err = ensureSchemaExists(ctx, connData)
err = ensureSchemaExists(ctx, ms.conn)
if err != nil {
return nil, err
}
newDb = postgres.NewPgDb().WithSchema(connData.Domain())
newDb = postgres.NewPgDb().WithSchema(ms.conn.Domain())
} else if dbTyp == DBTYPE_GDBM {
err = ms.ensureDbDir(connStr)
err = ms.ensureDbDir()
if err != nil {
return nil, err
}
connStr = path.Join(connStr, section)
newDb = gdbmstorage.NewThreadGdbmDb()
} else if dbTyp == DBTYPE_FS {
err = ms.ensureDbDir(connStr)
err = ms.ensureDbDir()
if err != nil {
return nil, err
}
fsdbInstance := fsdb.NewFsDb()
if connData.Mode() == DBMODE_BINARY {
fsdbInstance = fsdbInstance.WithBinary()
}
newDb = fsdbInstance
newDb = fsdb.NewFsDb().WithBinary()
} else if dbTyp == DBTYPE_MEM {
logg.WarnCtxf(ctx, "using volatile storage (memdb)")
newDb = memdb.NewMemDb()
} else {
return nil, fmt.Errorf("unsupported connection string: '%s'\n", connData.Raw())
return nil, fmt.Errorf("unsupported connection string: '%s'\n", ms.conn.String())
}
logg.InfoCtxf(ctx, "connecting to db", "conn", connData, "typ", typ)
logg.DebugCtxf(ctx, "connecting to db", "conn", connStr, "conndata", ms.conn, "typ", typ)
err = newDb.Connect(ctx, connStr)
if err != nil {
return nil, err
}
ms.store[typ] = newDb
return newDb, nil
}
// WithGettext triggers use of gettext for translation of templates and menus.
//
// The first language in `lns` will be used as default language, to resolve node keys to
// The first language in `lns` will be used as default language, to resolve node keys to
// language strings.
//
// If `lns` is an empty array, gettext will not be used.
@ -140,7 +107,7 @@ func (ms *MenuStorageService) WithGettext(path string, lns []lang.Language) *Men
}
rs := resource.NewPoResource(lns[0], path)
for _, ln := range lns {
for _, ln := range(lns) {
rs = rs.WithLanguage(ln)
}
@ -166,49 +133,38 @@ func ensureSchemaExists(ctx context.Context, conn ConnData) error {
return nil
}
func applySession(ctx context.Context, store db.Db) error {
sessionId, ok := ctx.Value("SessionId").(string)
if !ok {
logg.DebugCtxf(ctx, "missing session to apply", "store", store)
return nil
//return fmt.Errorf("missing session to apply to store: %v", store)
}
store.SetSession(sessionId)
return nil
}
func (ms *MenuStorageService) GetPersister(ctx context.Context) (*persist.Persister, error) {
stateStore, err := ms.GetStateStore(ctx)
if err != nil {
return nil, err
}
err = applySession(ctx, stateStore)
if err != nil {
return nil, err
}
pr := persist.NewPersister(stateStore)
logg.TraceCtxf(ctx, "menu storage service", "persist", pr, "store", stateStore)
return pr, nil
}
func (ms *MenuStorageService) GetUserdataDb(ctx context.Context) (db.Db, error) {
userStore, err := ms.getOrCreateDb(ctx, "userdata.gdbm", STORETYPE_USER)
if ms.userDataStore != nil {
return ms.userDataStore, nil
}
userDataStore, err := ms.getOrCreateDb(ctx, ms.userDataStore, "userdata.gdbm", "userdata")
if err != nil {
return nil, err
}
err = applySession(ctx, userStore)
if err != nil {
return nil, err
}
return userStore, nil
ms.userDataStore = userDataStore
return ms.userDataStore, nil
}
func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resource, error) {
store, err := ms.getOrCreateDb(ctx, "resource.gdbm", STORETYPE_RESOURCE)
ms.resourceStore = fsdb.NewFsDb()
err := ms.resourceStore.Connect(ctx, ms.resourceDir)
if err != nil {
return nil, err
}
rfs := resource.NewDbResource(store)
rfs := resource.NewDbResource(ms.resourceStore)
if ms.poResource != nil {
logg.InfoCtxf(ctx, "using poresource for menu and template")
rfs.WithMenuGetter(ms.poResource.GetMenu)
@ -218,34 +174,33 @@ func (ms *MenuStorageService) GetResource(ctx context.Context) (resource.Resourc
}
func (ms *MenuStorageService) GetStateStore(ctx context.Context) (db.Db, error) {
return ms.getOrCreateDb(ctx, "state.gdbm", STORETYPE_STATE)
}
if ms.stateStore != nil {
return ms.stateStore, nil
}
func (ms *MenuStorageService) ensureDbDir(path string) error {
err := os.MkdirAll(path, 0700)
stateStore, err := ms.getOrCreateDb(ctx, ms.stateStore, "state.gdbm", "state")
if err != nil {
return fmt.Errorf("store dir create exited with error: %v\n", err)
return nil, err
}
ms.stateStore = stateStore
return ms.stateStore, nil
}
func (ms *MenuStorageService) ensureDbDir() error {
err := os.MkdirAll(ms.conn.String(), 0700)
if err != nil {
return fmt.Errorf("state dir create exited with error: %v\n", err)
}
return nil
}
// TODO: how to handle persister here?
func (ms *MenuStorageService) Close(ctx context.Context) error {
var errs []error
var haveErr bool
for i := range _STORETYPE_MAX {
err := ms.store[int8(i)].Close(ctx)
if err != nil {
haveErr = true
}
errs = append(errs, err)
}
if haveErr {
errStr := ""
for i, err := range errs {
errStr += fmt.Sprintf("(%d: %v)", i, err)
}
return errors.New(errStr)
func (ms *MenuStorageService) Close() error {
errA := ms.stateStore.Close()
errB := ms.userDataStore.Close()
errC := ms.resourceStore.Close()
if errA != nil || errB != nil || errC != nil {
return fmt.Errorf("%v %v %v", errA, errB, errC)
}
return nil
}

View File

@ -1,114 +0,0 @@
package storage
import (
"context"
"os"
"testing"
fsdb "git.defalsify.org/vise.git/db/fs"
)
func TestMenuStorageServiceOneSet(t *testing.T) {
d, err := os.MkdirTemp("", "visedriver-menustorageservice")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(d)
conns := NewConns()
connData, err := ToConnData(d)
if err != nil {
t.Fatal(err)
}
conns.Set(connData, STORETYPE_STATE)
ctx := context.Background()
ms := NewMenuStorageService(conns)
_, err = ms.GetStateStore(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetResource(ctx)
if err == nil {
t.Fatalf("expected error getting resource")
}
_, err = ms.GetUserdataDb(ctx)
if err == nil {
t.Fatalf("expected error getting userdata")
}
}
func TestMenuStorageServiceExplicit(t *testing.T) {
d, err := os.MkdirTemp("", "visedriver-menustorageservice")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(d)
conns := NewConns()
connData, err := ToConnData(d)
if err != nil {
t.Fatal(err)
}
conns.Set(connData, STORETYPE_STATE)
ctx := context.Background()
d, err = os.MkdirTemp("", "visedriver-menustorageservice")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(d)
store := fsdb.NewFsDb()
err = store.Connect(ctx, d)
if err != nil {
t.Fatal(err)
}
ms := NewMenuStorageService(conns)
ms = ms.WithDb(store, STORETYPE_RESOURCE)
_, err = ms.GetStateStore(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetResource(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetUserdataDb(ctx)
if err == nil {
t.Fatalf("expected error getting userdata")
}
}
func TestMenuStorageServiceReuse(t *testing.T) {
d, err := os.MkdirTemp("", "visedriver-menustorageservice")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(d)
conns := NewConns()
connData, err := ToConnData(d)
if err != nil {
t.Fatal(err)
}
conns.Set(connData, STORETYPE_STATE)
conns.Set(connData, STORETYPE_USER)
ctx := context.Background()
ctx = context.WithValue(ctx, "SessionId", "foo")
ms := NewMenuStorageService(conns)
stateStore, err := ms.GetStateStore(ctx)
if err != nil {
t.Fatal(err)
}
_, err = ms.GetResource(ctx)
if err == nil {
t.Fatalf("expected error getting resource")
}
userStore, err := ms.GetUserdataDb(ctx)
if err != nil {
t.Fatal(err)
}
if userStore != stateStore {
t.Fatalf("expected same store, but they are %p and %p", userStore, stateStore)
}
}

View File

@ -10,7 +10,7 @@ type MockEngine struct {
InitFunc func(context.Context) (bool, error)
ExecFunc func(context.Context, []byte) (bool, error)
FlushFunc func(context.Context, io.Writer) (int, error)
FinishFunc func(context.Context) error
FinishFunc func() error
}
func (m *MockEngine) Init(ctx context.Context) (bool, error) {
@ -25,6 +25,6 @@ func (m *MockEngine) Flush(ctx context.Context, w io.Writer) (int, error) {
return m.FlushFunc(ctx, w)
}
func (m *MockEngine) Finish(ctx context.Context) error {
return m.FinishFunc(ctx)
func (m *MockEngine) Finish() error {
return m.FinishFunc()
}

View File

@ -1,8 +1,6 @@
package httpmocks
import (
"context"
"git.defalsify.org/vise.git/engine"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
@ -15,8 +13,8 @@ type MockRequestHandler struct {
GetConfigFunc func() engine.Config
GetEngineFunc func(cfg engine.Config, rs resource.Resource, pe *persist.Persister) engine.Engine
OutputFunc func(rs request.RequestSession) (request.RequestSession, error)
ResetFunc func(ctx context.Context, rs request.RequestSession) (request.RequestSession, error)
ShutdownFunc func(ctx context.Context)
ResetFunc func(rs request.RequestSession) (request.RequestSession, error)
ShutdownFunc func()
GetRequestParserFunc func() request.RequestParser
}
@ -36,12 +34,12 @@ func (m *MockRequestHandler) Output(rs request.RequestSession) (request.RequestS
return m.OutputFunc(rs)
}
func (m *MockRequestHandler) Reset(ctx context.Context, rs request.RequestSession) (request.RequestSession, error) {
return m.ResetFunc(ctx, rs)
func (m *MockRequestHandler) Reset(rs request.RequestSession) (request.RequestSession, error) {
return m.ResetFunc(rs)
}
func (m *MockRequestHandler) Shutdown(ctx context.Context) {
m.ShutdownFunc(ctx)
func (m *MockRequestHandler) Shutdown() {
m.ShutdownFunc()
}
func (m *MockRequestHandler) GetRequestParser() request.RequestParser {

View File

@ -22,4 +22,4 @@ func (m *MockWriter) Header() http.Header {
return http.Header{}
}
func (m *MockWriter) WriteHeader(statusCode int) {}
func (m *MockWriter) WriteHeader(statusCode int) {}

View File

@ -5,8 +5,8 @@ import (
"git.defalsify.org/vise.git/db"
memdb "git.defalsify.org/vise.git/db/mem"
"git.defalsify.org/vise.git/persist"
"git.defalsify.org/vise.git/resource"
"git.defalsify.org/vise.git/persist"
)
type MemStorageService struct {
@ -39,3 +39,4 @@ func (mss *MemStorageService) GetUserdataDb(ctx context.Context) (db.Db, error)
func (mss *MemStorageService) GetResource(ctx context.Context) (resource.Resource, error) {
return mss.rs, nil
}