implement data path for database

chainqueue/runnable/server.py

@@ -12,8 +12,9 @@ import uuid
 # external imports
 import confini
 from xdg.BaseDirectory import (
-        load_first_config,
+        xdg_data_dirs,
         get_runtime_dir,
+        load_first_config,
         )
 from hexathon import strip_0x
 from chainlib.chain import ChainSpec
@@ -30,13 +31,15 @@ logg = logging.getLogger()
 default_config_dir = load_first_config('chainqueue')
 config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
 
-runtime_dir = os.path.join(get_runtime_dir(), 'chainqueue')
+default_runtime_dir = os.path.join(get_runtime_dir(), 'chainqueue')
+default_data_dir = os.path.join(xdg_data_dirs[0], 'chainqueue')
 
 argparser = argparse.ArgumentParser('chainqueue transaction submission and trigger server')
 argparser.add_argument('-c', '--config', dest='c', type=str, default=config_dir, help='configuration directory')
 argparser.add_argument('-p', type=str, help='rpc endpoint')
 argparser.add_argument('-i', type=str, help='chain spec')
-argparser.add_argument('--runtime-dir', dest='runtime_dir', type=str, default=runtime_dir, help='runtime directory')
+argparser.add_argument('--runtime-dir', dest='runtime_dir', type=str, default=default_runtime_dir, help='runtime directory')
+argparser.add_argument('--data-dir', dest='data_dir', type=str, default=default_data_dir, help='data directory')
 argparser.add_argument('--session-id', dest='session_id', type=str, default=str(uuid.uuid4()), help='session id to use for session')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be very verbose')
@@ -47,12 +50,14 @@ if args.vv:
 elif args.v:
     logg.setLevel(logging.INFO)
 
+# process config
 config = confini.Config(args.c)
 config.process()
 args_override = {
             'SESSION_RUNTIME_DIR': getattr(args, 'runtime_dir'),
             'SESSION_CHAIN_SPEC': getattr(args, 'i'),
             'RPC_ENDPOINT': getattr(args, 'p'),
+            'PATH_DATA': getattr(args, 'data_dir'),
         }
 config.dict_override(args_override, 'cli args')
 config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
@@ -60,9 +65,22 @@ config.add(getattr(args, 'session_id'), '_SESSION_ID', True)
 if not config.get('SESSION_SOCKET_PATH'):
     socket_path = os.path.join(config.get('SESSION_RUNTIME_DIR'), config.get('_SESSION_ID'), 'chainqueue.sock')
     config.add(socket_path, 'SESSION_SOCKET_PATH', True)
+
+if config.get('DATABASE_ENGINE') == 'sqlite':
+    config.add(os.path.join(config.get('PATH_DATA'), config.get('DATABASE_NAME')), 'DATABASE_NAME', True)
+    
+
 logg.debug('config loaded:\n{}'.format(config))
 
 
+# verify setup
+try:
+    os.stat(config.get('DATABASE_NAME'))
+except FileNotFoundError:
+    sys.stderr.write('database file {} not found. please run database migration script first'.format(config.get('DATABASE_NAME')))
+    sys.exit(1)
+
+
 class SessionController:
 
     def __init__(self, config):

scripts/migrate.py

@@ -8,6 +8,7 @@ import alembic
 from alembic.config import Config as AlembicConfig
 import confini
 from xdg.BaseDirectory import (
+        xdg_data_dirs,
         load_first_config,
     )
 
@@ -17,10 +18,10 @@ from chainqueue.db import dsn_from_config
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-# BUG: the dbdir doesn't work after script install
 rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'chainqueue', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
+default_migrations_dir = os.path.join(dbdir, 'migrations')
+default_data_dir = os.path.join(xdg_data_dirs[0], 'chainqueue')
 
 default_config_dir = load_first_config('chainqueue')
 config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
@@ -28,7 +29,9 @@ config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--data-dir', dest='data_dir', type=str, default=default_data_dir, help='data directory')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='reset exsting database')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -38,25 +41,41 @@ if args.vv:
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
+# process config
 config = confini.Config(args.c, args.env_prefix)
 config.process()
+args_override = {
+            'PATH_DATA': getattr(args, 'data_dir'),
+        }
+config.dict_override(args_override, 'cli args')
+
+if config.get('DATABASE_ENGINE') == 'sqlite':
+    config.add(os.path.join(config.get('PATH_DATA'), config.get('DATABASE_NAME')), 'DATABASE_NAME', True)
+ 
 config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
 
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+config.add(os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE')), '_MIGRATIONS_DIR', True)
-if not os.path.isdir(migrations_dir):
+if not os.path.isdir(config.get('_MIGRATIONS_DIR')):
     logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
-    migrations_dir = os.path.join(args.migrations_dir, 'default')
+    config.add(os.path.join(args.migrations_dir, 'default'), '_MIGRATIONS_DIR', True)
+logg.debug('config loaded:\n{}'.format(config))
+
+os.makedirs(config.get('PATH_DATA'), exist_ok=True)
 
-# connect to database
 dsn = dsn_from_config(config)
 
+def main():
+    logg.info('using migrations dir {}'.format(config.get('_MIGRATIONS_DIR')))
+    logg.info('using db {}'.format(dsn))
+    ac = AlembicConfig(os.path.join(config.get('_MIGRATIONS_DIR'), 'alembic.ini'))
+    ac.set_main_option('sqlalchemy.url', dsn)
+    ac.set_main_option('script_location', config.get('_MIGRATIONS_DIR'))
 
-logg.info('using migrations dir {}'.format(migrations_dir))
+    if args.reset:
-logg.info('using db {}'.format(dsn))
+        logg.debug('reset is set, purging existing content')
-ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
+        alembic.command.downgrade(ac, 'base')
-ac.set_main_option('sqlalchemy.url', dsn)
-ac.set_main_option('script_location', migrations_dir)
 
-alembic.command.upgrade(ac, 'head')
+    alembic.command.upgrade(ac, 'head')
+
+if __name__ == '__main__':
+    main()