Add demurrage token amount adjust task

apps/cic-eth/MANIFEST.in

@@ -1,2 +1,2 @@
-include *requirements.txt
+include *requirements.txt config/test/*
 

apps/cic-eth/cic_eth/api/api_task.py

@@ -11,6 +11,7 @@ import celery
 from chainlib.chain import ChainSpec
 
 # local imports
+from cic_eth.api.base import ApiBase
 from cic_eth.enum import LockEnum
 
 app = celery.current_app
@@ -18,48 +19,8 @@ app = celery.current_app
 logg = logging.getLogger(__name__)
 
 
-class Api:
-    """Creates task chains to perform well-known CIC operations.
-
-    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
-
-    :param callback_param: Static value to pass to callback
-    :type callback_param: str
-    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
-    :type callback_task: string
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
-        self.chain_str = chain_str
-        self.chain_spec = ChainSpec.from_chain_str(chain_str)
-        self.callback_param = callback_param
-        self.callback_task = callback_task
-        self.queue = queue
-        logg.debug('api using queue {}'.format(self.queue))
-        self.callback_success = None
-        self.callback_error = None
-        if callback_queue == None:
-            callback_queue=self.queue
-
-        if callback_param != None:
-            self.callback_success = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        0,
-                        ],
-                    queue=callback_queue,
-                    )
-            self.callback_error = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        1,
-                        ],
-                    queue=callback_queue,
-                    )       
-
+class Api(ApiBase):
+    
 
     def default_token(self):
         s_token = celery.signature(

apps/cic-eth/cic_eth/api/base.py

@@ -0,0 +1,52 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+from chainlib.chain import ChainSpec
+
+logg = logging.getLogger(__name__)
+
+class ApiBase:
+    """Creates task chains to perform well-known CIC operations.
+
+    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
+
+    :param callback_param: Static value to pass to callback
+    :type callback_param: str
+    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
+    :type callback_task: string
+    :param queue: Name of worker queue to submit tasks to
+    :type queue: str
+    """
+    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
+        self.chain_str = chain_str
+        self.chain_spec = ChainSpec.from_chain_str(chain_str)
+        self.callback_param = callback_param
+        self.callback_task = callback_task
+        self.queue = queue
+        logg.debug('api using queue {}'.format(self.queue))
+        self.callback_success = None
+        self.callback_error = None
+        if callback_queue == None:
+            callback_queue=self.queue
+
+        if callback_param != None:
+            self.callback_success = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        0,
+                        ],
+                    queue=callback_queue,
+                    )
+            self.callback_error = celery.signature(
+                    callback_task,
+                    [
+                        callback_param,
+                        1,
+                        ],
+                    queue=callback_queue,
+                    )       
+
+

apps/cic-eth/cic_eth/pytest/fixtures_celery.py

@@ -37,7 +37,7 @@ def celery_includes():
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
-        'tests.mock.filter',
+        'cic_eth.pytest.mock.filter',
     ]
 
 

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -13,7 +13,9 @@ logg = logging.getLogger(__name__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, 'config/test')
+    config_dir = os.environ.get('CONFINI_DIR')
+    if config_dir == None:
+        config_dir = os.path.join(root_dir, 'config/test')
     conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))

apps/cic-eth/cic_eth/pytest/fixtures_token.py

@@ -0,0 +1,19 @@
+# external imports
+import pytest
+from eth_erc20 import ERC20
+
+# TODO: missing dep fixture includes
+
+
+@pytest.fixture(scope='function')
+def foo_token_symbol(
+    default_chain_spec,
+    foo_token,
+    eth_rpc,
+    contract_roles,
+    ):
+    
+    c = ERC20(default_chain_spec)
+    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    return c.parse_symbol(r)

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -7,6 +7,8 @@ import tempfile
 import re
 import urllib
 import websocket
+import stat
+import importlib
 
 # external imports
 import celery
@@ -68,6 +70,8 @@ from cic_eth.task import BaseTask
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
 config_dir = os.path.join('/usr/local/etc/cic-eth')
 
 argparser = argparse.ArgumentParser()
@@ -79,6 +83,8 @@ argparser.add_argument('--default-token-symbol', dest='default_token_symbol', ty
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
+argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -109,6 +115,8 @@ if len(health_modules) != 0:
     health_modules = health_modules.split(',')
 logg.debug('health mods {}'.format(health_modules))
 
+
+
 # connect to database
 dsn = dsn_from_config(config)
 SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
@@ -167,6 +175,84 @@ Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 #    raise RuntimeError()
 liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
 
+rpc = RPCConnection.connect(chain_spec, 'default')
+try:
+    registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+except UnknownContractError as e:
+    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+    sys.exit(1)
+logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
+
+trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+if trusted_addresses_src == None:
+    logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+    sys.exit(1)
+trusted_addresses = trusted_addresses_src.split(',')
+for address in trusted_addresses:
+    logg.info('using trusted address {}'.format(address))
+
+connect_declarator(rpc, chain_spec, trusted_addresses)
+connect_token_registry(rpc, chain_spec)
+
+# detect aux 
+# TODO: move to separate file
+#aux_dir = os.path.join(script_dir, '..', '..', 'aux')
+aux = []
+if args.aux_all:
+    if len(args.aux) > 0:
+        logg.warning('--aux-all is set so --aux will have no effect')
+    for p in sys.path:
+        logg.debug('checking for aux modules in {}'.format(p))
+        aux_dir = os.path.join(p, 'cic_eth_aux')
+        try:
+            d = os.listdir(aux_dir)
+        except FileNotFoundError:
+            logg.debug('no aux module found in {}'.format(aux_dir))
+            continue
+        for v in d:
+            if v[:1] == '.':
+                logg.debug('dotfile, skip {}'.format(v))
+                continue
+            aux_mod_path = os.path.join(aux_dir, v)
+            st = os.stat(aux_mod_path)
+            if not stat.S_ISDIR(st.st_mode):
+                logg.debug('not a dir, skip {}'.format(v))
+                continue
+            aux_mod_file = os.path.join(aux_dir, v,'__init__.py')
+            try:
+                st = os.stat(aux_mod_file)
+            except FileNotFoundError:
+                logg.debug('__init__.py not found, skip {}'.format(v))
+                continue 
+            aux.append(v)
+            logg.debug('found module {} in {}'.format(v, aux_dir))
+
+elif len(args.aux) > 0:
+    for p in sys.path:
+        v_found = None
+        for v in args.aux:
+            aux_dir = os.path.join(p, 'cic_eth_aux')
+            aux_mod_file = os.path.join(aux_dir, v, '__init__.py')
+            try:
+                st = os.stat(aux_mod_file)
+                v_found = v
+            except FileNotFoundError:
+                logg.debug('cannot find explicity requested aux module {} in path {}'.format(v, aux_dir))
+                continue
+        if v_found == None:
+            logg.critical('excplicity requested aux module {} not found in any path'.format(v))
+            sys.exit(1)
+
+        logg.info('aux module {} found in path {}'.format(v, aux_dir))
+        aux.append(v)
+
+for v in aux:
+    mname = 'cic_eth_aux.' + v
+    mod = importlib.import_module(mname)
+    mod.aux_setup(rpc, config)
+    logg.info('loaded aux module {}'.format(mname))
+
+
 def main():
     argv = ['worker']
     if args.vv:
@@ -189,24 +275,6 @@ def main():
 
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    try:
-        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-    except UnknownContractError as e:
-        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-        sys.exit(1)
-    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
-
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-
-    connect_declarator(rpc, chain_spec, trusted_addresses)
-    connect_token_registry(rpc, chain_spec)
-
     BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
     BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
     default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)

apps/cic-eth/cic_eth/task.py

@@ -1,6 +1,5 @@
 # import
 import time
-import requests
 import logging
 import uuid
 
@@ -76,7 +75,7 @@ class CriticalSQLAlchemyTask(CriticalTask):
 
 class CriticalWeb3Task(CriticalTask):
     autoretry_for = (
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
@@ -86,7 +85,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
     autoretry_for = (
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
@@ -102,7 +101,7 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
 
 class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
-        requests.exceptions.ConnectionError,
+        ConnectionError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         0,
-        'alpha.1',
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.8.6-slim-buster as compile
 
-WORKDIR /usr/src/cic-eth
+WORKDIR /usr/src
 
 RUN apt-get update && \
 	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
@@ -13,10 +13,17 @@ ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages
 RUN /usr/local/bin/python -m pip install --upgrade pip
 RUN pip install semver
 
-COPY cic-eth/ .
+COPY cic-eth-aux/  ./cic-eth-aux/
+WORKDIR /usr/src/cic-eth-aux/erc20-demurrage-token
 RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
     	--extra-index-url $EXTRA_INDEX_URL .
 
+WORKDIR /usr/src/cic-eth
+
+COPY cic-eth/ .
+RUN	pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    	--extra-index-url $EXTRA_INDEX_URL .[services]
+
 # --- TEST IMAGE ---
 FROM python:3.8.6-slim-buster as test 
 

apps/cic-eth/docker/start_tasker.sh

@@ -8,7 +8,7 @@ set -e
 echo "!!! starting signer"
 python /usr/local/bin/crypto-dev-daemon -c /usr/local/etc/crypto-dev-signer -vv 2> /tmp/signer.log &
 
-echo "!!! starting tracker"
+echo "!!! starting taskerd"
 /usr/local/bin/cic-eth-taskerd $@
 
 # thanks! https://docs.docker.com/config/containers/multi-service_container/

apps/cic-eth/services_requirements.txt

@@ -5,3 +5,10 @@ confini~=0.3.6rc4
 redis==3.5.3
 hexathon~=0.0.1a7
 pycryptodome==3.10.1
+liveness~=0.0.1a7
+eth-address-index~=0.1.2a1
+eth-accounts-index~=0.0.12a1
+cic-eth-registry~=0.5.6a1
+erc20-faucet~=0.2.2a1
+sarafu-faucet~=0.0.4a1
+moolb~=0.1.1b2

apps/cic-eth/setup.cfg

@@ -39,6 +39,9 @@ packages =
 	cic_eth.callbacks
 	cic_eth.sync
 	cic_eth.check
+	# should be concealed behind extras "test" if possible (but its not unfortunately)
+	cic_eth.pytest
+	cic_eth.pytest.mock
 scripts =
 	./scripts/migrate.py
 

apps/cic-eth/setup.py

@@ -48,4 +48,4 @@ setup(
         'admin_api': admin_requirements,
         'services': services_requirements,
         }
-        )
+    )

apps/cic-eth/tests/conftest.py

@@ -5,6 +5,7 @@ import logging
 import uuid
 
 # external imports
+import pytest
 from eth_erc20 import ERC20
 import redis
 
@@ -22,6 +23,7 @@ from cic_eth.pytest.fixtures_celery import *
 from cic_eth.pytest.fixtures_database import *
 from cic_eth.pytest.fixtures_role import *
 from cic_eth.pytest.fixtures_contract import *
+from cic_eth.pytest.fixtures_token import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *
@@ -37,20 +39,6 @@ def api(
     return Api(chain_str, queue=None, callback_param='foo')
 
 
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
 @pytest.fixture(scope='function')
 def default_token(
         foo_token,

apps/cic-eth/tests/task/api/test_list.py

@@ -15,7 +15,7 @@ from cic_eth.db.models.nonce import (
         )
 
 # test imports
-from tests.mock.filter import (
+from cic_eth.pytest.mock.filter import (
         block_filter,
         tx_filter,
         )
@@ -110,7 +110,7 @@ def test_list_tx(
     logg.debug('r {}'.format(r))
 
     # test the api
-    t = api.list(agent_roles['ALICE'], external_task='tests.mock.filter.filter')
+    t = api.list(agent_roles['ALICE'], external_task='cic_eth.pytest.mock.filter.filter')
     r = t.get_leaf()
     assert t.successful()