add build and publish for base

Add allowance check and transferFrom task

See merge request grassrootseconomics/cic-internal-integration!203

.gitignore

@@ -12,4 +12,6 @@ build/
 **/.nyc_output
 **/coverage
 **/.venv
+**/venv
+**/dist
 .idea

.gitlab-ci.yml

@@ -11,4 +11,4 @@ include:
 stages:
   - build
   - test
-  - release
+  - publish

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-base/.gitignore

@@ -0,0 +1,7 @@
+gmon.out
+__pycache__
+*.pyc
+inc.sh
+*.egg-info
+build/
+dist/

apps/cic-base/.gitlab-ci.yml

@@ -0,0 +1,20 @@
+.cic_base_variables:
+    variables:
+        APP_NAME: cic-base
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_base_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
+
+build-mr-cic-base:
+    extends:
+        - .cic_base_changes_target
+        - .py_build_merge_request
+        - .cic_base_variables
+
+publish_python:
+  extends:
+    - .publish_python
+    - .cic_base_variables

apps/cic-base/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt

apps/cic-base/cic_base/__init__.py

@@ -0,0 +1,8 @@
+#from . import (
+#        config,
+#        argparse,
+#        rpc,
+#        signer,
+#        log,
+#        version,
+#        )

apps/cic-base/cic_base/argparse.py

@@ -0,0 +1,87 @@
+# standard imports
+import logging
+import argparse
+import os
+import sys
+
+# external imports
+from xdg.BaseDirectory import (
+        xdg_config_dirs,
+        load_first_config,
+        )
+
+logg = logging.getLogger(__file__)
+
+fallback_config_path = '/usr/local/etc'
+xdg_config_dirs += [fallback_config_path]
+default_config_dir = load_first_config('cic')
+if default_config_dir == None:
+    default_config_dir = os.path.join('.', '.cic')
+env_config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
+
+full_template = { 
+        # (long arg and key name, short var, type, default, help,)
+        'provider': ('p', str, None, 'RPC provider url',),
+        'registry_address': ('r', str, None, 'CIC registry address',),
+        'keystore_file': ('y', str, None, 'Keystore file',),
+        'config_dir': ('c', str, env_config_dir, 'Configuration directory',),
+        'queue': ('q', str, 'cic-eth', 'Celery task queue',),
+        'chain_spec': ('i', str, None, 'Chain spec string',),
+        'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
+        }
+
+default_include_args = [
+    'config_dir',
+    'provider',
+    'env_prefix',
+    ]
+
+sub = None
+
+def create(caller_dir, include_args=default_include_args):
+
+    argparser = argparse.ArgumentParser()
+
+    for k in include_args:
+        a = full_template[k]
+        long_flag = '--' + k.replace('_', '-')
+        short_flag = None
+        dest = None
+        if a[0] != None:
+            short_flag = '-' + a[0]
+            dest = a[0]
+        else:
+            dest = k
+        default = a[2]
+        if default == None and k == 'config_dir':
+            default = os.path.join(os.getcwd(), 'config')
+        
+        if short_flag == None:
+            argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
+        else:
+            argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
+
+    argparser.add_argument('-v', action='store_true', help='Be verbose')
+    argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+
+    return argparser
+
+
+def add(argparser, processor, name, description=None):
+    processor(argparser)
+
+    return argparser
+
+
+def parse(argparser, logger=None):
+
+    args = argparser.parse_args(sys.argv[1:])
+
+    # handle logging input
+    if logger != None:
+        if args.vv:
+            logger.setLevel(logging.DEBUG)
+        elif args.v:
+            logger.setLevel(logging.INFO)
+
+    return args

apps/cic-base/cic_base/config.py

@@ -0,0 +1,52 @@
+# standard imports
+import logging
+
+# external imports
+import confini
+
+# local imports
+from .error import ConfigError
+
+logg = logging.getLogger(__name__)
+
+
+default_arg_overrides = {
+    'p': 'ETH_PROVIDER',
+    'i': 'CIC_CHAIN_SPEC',
+    'r': 'CIC_REGISTRY_ADDRESS',
+    }
+
+
+def override(config, override_dict, label):
+    config.dict_override(override_dict, label)
+    config.validate()
+    return config
+
+
+def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
+    # handle config input
+    config = None
+    try:
+        config = confini.Config(config_dir, env_prefix)
+    except OSError:
+        pass
+
+    if config == None:
+        raise ConfigError('directory {} not found'.format(config_dir))
+
+    config.process()
+    if arg_overrides != None and args != None:
+        override_dict = {}
+        for k in arg_overrides:
+            v = getattr(args, k)
+            if v != None:
+                override_dict[arg_overrides[k]] = v
+        config = override(config, override_dict, 'args')
+    else:
+        config.validate()
+
+    return config
+
+
+def log(config):
+    logg.debug('config loaded:\n{}'.format(config))

apps/cic-base/cic_base/error.py

@@ -0,0 +1,2 @@
+class ConfigError(Exception):
+    pass

apps/cic-base/cic_base/log.py

@@ -0,0 +1,18 @@
+# standard imports
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+
+default_mutelist = [
+        'urllib3',
+        'websockets.protocol',
+        'web3.RequestManager',
+        'web3.providers.WebsocketProvider',
+        'web3.providers.HTTPProvider',
+        ]
+
+def create(name=None, mutelist=default_mutelist):
+    logg = logging.getLogger(name)
+    for m in mutelist:
+        logging.getLogger(m).setLevel(logging.CRITICAL)
+    return logg

apps/cic-base/cic_base/rpc.py

@@ -0,0 +1,13 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.eth.sign import (
+        sign_transaction,
+        sign_message,
+        )
+
+
+def setup(chain_spec, evm_provider, signer_provider=None):
+    RPCConnection.register_location(evm_provider, chain_spec, 'default')
+    if signer_provider != None:
+        RPCConnection.register_location(signer_provider, chain_spec, 'signer', constructor=EthUnixSignerConnection)

apps/cic-base/cic_base/signer.py

@@ -0,0 +1,27 @@
+# standard imports
+import logging
+import os
+
+# external imports 
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore.dict import DictKeystore
+
+logg = logging.getLogger(__name__)
+
+keystore = DictKeystore()
+
+default_passphrase = os.environ.get('ETH_PASSPHRASE', '')
+
+
+def from_keystore(keyfile, passphrase=default_passphrase):
+    global keystore
+
+    # signer
+    if keyfile == None:
+        raise ValueError('please specify signer keystore file')
+
+    logg.debug('loading keystore file {}'.format(keyfile))
+    address = keystore.import_keystore_file(keyfile, password=passphrase)
+
+    signer = EIP155Signer(keystore)
+    return (address, signer,)

apps/cic-base/cic_base/sqlalchemy.py

@@ -0,0 +1,122 @@
+# stanard imports
+import logging
+
+# third-party imports
+from sqlalchemy import Column, Integer
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import (
+        StaticPool,
+        QueuePool,
+        AssertionPool,
+        )
+
+logg = logging.getLogger()
+
+Model = declarative_base(name='Model')
+
+
+class SessionBase(Model):
+    """The base object for all SQLAlchemy enabled models. All other models must extend this.
+    """
+    __abstract__ = True
+ 
+    id = Column(Integer, primary_key=True)
+
+    engine = None
+    """Database connection engine of the running aplication"""
+    sessionmaker = None
+    """Factory object responsible for creating sessions from the  connection pool"""
+    transactional = True
+    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
+    poolable = True
+    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
+    procedural = True
+    """Whether the database backend supports stored procedures"""
+    localsessions = {}
+    """Contains dictionary of sessions initiated by db model components"""
+
+
+    @staticmethod
+    def create_session():
+        """Creates a new database session.
+        """
+        return SessionBase.sessionmaker()
+
+
+    @staticmethod
+    def _set_engine(engine):
+        """Sets the database engine static property
+        """
+        SessionBase.engine = engine
+        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
+
+
+    @staticmethod
+    def connect(dsn, pool_size=8, debug=False):
+        """Create new database connection engine and connect to database backend.
+
+        :param dsn: DSN string defining connection.
+        :type dsn: str
+        """
+        e = None
+        if SessionBase.poolable:
+            poolclass = QueuePool
+            if pool_size > 1:
+                e = create_engine(
+                        dsn,
+                        max_overflow=pool_size*3,
+                        pool_pre_ping=True,
+                        pool_size=pool_size,
+                        pool_recycle=60,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+            else:
+                if debug:
+                    poolclass = AssertionPool
+                else:
+                    poolclass = StaticPool
+
+                e = create_engine(
+                        dsn,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+        else:
+            e = create_engine(
+                    dsn,
+                    echo=debug,
+                    )
+
+        SessionBase._set_engine(e)
+
+
+    @staticmethod
+    def disconnect():
+        """Disconnect from database and free resources.
+        """
+        SessionBase.engine.dispose()
+        SessionBase.engine = None
+
+
+    @staticmethod
+    def bind_session(session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+            localsession_key = str(id(localsession))
+            logg.debug('creating new session {}'.format(localsession_key))
+            SessionBase.localsessions[localsession_key] = localsession
+        return localsession
+
+
+    @staticmethod
+    def release_session(session=None):
+        session.flush()
+        session_key = str(id(session))
+        if SessionBase.localsessions.get(session_key) != None:
+            logg.debug('destroying session {}'.format(session_key))
+            session.commit()
+            session.close()

apps/cic-base/cic_base/version.py

@@ -0,0 +1,43 @@
+# standard imports
+import os
+import time
+import logging
+
+# third-party imports
+import semver
+
+version = (
+        0,
+        1,
+        2,
+        'beta.22',
+        )
+
+version_object = semver.VersionInfo(
+        major=version[0],
+        minor=version[1],
+        patch=version[2],
+        prerelease=version[3],
+        )
+
+
+def git_hash():
+    import subprocess
+    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+version_string = str(version_object)
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+            time_string_pair[0],
+            int(time_string_pair[1]),
+            )
+
+__version_string__ = version_string

apps/cic-base/docker/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.8.6-slim-buster
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client cargo
+
+WORKDIR /usr/src/cic-base
+COPY . /usr/src/cic-base/
+
+#RUN mkdir python 
+#WORKDIR ./python
+#COPY ./pep503.sh .
+#RUN pip download --no-cache-dir --extra-index-url https://pip.grassrootseconomics.net:8433 cic-base[full_graph]==0.1.1a6
+RUN pip install -r requirements.txt
+RUN python setup.py bdist_wheel
+RUN pip download --extra-index-url https://pip.grassrootseconomics.net:8433 dist/$(basename $(ls dist/*))
+RUN mkdir packages && \
+	cd packages && \
+	bash ../docker/pep503.sh ..
+
+WORKDIR /usr/src/cic-base/packages
+
+RUN ls
+RUN ls ..
+
+ENTRYPOINT ["python", "-m", "http.server", "8080"]

apps/cic-base/docker/pep503.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env 
+
+dest=`pwd`
+d=$1
+for df in `find $d -name "*.whl" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done
+for df in `find $d -name "*.tar.gz" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done
+for df in `find $d -name "*.zip" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done

apps/cic-base/examples/cli.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+import cic_base.argparse
+import cic_base.config
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+def more_argparse(argparser):
+    argparser.add_argument('--foo', type=str, help='foo')
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+
+argparser = cic_base.argparse.create(script_dir, include_args=cic_base.argparse.full_template)
+args = cic_base.argparse.parse(argparser, logger=logg)
+config = cic_base.config.create(args.c, args, env_prefix=args.env_prefix)
+
+cic_base.config.log(config)

apps/cic-base/examples/config/foo.ini

@@ -0,0 +1,3 @@
+[foo]
+bar = 42
+baz = xyzzy

apps/cic-base/full_graph_requirements.txt

@@ -0,0 +1,98 @@
+africastalking==1.2.3
+alembic==1.4.2
+amqp==2.6.1
+attrs==20.3.0
+base58==2.1.0
+bcrypt==3.2.0
+billiard==3.6.3.0
+bip-utils==1.4.0
+bitarray==1.2.2
+blake2b-py==0.1.4
+cached-property==1.5.2
+celery==4.4.7
+certifi==2020.12.5
+cffi==1.14.3
+chainlib==0.0.1a18
+chainsyncer==0.0.1a18
+chardet==3.0.4
+confini==0.3.6rc3
+contextlib2==0.6.0.post1
+coverage==5.4
+cryptography==3.2.1
+cytoolz==0.11.0
+ecdsa==0.16.1
+ecuth==0.4.5a1
+Faker==4.17.1
+hexathon==0.0.1a3
+hexbytes==0.2.1
+http-hoba-auth==0.2.0
+idna==2.10
+iniconfig==1.1.1
+ipfshttpclient==0.6.1
+json-rpc==1.13.0
+jsonschema==3.2.0
+kombu==4.6.11
+lru-dict==1.1.7
+Mako==1.1.3
+MarkupSafe==1.1.1
+mirakuru==2.3.0
+moolb==0.1.1b2
+more-itertools==8.7.0
+multiaddr==0.0.9
+mypy-extensions==0.4.3
+netaddr==0.8.0
+packaging==20.9
+parsimonious==0.8.1
+phonenumbers==8.12.12
+pluggy==0.13.1
+port-for==0.4
+protobuf==3.15.1
+psutil==5.8.0
+psycopg2==2.8.6
+py==1.9.0
+py-ecc==4.1.0
+py-eth==0.1.1
+pycparser==2.20
+pycryptodome==3.10.1
+pyethash==0.1.27
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pysha3==1.0.2
+pytest==6.0.1
+pytest-alembic==0.2.5
+pytest-celery==0.0.0a1
+pytest-cov==2.10.1
+pytest-mock==3.3.1
+pytest-redis==2.0.0
+python-dateutil==2.8.1
+python-editor==1.0.4
+python-gnupg==0.4.6
+python-i18n==0.3.9
+pytz==2021.1
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+rlp==2.0.1
+schema==0.7.4
+semantic-version==2.8.5
+semver==2.13.0
+six==1.15.0
+sortedcontainers==2.3.0
+SQLAlchemy==1.3.20
+sqlparse==0.4.1
+text-unidecode==1.3
+tinydb==4.2.0
+toml==0.10.2
+toolz==0.11.1
+transitions==0.8.4
+trie==2.0.0a5
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+uWSGI==2.0.19.1
+varint==1.0.2
+vine==1.3.0
+vobject==0.9.6.1
+web3==5.12.2
+websocket-client==0.57.0
+websockets==8.1
+yaml-acl==0.0.1

apps/cic-base/full_requirements.txt

@@ -0,0 +1,44 @@
+africastalking==1.2.3
+alembic==1.4.2
+bcrypt==3.2.0
+celery==4.4.7
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b4
+cryptography==3.2.1
+ecuth==0.4.5a5
+eth-accounts-index==0.0.11a14
+eth-address-index==0.1.1a12
+eth-contract-registry==0.5.5a3
+erc20-transfer-authorization==0.3.1a7
+erc20-faucet==0.2.1a5
+faker==4.17.1
+http-hoba-auth==0.2.1a2
+moolb==0.1.1b2
+phonenumbers==8.12.12
+psycopg2==2.8.6
+py-eth~=0.1.1
+pytest==6.0.1
+pytest-alembic==0.2.5
+pytest-celery==0.0.0a1
+pytest-cov==2.10.1
+pytest-mock==3.3.1
+pytest-redis==2.0.0
+python-i18n==0.3.9
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+semver==2.13.0
+SQLAlchemy==1.3.20
+sqlparse==0.4.1
+tinydb==4.2.0
+transitions==0.8.4
+uWSGI==2.0.19.1
+vobject==0.9.6.1
+web3==5.12.2
+websockets==8.1
+yaml-acl==0.0.1
+rlp==2.0.1
+cryptocurrency-cli-tools==0.0.5
+websocket-client==0.57.0
+hexathon==0.0.1a7
+chainsyncer~=0.0.2a5

apps/cic-base/mac_requirements.txt

@@ -0,0 +1,40 @@
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b1
+semver==2.13.0
+SQLAlchemy==1.3.20
+pyxdg==0.27
+chainlib==0.0.2a10
+alembic==1.4.2
+celery==4.4.7
+cryptography==3.2.1
+ecuth==0.4.5a1
+eth-accounts-index==0.0.11a8
+eth-address-index==0.1.1a8
+eth-contract-registry==0.5.4a9
+erc20-transfer-authorization==0.3.1a4
+erc20-single-shot-faucet==0.2.0a11
+faker==4.17.1
+http-hoba-auth==0.2.0
+moolb==0.1.1b2
+phonenumbers==8.12.12
+psycopg2==2.8.6
+python-i18n==0.3.9
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+sqlparse==0.4.1
+transitions==0.8.4
+uWSGI==2.0.19.1
+vobject==0.9.6.1
+web3==5.12.2
+websockets==8.1
+yaml-acl==0.0.1
+rlp==2.0.1
+cryptocurrency-cli-tools==0.0.4
+giftable-erc20-token==0.0.8a8
+websocket-client==0.57.0
+hexathon==0.0.1a7
+chainsyncer==0.0.2b1
+sarafu-faucet==0.0.2a20
+cic-types==0.1.0a10
+cic-eth-registry==0.5.4a13

apps/cic-base/requirements.txt

@@ -0,0 +1,9 @@
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b4
+semver==2.13.0
+SQLAlchemy==1.3.20
+pyxdg==0.27
+chainlib==0.0.3rc3
+eth-erc20==0.0.9a4
+liveness==0.0.1a7
+requirements-magic~=0.0.1a2

apps/cic-base/setup.cfg

@@ -0,0 +1,35 @@
+[metadata]
+name = cic-base
+version = attr: cic_base.version.__version_string__
+description = CIC python base
+author = Louis Holbrook
+author_email = dev@holbrook.no
+url = https://gitlab.com/grassrootseconomics/cic-eth
+keywords =
+	cic
+	cryptocurrency
+	ethereum
+classifiers =
+	Programming Language :: Python :: 3
+	Operating System :: OS Independent
+	Development Status :: 3 - Alpha
+	Environment :: No Input/Output (Daemon)
+	Intended Audience :: Developers
+	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+	Topic :: Internet
+#	Topic :: Blockchain :: EVM
+license = GPL3
+licence_files =
+	LICENSE.txt
+
+[options]
+python_requires = >= 3.6
+packages = 
+	cic_base
+
+
+[options.entry_points]
+console_scripts =
+	cic-base-audit = cic_base.runnable.audit:main
+	cic-base-merge = cic_base.runnable.merge:main
+	cic-base-update = cic_base.runnable.update:main

apps/cic-base/setup.py

@@ -0,0 +1,50 @@
+from setuptools import setup
+import configparser
+import os
+import logging
+import re
+
+logg = logging.getLogger(__name__)
+
+
+re_v = r'[~><=]='
+def merge(requirements_files, base_dir='.'):
+
+    requirements = {}
+    for r in requirements_files:
+        filepath = os.path.join(base_dir, r)
+        logg.debug('reading {}'.format(filepath))
+        f = open(filepath, 'r')
+        while True:
+            l = f.readline()
+            if l == '':
+                break
+            l = l.rstrip()
+            m = re.split(re_v, l)
+            k = m[0]
+            if k == None:
+                raise ValueError('invalid requirement line {}'.format(l))
+            if requirements.get(k) == None:
+                logg.info('adding {} -> {}'.format(k, l))
+                requirements[k] = l
+            else:
+                logg.debug('skipping {}'.format(l))
+        f.close()
+
+    return list(requirements.values())
+
+
+requirements = []
+f = open('requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    requirements.append(l.rstrip())
+f.close()
+
+
+
+setup(
+    install_requires=requirements,
+        )

apps/cic-base/tests/base.py

@@ -0,0 +1,22 @@
+# standard imports
+import logging
+import unittest
+
+# external imports
+from chainlib.chain import ChainSpec
+
+# local imports
+from cic_base.rpc import setup as rpc_setup
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestBase(unittest.TestCase):
+
+    def setUp(self):
+        self.chain_spec = ChainSpec('evm', 'foo', 42)
+        rpc_setup(self.chain_spec, 'http://localhost:8545', signer_provider='ipc://tmp/foo')
+
+    def tearDown(self):
+        pass

apps/cic-base/tests/test_basic.py

@@ -0,0 +1,14 @@
+# standard imports
+import unittest
+
+# local imports
+from tests.base import TestBase
+
+
+class TestBasic(TestBase):
+    
+    def test_basic(self):
+        pass
+
+if __name__ == '__main__':
+    unittest.main()

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,6 +16,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -28,10 +29,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -113,10 +112,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,12 +1,14 @@
-cic-base~=0.1.2b10
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+rlp==2.0.1
+chainsyncer[sql]~=0.0.3a3
+erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,6 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-web3==5.12.2
-cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8
+cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.4a1

apps/cic-eth/MANIFEST.in

@@ -0,0 +1,2 @@
+include *requirements.txt
+

apps/cic-eth/cic_eth/api/api_admin.py

@@ -562,13 +562,13 @@ class AdminApi:
             tx['source_token_symbol'] = source_token.symbol
             o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['sender_token_balance'] = erc20_c.parse_balance(r)
 
         if destination_token != None:
             tx['destination_token_symbol'] = destination_token.symbol
             o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
             #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which

apps/cic-eth/cic_eth/api/api_task.py

@@ -204,6 +204,82 @@ class Api:
 #        return t
 
 
+    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
+        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param to_address: Ethereum address of recipient
+        :type to_address: str, 0x-hex
+        :param value: Estimated return from conversion
+        :type  value: int
+        :param token_symbol: ERC20 token symbol of token to send
+        :type token_symbol: str
+        :param spender_address: Ethereum address of recipient
+        :type spender_address: str, 0x-hex
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_allow = celery.signature(
+                'cic_eth.eth.erc20.check_allowance',
+                [
+                    from_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_transfer = celery.signature(
+                'cic_eth.eth.erc20.transfer_from',
+                [
+                    from_address,
+                    to_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens.link(s_allow)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_transfer.link(self.callback_success)
+            s_allow.link(s_transfer).on_error(self.callback_error)
+        else:
+            s_allow.link(s_transfer)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
+
+
+
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/error.py

@@ -80,3 +80,8 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
+
+
+class YouAreBrokeError(Exception):
+    """Exception raised when a value transfer is attempted without access to sufficient funds
+    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,6 +24,7 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
+        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -71,6 +72,117 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
+@celery_app.task(bind=True)
+def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
+    """Best-effort verification that the allowance for a transfer from spend is sufficient.
+
+    :raises YouAreBrokeError: If allowance is insufficient
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :return: Token list as passed to task
+    :rtype: dict
+    """
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    caller_address = ERC20Token.caller_address 
+    c = ERC20(chain_spec)
+    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
+    r = rpc.do(o)
+    allowance = c.parse_allowance(r)
+    if allowance < value:
+        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
+        logg.error(errstr)
+        raise YouAreBrokeError(errstr)
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
+    """Transfer ERC20 tokens between addresses
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    try:
+        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
+    except FileNotFoundError as e:
+        raise SignerError(e)
+    except ConnectionError as e:
+        raise SignerError(e)
+    
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -232,6 +344,7 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
+            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -279,6 +392,48 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_transfer_from_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_transfer_from
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+
+    tx_data = ERC20.parse_transfer_from_request(tx['data'])
+    spender_address = tx_data[0]
+    recipient_address = tx_data[1]
+    token_value = tx_data[2]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-logg = logging.getLogger(__file__)
+root_dir = os.path.dirname(os.path.dirname(script_dir))
+logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -37,7 +37,8 @@ def init_database(
         database_engine,
         ):
 
-    rootdir = os.path.dirname(os.path.dirname(__file__))
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    rootdir = os.path.dirname(os.path.dirname(script_dir))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,136 +0,0 @@
-# standard imports
-import os
-import re
-import logging
-import argparse
-import json
-
-# third-party imports
-import web3
-import confini
-import celery
-from json.decoder import JSONDecodeError
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import dsn_from_config
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_eth', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-queue = args.q
-
-re_something = r'^/something/?'
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-def process_something(session, env):
-    r = re.match(re_something, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    #if env.get('CONTENT_TYPE') != 'application/json':
-    #    raise AttributeError('content type')
-
-    #if env.get('REQUEST_METHOD') != 'POST':
-    #    raise AttributeError('method')
-
-    #post_data = json.load(env.get('wsgi.input'))
-    
-    #return ('text/plain', 'foo'.encode('utf-8'),)
-
-
-# uwsgi application
-def application(env, start_response):
-    
-    for k in env.keys():
-        logg.debug('env {} {}'.format(k, env[k]))
-
-    headers = []
-    content = b''
-    err = None
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_something,
-            ]:
-        try:
-            r = handler(session, env)
-        except AttributeError as e:
-            logg.error('handler fail attribute {}'.format(e))
-            err = '400 Impertinent request'
-            break
-        except JSONDecodeError as e:
-            logg.error('handler fail json {}'.format(e))
-            err = '400 Invalid data format'
-            break
-        except KeyError as e:
-            logg.error('handler fail key {}'.format(e))
-            err = '400 Invalid JSON'
-            break
-        except ValueError as e:
-            logg.error('handler fail value {}'.format(e))
-            err = '400 Invalid data'
-            break
-        except RuntimeError as e:
-            logg.error('task fail value {}'.format(e))
-            err = '500 Task failed, sorry I cannot tell you more'
-            break
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    if err != None:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response(err, headers)
-        session.close()
-        return [content]
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,10 +27,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         11,
-        0,
-        'beta.16',
+        1,
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base~=0.1.2b15
-celery==4.4.7
-crypto-dev-signer~=0.4.14b3
-confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a7
-redis==3.5.3
 alembic==1.4.2
-websockets==8.1
-requests~=2.24.0
-eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a7
-uWSGI==2.0.19.1
-semver==2.13.0
-websocket-client==0.57.0
-moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3rc2
-hexathon~=0.0.1a7
-chainsyncer[sql]==0.0.2a5
-chainqueue~=0.0.2b3
-sarafu-faucet~=0.0.3a3
-erc20-faucet~=0.2.1a5
+celery==4.4.7
+chainlib-eth~=0.0.5a1
+chainqueue~=0.0.2b5
+chainsyncer[sql]~=0.0.3a3
+cic-base==0.1.3a3+build.984b5cff
+cic-eth-registry~=0.5.6a1
 coincurve==15.0.0
+confini~=0.3.6rc3
+crypto-dev-signer~=0.4.14b6
+erc20-faucet~=0.2.2a1
+erc20-transfer-authorization~=0.3.2a1
+eth-address-index~=0.1.2a1
+eth_accounts_index~=0.0.12a1
+hexathon~=0.0.1a7
+moolb~=0.1.1b2
 potaahto~=0.0.1a2
 pycryptodome==3.10.1
+redis==3.5.3
+requests~=2.24.0
+sarafu-faucet~=0.0.4a1
+semver==2.13.0
+uWSGI==2.0.19.1
+websocket-client==0.57.0
+websockets==8.1

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/tests/conftest.py

@@ -17,11 +17,11 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from tests.fixtures_config import *
-from tests.fixtures_database import *
-from tests.fixtures_celery import *
-from tests.fixtures_role import *
-from tests.fixtures_contract import *
+from cic_eth.pytest.fixtures_config import *
+from cic_eth.pytest.fixtures_celery import *
+from cic_eth.pytest.fixtures_database import *
+from cic_eth.pytest.fixtures_role import *
+from cic_eth.pytest.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from tests.fixtures_celery import *
+from cic_eth.pytest.fixtures_celery import *

apps/cic-eth/tests/task/test_task_erc20.py

@@ -13,6 +13,7 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
+from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -167,3 +168,101 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
+
+
+def test_erc20_transfer_from_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    s_nonce = celery.signature(
+        'cic_eth.eth.nonce.reserve_nonce',
+        [
+            [token_object],
+            default_chain_spec.asdict(),
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            ],
+        queue=None,
+            )
+    s_transfer = celery.signature(
+            'cic_eth.eth.erc20.transfer_from',
+            [
+                custodial_roles['FOO_TOKEN_GIFTER'],
+                agent_roles['BOB'],
+                transfer_value,
+                default_chain_spec.asdict(),
+                agent_roles['ALICE'],
+                ],
+            queue=None,
+            )
+    s_nonce.link(s_transfer)
+    t = s_nonce.apply_async()
+    r = t.get_leaf()
+
+    logg.debug('result {}'.format(r))
+
+
+def test_erc20_allowance_check_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+        'symbol': 'FOO',
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    s_check = celery.signature(
+        'cic_eth.eth.erc20.check_allowance',
+        [
+            [token_object],
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            transfer_value,
+            default_chain_spec.asdict(),
+            agent_roles['ALICE']
+            ],
+        queue=None,
+            )
+    t = s_check.apply_async()
+    with pytest.raises(YouAreBrokeError):
+        t.get() 
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = s_check.apply_async()
+    t.get() 
+    assert t.successful()

apps/cic-meta/scripts/server/handlers.ts

@@ -147,7 +147,7 @@ function handleClientMergeGet(db, digest, keystore) {
 					doh(e);
 				});
 			}).catch((e) => {
-				console.error('message', e);
+				console.error('mesage', e);
 				doh(e);
 			});
 		});

apps/cic-meta/scripts/server/server.ts

@@ -87,7 +87,7 @@ async function startServer() {
 	http.createServer(processRequest).listen(config.get('SERVER_PORT'));
 }
 
-const re_digest = /^([a-fA-F0-9]{64})\/?$/;
+const re_digest = /^\/([a-fA-F0-9]{64})\/?$/;
 function parseDigest(url) {
 	const digest_test = url.match(re_digest);
 	if (digest_test === null) {
@@ -96,42 +96,6 @@ function parseDigest(url) {
 	return digest_test[1].toLowerCase();
 }
 
-function getIds(url: string): Array<string> {
-	const params: Array<string> = url.split('?')[1].split('&');
-	let ids: Array<string> = [];
-	for (let param of params) {
-		const splitParam: Array<string> = param.split('=');
-		if (splitParam[0] === 'id') {
-			ids.push(parseDigest(splitParam[1]));
-		}
-	}
-	return ids;
-}
-
-function generateResponseBody(digest: string, data: string | boolean): string {
-	let response = {
-		id: digest,
-		status: 0,
-		headers: {},
-		body: ''
-	}
-	if (typeof data === 'boolean' || data === undefined) {
-		response.body = `Metadata for identifier ${digest} not found!`;
-		response.status = 404;
-		response.headers = {"Content-Type": "text/plain"}
-	} else {
-		const responseContentLength = (new TextEncoder().encode(data)).length;
-		response.body = data;
-		response.status = 200;
-		response.headers = {
-			"Access-Control-Allow-Origin": "*",
-			"Content-Type": 'application/json',
-			"Content-Length": responseContentLength,
-		}
-	}
-	return JSON.stringify(response);
-}
-
 async function processRequest(req, res) {
 	let digest = undefined;
 	const headers = {
@@ -155,16 +119,7 @@ async function processRequest(req, res) {
 	}
 
 	try {
-		if (req.url.includes('id')) {
-			if (req.method !== 'GET') {
-				res.writeHead(405, {"Content-Type": "text/plain"});
-				res.end();
-				return;
-			}
-			digest = getIds(req.url);
-		} else {
-			digest = parseDigest(req.url.substring(1));
-		}
+		digest = parseDigest(req.url);
 	} catch(e) {
 		console.error('digest error: ' + e)
 		res.writeHead(400, {"Content-Type": "text/plain"});
@@ -207,24 +162,7 @@ async function processRequest(req, res) {
 					break;
 
 				case 'get:automerge:client':
-					if (digest instanceof Array) {
-						let response = [];
-						for (let dg of digest) {
-							const metadata = await handlers.handleClientMergeGet(db, dg, keystore);
-							response.push(generateResponseBody(dg, metadata));
-						}
-						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
-						res.writeHead(207, {
-							"Access-Control-Allow-Origin": "*",
-							"Content-Type": contentType,
-							"Content-Length": responseContentLength,
-						});
-						res.write(response.toString());
-						res.end();
-						return;
-					} else {
-						content = await handlers.handleClientMergeGet(db, digest, keystore);
-					}
+					content = await handlers.handleClientMergeGet(db, digest, keystore);	
 					break;
 
 				case 'post:automerge:server':
@@ -244,30 +182,13 @@ async function processRequest(req, res) {
 				//	break;
 
 				case 'get:automerge:none':
-					if (digest instanceof Array) {
-						let response = [];
-						for (let dg of digest) {
-							const metadata = await handlers.handleNoMergeGet(db, dg, keystore);
-							response.push(generateResponseBody(dg, metadata));
-						}
-						const responseContentLength = (new TextEncoder().encode(response.toString())).length;
-						res.writeHead(207, {
-							"Access-Control-Allow-Origin": "*",
-							"Content-Type": contentType,
-							"Content-Length": responseContentLength,
-						});
-						res.write(response.toString());
+					r = await handlers.handleNoMergeGet(db, digest, keystore);	
+					if (r == false) {
+						res.writeHead(404, {"Content-Type": "text/plain"});
 						res.end();
 						return;
-					} else {
-						r = await handlers.handleNoMergeGet(db, digest, keystore);
-						if (r == false) {
-							res.writeHead(404, {"Content-Type": "text/plain"});
-							res.end();
-							return;
-						}
-						content = r;
 					}
+					content = r;
 					break;
 
 				default:
@@ -284,7 +205,7 @@ async function processRequest(req, res) {
 
 		if (content === undefined) {
 			console.error('empty content', data);
-			res.writeHead(404, {"Content-Type": "text/plain"});
+			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;
 		}

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.5')
+version = (0, 4, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1,6 @@
-cic_base[full_graph]~=0.1.2a61
+cic_base==0.1.3a3+build.984b5cff
+africastalking==1.2.3
+celery==4.4.7
+confini==0.3.6rc3
+semver==2.13.0
+SQLAlchemy==1.3.20

apps/cic-notify/test_requirements.txt

@@ -2,4 +2,3 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
-

apps/cic-ussd/cic_ussd/processor.py

@@ -294,6 +294,7 @@ def process_display_user_metadata(user: Account, display_key: str):
             preferred_language=user.preferred_language,
             full_name=absent,
             gender=absent,
+            age=absent,
             location=absent,
             products=absent
         )

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b21
-cic-eth~=0.11.0b16
-cic-notify~=0.4.0a5
+cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic-eth~=0.11.1a3
+cic-notify~=0.4.0a7
 cic-types~=0.1.0a11

apps/cic-ussd/test_requirements.txt

@@ -8,4 +8,4 @@ pytest-mock==3.3.1
 pytest-ordering==0.6
 pytest-redis==2.0.0
 requests-mock==1.8.0
-tavern==1.14.2
+tavern==1.14.2

apps/cic-ussd/tests/fixtures/integration.py

@@ -124,46 +124,6 @@ def second_profile_management_session_id() -> str:
     return session_id()
 
 
-@pytest.fixture(scope='session')
-def first_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_product() -> str:
-    return fake.color_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_product() -> str:
-    return fake.color_name()
-
-
 @pytest.fixture(scope='session')
 def first_profile_management_session_id_1() -> str:
     return session_id()

apps/cic-ussd/tests/integration/README.md

@@ -0,0 +1,25 @@
+# INTEGRATION TESTING
+
+This folder contains integration tests.
+
+## OVERVIEW
+
+There are four files defining the integration tests.
+
+* **test_account_creation**: Tests account sign up process.
+* **test_transactions**: Tests transactions between two accounts.
+* **test_profile_management**: Tests that account metadata can be edited.
+* **test_account_management**: Tests that account management functionalities are intact.
+
+## REQUIREMENTS
+
+In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000`
+which is the value set in the config file `.config/test/integration.ini`.
+
+This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image:
+
+```shell
+export DEV_FAUCET_AMOUNT=50000000
+RUN_MASK=1 docker-compose up contract-migration
+RUN_MASK=2 docker-compose up contract-migration
+```

apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml

@@ -214,12 +214,13 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '28'
+        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Enter first name\n0. Back"
+          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
+    delay_before: 10
 
   - name: Pin number confirmation [{second_account_pin_number} - second account]
     request:
@@ -232,227 +233,6 @@ stages:
       headers:
         content-type: "application/x-www-form-urlencoded"
       method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la kwanza\n0. Nyuma"
-
-  - name: Enter first name [first_account_given_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '29'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter family name\n0. Back"
-
-  - name: Enter first name [second_account_given_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la mwisho\n0. Nyuma"
-
-  - name: Enter last name [first_account_family_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back"
-
-  - name: Enter last name [second_account_family_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '64'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma"
-
-  - name: Select gender [Male - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '31'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter your location\n0. Back"
-
-  - name: Select gender [Female - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '27'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka eneo lako\n0. Nyuma"
-
-  - name: Enter location [first_account_location - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '55'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Please enter a product or service you offer\n0. Back"
-
-  - name: Enter location [second_account_location - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '42'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma"
-
-  - name: Enter product [first_account_product - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}*{first_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
-    delay_before: 10
-
-  - name: Enter product [second_account_product - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}*{second_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
     response:
       status_code:
         - 200

apps/cic-ussd/tests/integration/test_account_management.tavern.yaml

@@ -31,7 +31,6 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response

apps/cic-ussd/tests/integration/test_transactions.tavern.yaml

@@ -170,7 +170,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
+          expected_response: "CON {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
 
   - name: Enter transcation amount [second account]
     request:
@@ -191,7 +191,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
+          expected_response: "CON {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
 
   - name: Pin to authorize transaction [first account]
     request:
@@ -212,7 +212,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Your request has been sent. {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\n00. Back\n99. Exit"
+          expected_response: "CON Your request has been sent. {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\n00. Back\n99. Exit"
 
   - name: Pin to authorize transaction [second account]
     request:
@@ -233,7 +233,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Ombi lako limetumwa. {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
+          expected_response: "CON Ombi lako limetumwa. {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
 
   - name: Verify balance changes [first account]
     delay_before: 10

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -12,7 +12,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_date_of_birth",
-    "dest": "enter_gender",
+    "dest": "enter_location",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/gender_setting_transitions.json

@@ -2,7 +2,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_gender",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/name_setting_transitions.json

@@ -15,7 +15,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_family_name",
-    "dest": "enter_date_of_birth",
+    "dest": "enter_gender",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
   },

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -55,7 +55,7 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit Age
+      3. Edit age
       4. Edit location
       5. Edit products
       6. View my profile

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -55,7 +55,7 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3  Weka umri
+      3. Weka umri
       4. Weka eneo
       5. Weka bidhaa
       6. Angalia wasifu wako

apps/contract-migration/requirements.txt

@@ -1,4 +1,4 @@
-	cic-base[full_graph]==0.1.2b15
+	cic-base==0.1.2b15
 	sarafu-faucet==0.0.3a3
 	sarafu-token==0.0.1a8
 	cic-eth==0.11.0b16 

apps/data-seeding/README.md

@@ -171,7 +171,11 @@ Then, in sequence, run in first terminal:
 
 In second terminal:
 
-`python cic_ussd/import_users.py -v -c config out`
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> -c config out`
+
+In the event that you are running the command in a local environment you may want to consider passing the `--ussd-no-ssl` flag i.e:
+
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
 
 
 
@@ -199,6 +203,13 @@ If _number of users_ is omitted the script will run until manually interrupted.
 If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
 
 
+### Importing preferences metadata
+
+`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
+
+If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
+
+
 ##### Importing pins and ussd data (optional)
 
 Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:

apps/data-seeding/cic_eth/import_balance.py

@@ -18,19 +18,17 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
-        block_by_number,
-        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -38,6 +36,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -70,13 +69,14 @@ elif args.vv == True:
 config_dir = os.path.join(args.c)
 os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
-config.process()
 # override args
+config.process()
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y'),
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -185,27 +185,6 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
-#class BlockGetter:
-#
-#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
-#        self.conn = conn
-#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
-#
-#
-#    def get(self, n):
-#        o = block_by_number(n)
-#        r = self.conn.do(o)
-#        b = None
-#        try:
-#            b = Block(r)
-#        except TypeError as e:
-#            if r == None:
-#                logg.debug('block not found {}'.format(n))
-#            else:
-#                logg.error('block retrieve error {}'.format(e))
-#        return b
-
-
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -226,11 +205,13 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-    
-    o = jsonrpc_template()
+   
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -244,10 +225,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -305,7 +287,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/config/cic.ini

@@ -7,4 +7,5 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-user_ussd_svc_service_port=
+user_ussd_svc_service_port =
+

apps/data-seeding/config/eth.ini

@@ -1,8 +1,2 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_provider_address = 
-#chain_id = 
-abi_dir = /usr/local/share/cic/solidity/abi
-account_accounts_index_writer = 

apps/data-seeding/create_import_users.py

@@ -204,9 +204,9 @@ def gen():
          ]))
     if random.randint(0, 1):
         # fake.local_latitude()
-        p.location['latitude'] = (random.random() + 180) - 90
+        p.location['latitude'] = (random.random() * 180) - 90
         # fake.local_latitude()
-        p.location['longitude'] = (random.random() + 360) - 180
+        p.location['longitude'] = (random.random() * 360) - 179
 
     return (old_blockchain_checksum_address, phone, p)
 

apps/data-seeding/docker/Dockerfile

@@ -1,19 +1,21 @@
 # syntax = docker/dockerfile:1.2
-FROM python:3.8.6-slim-buster as compile-image
-
-WORKDIR /root
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
+#FROM python:3.8.6-slim-buster as compile-image
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
 
 RUN mkdir -vp /usr/local/etc/cic
 
+COPY data-seeding/package.json \
+     data-seeding/package-lock.json \
+     .
+
+RUN npm install
+
 COPY data-seeding/requirements.txt . 
 
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
+
 COPY data-seeding/ . 
 
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
-
-
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -18,25 +18,24 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
-        block_by_number,
-        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -75,7 +74,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -184,27 +183,6 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
-#class BlockGetter:
-#
-#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
-#        self.conn = conn
-#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
-#
-#
-#    def get(self, n):
-#        o = block_by_number(n)
-#        r = self.conn.do(o)
-#        b = None
-#        try:
-#            b = Block(r)
-#        except TypeError as e:
-#            if r == None:
-#                logg.debug('block not found {}'.format(n))
-#            else:
-#                logg.error('block retrieve error {}'.format(e))
-#        return b
-
-
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -225,11 +203,13 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-    
-    o = jsonrpc_template()
+   
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -243,10 +223,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -300,7 +281,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/eth/import_users.py

@@ -59,7 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b15
-sarafu-faucet==0.0.3a3
-cic-eth==0.11.0b16
-cic-types==0.1.0a11
-crypto-dev-signer==0.4.14b3
+cic_base[full_graph]==0.1.3a3+build.984b5cff
+sarafu-faucet==0.0.4a1
+cic-eth==0.11.1a1
+cic-types==0.1.0a13
+crypto-dev-signer==0.4.14b6

apps/data-seeding/verify.py

@@ -25,7 +25,7 @@ from chainlib.eth.gas import (
 )
 from chainlib.eth.tx import TxFactory
 from chainlib.hash import keccak256_string_to_hex
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from cic_types.models.person import (
     Person,
     generate_metadata_pointer,
@@ -254,6 +254,10 @@ class Verifier:
             if len(k) > 7 and k[:7] == 'verify_':
                 logg.debug('verifier has verify method {}'.format(k))
                 verifymethods.append(k[7:])
+        o = self.faucet_tx_factory.token_amount(self.faucet_address, sender_address=ZERO_ADDRESS)
+        r = self.conn.do(o)
+        self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
+        logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
 
         self.state = VerifierState(verifymethods, active_tests=active_tests)
 
@@ -264,9 +268,11 @@ class Verifier:
         data += eth_abi.encode_single('address', address).hex()
         tx = self.tx_factory.set_code(tx, data)
         tx = self.tx_factory.normalize(tx)
-        o = jsonrpc_template()
+        j = JSONRPCRequest()
+        o = j.template()
         o['method'] = 'eth_call'
         o['params'].append(tx)
+        o = j.finalize(o)
         r = self.conn.do(o)
         logg.debug('index check for {}: {}'.format(address, r))
         n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r)))
@@ -282,6 +288,7 @@ class Verifier:
         except ValueError:
             actual_balance = int(r)
         balance = int(balance / 1000000) * 1000000
+        balance += self.faucet_amount
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
             raise VerifierError((actual_balance, balance), 'balance')
@@ -429,10 +436,12 @@ def main():
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -441,10 +450,11 @@ def main():
     data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found account index address {}'.format(account_index_address))
@@ -453,10 +463,11 @@ def main():
     data += eth_abi.encode_single('bytes32', b'Faucet').hex()
     txf.set_code(tx, data)
     
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found faucet {}'.format(faucet_address))
@@ -471,10 +482,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token address {}'.format(sarafu_token_address))

apps/util/requirements/base_requirement.txt

@@ -0,0 +1 @@
+cic-base==0.1.3a3+build.984b5cff

apps/util/requirements/requirements.txt

@@ -0,0 +1 @@
+requirements-magic~=0.0.2

apps/util/requirements/update_base.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+which pyreq-merge &> /dev/null
+if [ $? -gt 0 ]; then
+	>&2 echo pyreq-merge missing, please install requirements
+	exit 1
+fi
+
+t=$(mktemp)
+>&2 echo using tmp $t
+
+repos=(../../cic-cache ../../cic-eth ../../cic-ussd ../../data-seeding ../../cic-notify)
+
+for r in ${repos[@]}; do
+	f="$r/requirements.txt"
+	>&2 echo updating $f
+	pyreq-update $f base_requirement.txt -vv > $t 
+	cp $t $f
+
+	f="$r/test_requirements.txt"
+	>&2 echo updating $f
+	pyreq-update $f base_requirement.txt -vv > $t 
+	cp $t $f
+done

ci_templates/.cic-template.yml

@@ -46,4 +46,16 @@ variables:
         when: always
 
 
-
+.publish_python:
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev
+  stage: publish 
+  variables:
+    #CI_DEBUG_TRACE: "true"
+    CIC_PACKAGE_REGISTRY_PROJECT_ID: 27624814
+  script:
+    - python -m pip wheel --extra-index-url https://pip.grassrootseconomics.net:8433 --wheel-dir=/tmp/wheelhouse -r requirements.txt
+    - python -m pip install --no-index --find-links=/tmp/wheelhouse -r requirements.txt
+    - python setup.py sdist bdist_wheel
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CIC_PACKAGE_REGISTRY_PROJECT_ID}/packages/pypi dist/*
+    # Below are the built dependencies
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url ${CI_API_V4_URL}/projects/${CIC_PACKAGE_REGISTRY_PROJECT_ID}/packages/pypi /tmp/wheelhouse/*

docker-compose.yml

@@ -387,44 +387,6 @@ services:
     # command: "/root/start_retry.sh -q cic-eth -vv"
 
 
-#  cic-eth-server:
-#    build:
-#      context: apps/
-#      dockerfile: cic-eth/docker/Dockerfile
-#    environment:
-#      CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
-#      CELERY_BROKER_URL: $CELERY_BROKER_URL
-#      CELERY_RESULT_URL: $CELERY_RESULT_URL
-#      SERVER_PORT: 8000
-#    depends_on:
-#      - eth
-#      - postgres
-#      - redis
-#    ports:
-#      - ${HTTP_PORT_CIC_ETH:-63314}:8000
-#    deploy:
-#      restart_policy:
-#        condition: on-failure
-#    volumes:
-#        - contract-config:/tmp/cic/config/:ro
-#    command:
-#      - /bin/bash
-#      - -c
-#      - |
-#        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-#        "/usr/local/bin/uwsgi" \
-#        --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \
-#        --http :80 \
-#        --pyargv -vv
-##     entrypoint:
-##       - "/usr/local/bin/uwsgi"
-##       - "--wsgi-file"
-##       - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py"
-##       - "--http"
-##       - ":80"
-#    # command: "--pyargv -vv"
-
-
 
   cic-notify-tasker:
     build: