add build and publish for base

Add allowance check and transferFrom task

See merge request grassrootseconomics/cic-internal-integration!203

.gitignore

@@ -11,3 +11,7 @@ build/
 **/*sqlite
 **/.nyc_output
 **/coverage
+**/.venv
+**/venv
+**/dist
+.idea

.gitlab-ci.yml

@@ -11,4 +11,4 @@ include:
 stages:
   - build
   - test
-  - release
+  - publish

apps/cic-base-os/Dockerfile

@@ -1,34 +0,0 @@
-# The solc image messes up the alpine environment, so we have to go all over again
-FROM python:3.8.6-slim-buster
-
-LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
-LABEL spdx-license-identifier="GPL-3.0-or-later"
-LABEL description="Base layer for buiding development images for the cic component suite"
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
-
-
-RUN echo installing nodejs tooling
-
-COPY ./dev/nvm.sh /root/
-
-# Install nvm with node and npm
-# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
-ENV NVM_DIR /root/.nvm
-ENV NODE_VERSION 15.3.0
-ENV BANCOR_NODE_VERSION 10.16.0
-
-RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
-	&& . $NVM_DIR/nvm.sh \
-	&& nvm install $NODE_VERSION \
-	&& nvm alias default $NODE_VERSION \
-	&& nvm use $NODE_VERSION \
-# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
-# owner of these files is "1001" by default - wtf
-	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
-
-ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
-ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
-

apps/cic-base-os/README.md

@@ -1 +0,0 @@
-## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-base/.gitignore

@@ -0,0 +1,7 @@
+gmon.out
+__pycache__
+*.pyc
+inc.sh
+*.egg-info
+build/
+dist/

apps/cic-base/.gitlab-ci.yml

@@ -0,0 +1,20 @@
+.cic_base_variables:
+    variables:
+        APP_NAME: cic-base
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_base_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
+
+build-mr-cic-base:
+    extends:
+        - .cic_base_changes_target
+        - .py_build_merge_request
+        - .cic_base_variables
+
+publish_python:
+  extends:
+    - .publish_python
+    - .cic_base_variables

apps/cic-base/MANIFEST.in

@@ -0,0 +1 @@
+include *requirements.txt

apps/cic-base/cic_base/__init__.py

@@ -0,0 +1,8 @@
+#from . import (
+#        config,
+#        argparse,
+#        rpc,
+#        signer,
+#        log,
+#        version,
+#        )

apps/cic-base/cic_base/argparse.py

@@ -0,0 +1,87 @@
+# standard imports
+import logging
+import argparse
+import os
+import sys
+
+# external imports
+from xdg.BaseDirectory import (
+        xdg_config_dirs,
+        load_first_config,
+        )
+
+logg = logging.getLogger(__file__)
+
+fallback_config_path = '/usr/local/etc'
+xdg_config_dirs += [fallback_config_path]
+default_config_dir = load_first_config('cic')
+if default_config_dir == None:
+    default_config_dir = os.path.join('.', '.cic')
+env_config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
+
+full_template = { 
+        # (long arg and key name, short var, type, default, help,)
+        'provider': ('p', str, None, 'RPC provider url',),
+        'registry_address': ('r', str, None, 'CIC registry address',),
+        'keystore_file': ('y', str, None, 'Keystore file',),
+        'config_dir': ('c', str, env_config_dir, 'Configuration directory',),
+        'queue': ('q', str, 'cic-eth', 'Celery task queue',),
+        'chain_spec': ('i', str, None, 'Chain spec string',),
+        'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
+        }
+
+default_include_args = [
+    'config_dir',
+    'provider',
+    'env_prefix',
+    ]
+
+sub = None
+
+def create(caller_dir, include_args=default_include_args):
+
+    argparser = argparse.ArgumentParser()
+
+    for k in include_args:
+        a = full_template[k]
+        long_flag = '--' + k.replace('_', '-')
+        short_flag = None
+        dest = None
+        if a[0] != None:
+            short_flag = '-' + a[0]
+            dest = a[0]
+        else:
+            dest = k
+        default = a[2]
+        if default == None and k == 'config_dir':
+            default = os.path.join(os.getcwd(), 'config')
+        
+        if short_flag == None:
+            argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
+        else:
+            argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
+
+    argparser.add_argument('-v', action='store_true', help='Be verbose')
+    argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+
+    return argparser
+
+
+def add(argparser, processor, name, description=None):
+    processor(argparser)
+
+    return argparser
+
+
+def parse(argparser, logger=None):
+
+    args = argparser.parse_args(sys.argv[1:])
+
+    # handle logging input
+    if logger != None:
+        if args.vv:
+            logger.setLevel(logging.DEBUG)
+        elif args.v:
+            logger.setLevel(logging.INFO)
+
+    return args

apps/cic-base/cic_base/config.py

@@ -0,0 +1,52 @@
+# standard imports
+import logging
+
+# external imports
+import confini
+
+# local imports
+from .error import ConfigError
+
+logg = logging.getLogger(__name__)
+
+
+default_arg_overrides = {
+    'p': 'ETH_PROVIDER',
+    'i': 'CIC_CHAIN_SPEC',
+    'r': 'CIC_REGISTRY_ADDRESS',
+    }
+
+
+def override(config, override_dict, label):
+    config.dict_override(override_dict, label)
+    config.validate()
+    return config
+
+
+def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
+    # handle config input
+    config = None
+    try:
+        config = confini.Config(config_dir, env_prefix)
+    except OSError:
+        pass
+
+    if config == None:
+        raise ConfigError('directory {} not found'.format(config_dir))
+
+    config.process()
+    if arg_overrides != None and args != None:
+        override_dict = {}
+        for k in arg_overrides:
+            v = getattr(args, k)
+            if v != None:
+                override_dict[arg_overrides[k]] = v
+        config = override(config, override_dict, 'args')
+    else:
+        config.validate()
+
+    return config
+
+
+def log(config):
+    logg.debug('config loaded:\n{}'.format(config))

apps/cic-base/cic_base/error.py

@@ -0,0 +1,2 @@
+class ConfigError(Exception):
+    pass

apps/cic-base/cic_base/log.py

@@ -0,0 +1,18 @@
+# standard imports
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+
+default_mutelist = [
+        'urllib3',
+        'websockets.protocol',
+        'web3.RequestManager',
+        'web3.providers.WebsocketProvider',
+        'web3.providers.HTTPProvider',
+        ]
+
+def create(name=None, mutelist=default_mutelist):
+    logg = logging.getLogger(name)
+    for m in mutelist:
+        logging.getLogger(m).setLevel(logging.CRITICAL)
+    return logg

apps/cic-base/cic_base/rpc.py

@@ -0,0 +1,13 @@
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.connection import EthUnixSignerConnection
+from chainlib.eth.sign import (
+        sign_transaction,
+        sign_message,
+        )
+
+
+def setup(chain_spec, evm_provider, signer_provider=None):
+    RPCConnection.register_location(evm_provider, chain_spec, 'default')
+    if signer_provider != None:
+        RPCConnection.register_location(signer_provider, chain_spec, 'signer', constructor=EthUnixSignerConnection)

apps/cic-base/cic_base/signer.py

@@ -0,0 +1,27 @@
+# standard imports
+import logging
+import os
+
+# external imports 
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore.dict import DictKeystore
+
+logg = logging.getLogger(__name__)
+
+keystore = DictKeystore()
+
+default_passphrase = os.environ.get('ETH_PASSPHRASE', '')
+
+
+def from_keystore(keyfile, passphrase=default_passphrase):
+    global keystore
+
+    # signer
+    if keyfile == None:
+        raise ValueError('please specify signer keystore file')
+
+    logg.debug('loading keystore file {}'.format(keyfile))
+    address = keystore.import_keystore_file(keyfile, password=passphrase)
+
+    signer = EIP155Signer(keystore)
+    return (address, signer,)

apps/cic-base/cic_base/sqlalchemy.py

@@ -0,0 +1,122 @@
+# stanard imports
+import logging
+
+# third-party imports
+from sqlalchemy import Column, Integer
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import (
+        StaticPool,
+        QueuePool,
+        AssertionPool,
+        )
+
+logg = logging.getLogger()
+
+Model = declarative_base(name='Model')
+
+
+class SessionBase(Model):
+    """The base object for all SQLAlchemy enabled models. All other models must extend this.
+    """
+    __abstract__ = True
+ 
+    id = Column(Integer, primary_key=True)
+
+    engine = None
+    """Database connection engine of the running aplication"""
+    sessionmaker = None
+    """Factory object responsible for creating sessions from the  connection pool"""
+    transactional = True
+    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
+    poolable = True
+    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
+    procedural = True
+    """Whether the database backend supports stored procedures"""
+    localsessions = {}
+    """Contains dictionary of sessions initiated by db model components"""
+
+
+    @staticmethod
+    def create_session():
+        """Creates a new database session.
+        """
+        return SessionBase.sessionmaker()
+
+
+    @staticmethod
+    def _set_engine(engine):
+        """Sets the database engine static property
+        """
+        SessionBase.engine = engine
+        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
+
+
+    @staticmethod
+    def connect(dsn, pool_size=8, debug=False):
+        """Create new database connection engine and connect to database backend.
+
+        :param dsn: DSN string defining connection.
+        :type dsn: str
+        """
+        e = None
+        if SessionBase.poolable:
+            poolclass = QueuePool
+            if pool_size > 1:
+                e = create_engine(
+                        dsn,
+                        max_overflow=pool_size*3,
+                        pool_pre_ping=True,
+                        pool_size=pool_size,
+                        pool_recycle=60,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+            else:
+                if debug:
+                    poolclass = AssertionPool
+                else:
+                    poolclass = StaticPool
+
+                e = create_engine(
+                        dsn,
+                        poolclass=poolclass,
+                        echo=debug,
+                    )
+        else:
+            e = create_engine(
+                    dsn,
+                    echo=debug,
+                    )
+
+        SessionBase._set_engine(e)
+
+
+    @staticmethod
+    def disconnect():
+        """Disconnect from database and free resources.
+        """
+        SessionBase.engine.dispose()
+        SessionBase.engine = None
+
+
+    @staticmethod
+    def bind_session(session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+            localsession_key = str(id(localsession))
+            logg.debug('creating new session {}'.format(localsession_key))
+            SessionBase.localsessions[localsession_key] = localsession
+        return localsession
+
+
+    @staticmethod
+    def release_session(session=None):
+        session.flush()
+        session_key = str(id(session))
+        if SessionBase.localsessions.get(session_key) != None:
+            logg.debug('destroying session {}'.format(session_key))
+            session.commit()
+            session.close()

apps/cic-base/cic_base/version.py

@@ -0,0 +1,43 @@
+# standard imports
+import os
+import time
+import logging
+
+# third-party imports
+import semver
+
+version = (
+        0,
+        1,
+        2,
+        'beta.22',
+        )
+
+version_object = semver.VersionInfo(
+        major=version[0],
+        minor=version[1],
+        patch=version[2],
+        prerelease=version[3],
+        )
+
+
+def git_hash():
+    import subprocess
+    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+version_string = str(version_object)
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+            time_string_pair[0],
+            int(time_string_pair[1]),
+            )
+
+__version_string__ = version_string

apps/cic-base/docker/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.8.6-slim-buster
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client cargo
+
+WORKDIR /usr/src/cic-base
+COPY . /usr/src/cic-base/
+
+#RUN mkdir python 
+#WORKDIR ./python
+#COPY ./pep503.sh .
+#RUN pip download --no-cache-dir --extra-index-url https://pip.grassrootseconomics.net:8433 cic-base[full_graph]==0.1.1a6
+RUN pip install -r requirements.txt
+RUN python setup.py bdist_wheel
+RUN pip download --extra-index-url https://pip.grassrootseconomics.net:8433 dist/$(basename $(ls dist/*))
+RUN mkdir packages && \
+	cd packages && \
+	bash ../docker/pep503.sh ..
+
+WORKDIR /usr/src/cic-base/packages
+
+RUN ls
+RUN ls ..
+
+ENTRYPOINT ["python", "-m", "http.server", "8080"]

apps/cic-base/docker/pep503.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env 
+
+dest=`pwd`
+d=$1
+for df in `find $d -name "*.whl" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done
+for df in `find $d -name "*.tar.gz" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done
+for df in `find $d -name "*.zip" -type f`; do
+	f=`basename $df`
+	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
+	mkdir -v $dest/$pd
+	mv -v $df $dest/$pd/
+done

apps/cic-base/examples/cli.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+import cic_base.argparse
+import cic_base.config
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+def more_argparse(argparser):
+    argparser.add_argument('--foo', type=str, help='foo')
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+
+argparser = cic_base.argparse.create(script_dir, include_args=cic_base.argparse.full_template)
+args = cic_base.argparse.parse(argparser, logger=logg)
+config = cic_base.config.create(args.c, args, env_prefix=args.env_prefix)
+
+cic_base.config.log(config)

apps/cic-base/examples/config/foo.ini

@@ -0,0 +1,3 @@
+[foo]
+bar = 42
+baz = xyzzy

apps/cic-base/full_graph_requirements.txt

@@ -0,0 +1,98 @@
+africastalking==1.2.3
+alembic==1.4.2
+amqp==2.6.1
+attrs==20.3.0
+base58==2.1.0
+bcrypt==3.2.0
+billiard==3.6.3.0
+bip-utils==1.4.0
+bitarray==1.2.2
+blake2b-py==0.1.4
+cached-property==1.5.2
+celery==4.4.7
+certifi==2020.12.5
+cffi==1.14.3
+chainlib==0.0.1a18
+chainsyncer==0.0.1a18
+chardet==3.0.4
+confini==0.3.6rc3
+contextlib2==0.6.0.post1
+coverage==5.4
+cryptography==3.2.1
+cytoolz==0.11.0
+ecdsa==0.16.1
+ecuth==0.4.5a1
+Faker==4.17.1
+hexathon==0.0.1a3
+hexbytes==0.2.1
+http-hoba-auth==0.2.0
+idna==2.10
+iniconfig==1.1.1
+ipfshttpclient==0.6.1
+json-rpc==1.13.0
+jsonschema==3.2.0
+kombu==4.6.11
+lru-dict==1.1.7
+Mako==1.1.3
+MarkupSafe==1.1.1
+mirakuru==2.3.0
+moolb==0.1.1b2
+more-itertools==8.7.0
+multiaddr==0.0.9
+mypy-extensions==0.4.3
+netaddr==0.8.0
+packaging==20.9
+parsimonious==0.8.1
+phonenumbers==8.12.12
+pluggy==0.13.1
+port-for==0.4
+protobuf==3.15.1
+psutil==5.8.0
+psycopg2==2.8.6
+py==1.9.0
+py-ecc==4.1.0
+py-eth==0.1.1
+pycparser==2.20
+pycryptodome==3.10.1
+pyethash==0.1.27
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pysha3==1.0.2
+pytest==6.0.1
+pytest-alembic==0.2.5
+pytest-celery==0.0.0a1
+pytest-cov==2.10.1
+pytest-mock==3.3.1
+pytest-redis==2.0.0
+python-dateutil==2.8.1
+python-editor==1.0.4
+python-gnupg==0.4.6
+python-i18n==0.3.9
+pytz==2021.1
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+rlp==2.0.1
+schema==0.7.4
+semantic-version==2.8.5
+semver==2.13.0
+six==1.15.0
+sortedcontainers==2.3.0
+SQLAlchemy==1.3.20
+sqlparse==0.4.1
+text-unidecode==1.3
+tinydb==4.2.0
+toml==0.10.2
+toolz==0.11.1
+transitions==0.8.4
+trie==2.0.0a5
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+uWSGI==2.0.19.1
+varint==1.0.2
+vine==1.3.0
+vobject==0.9.6.1
+web3==5.12.2
+websocket-client==0.57.0
+websockets==8.1
+yaml-acl==0.0.1

apps/cic-base/full_requirements.txt

@@ -0,0 +1,44 @@
+africastalking==1.2.3
+alembic==1.4.2
+bcrypt==3.2.0
+celery==4.4.7
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b4
+cryptography==3.2.1
+ecuth==0.4.5a5
+eth-accounts-index==0.0.11a14
+eth-address-index==0.1.1a12
+eth-contract-registry==0.5.5a3
+erc20-transfer-authorization==0.3.1a7
+erc20-faucet==0.2.1a5
+faker==4.17.1
+http-hoba-auth==0.2.1a2
+moolb==0.1.1b2
+phonenumbers==8.12.12
+psycopg2==2.8.6
+py-eth~=0.1.1
+pytest==6.0.1
+pytest-alembic==0.2.5
+pytest-celery==0.0.0a1
+pytest-cov==2.10.1
+pytest-mock==3.3.1
+pytest-redis==2.0.0
+python-i18n==0.3.9
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+semver==2.13.0
+SQLAlchemy==1.3.20
+sqlparse==0.4.1
+tinydb==4.2.0
+transitions==0.8.4
+uWSGI==2.0.19.1
+vobject==0.9.6.1
+web3==5.12.2
+websockets==8.1
+yaml-acl==0.0.1
+rlp==2.0.1
+cryptocurrency-cli-tools==0.0.5
+websocket-client==0.57.0
+hexathon==0.0.1a7
+chainsyncer~=0.0.2a5

apps/cic-base/mac_requirements.txt

@@ -0,0 +1,40 @@
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b1
+semver==2.13.0
+SQLAlchemy==1.3.20
+pyxdg==0.27
+chainlib==0.0.2a10
+alembic==1.4.2
+celery==4.4.7
+cryptography==3.2.1
+ecuth==0.4.5a1
+eth-accounts-index==0.0.11a8
+eth-address-index==0.1.1a8
+eth-contract-registry==0.5.4a9
+erc20-transfer-authorization==0.3.1a4
+erc20-single-shot-faucet==0.2.0a11
+faker==4.17.1
+http-hoba-auth==0.2.0
+moolb==0.1.1b2
+phonenumbers==8.12.12
+psycopg2==2.8.6
+python-i18n==0.3.9
+PyYAML==5.3.1
+redis==3.5.3
+requests==2.24.0
+sqlparse==0.4.1
+transitions==0.8.4
+uWSGI==2.0.19.1
+vobject==0.9.6.1
+web3==5.12.2
+websockets==8.1
+yaml-acl==0.0.1
+rlp==2.0.1
+cryptocurrency-cli-tools==0.0.4
+giftable-erc20-token==0.0.8a8
+websocket-client==0.57.0
+hexathon==0.0.1a7
+chainsyncer==0.0.2b1
+sarafu-faucet==0.0.2a20
+cic-types==0.1.0a10
+cic-eth-registry==0.5.4a13

apps/cic-base/requirements.txt

@@ -0,0 +1,9 @@
+confini==0.3.6rc3
+crypto-dev-signer==0.4.14b4
+semver==2.13.0
+SQLAlchemy==1.3.20
+pyxdg==0.27
+chainlib==0.0.3rc3
+eth-erc20==0.0.9a4
+liveness==0.0.1a7
+requirements-magic~=0.0.1a2

apps/cic-base/setup.cfg

@@ -0,0 +1,35 @@
+[metadata]
+name = cic-base
+version = attr: cic_base.version.__version_string__
+description = CIC python base
+author = Louis Holbrook
+author_email = dev@holbrook.no
+url = https://gitlab.com/grassrootseconomics/cic-eth
+keywords =
+	cic
+	cryptocurrency
+	ethereum
+classifiers =
+	Programming Language :: Python :: 3
+	Operating System :: OS Independent
+	Development Status :: 3 - Alpha
+	Environment :: No Input/Output (Daemon)
+	Intended Audience :: Developers
+	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+	Topic :: Internet
+#	Topic :: Blockchain :: EVM
+license = GPL3
+licence_files =
+	LICENSE.txt
+
+[options]
+python_requires = >= 3.6
+packages = 
+	cic_base
+
+
+[options.entry_points]
+console_scripts =
+	cic-base-audit = cic_base.runnable.audit:main
+	cic-base-merge = cic_base.runnable.merge:main
+	cic-base-update = cic_base.runnable.update:main

apps/cic-base/setup.py

@@ -0,0 +1,50 @@
+from setuptools import setup
+import configparser
+import os
+import logging
+import re
+
+logg = logging.getLogger(__name__)
+
+
+re_v = r'[~><=]='
+def merge(requirements_files, base_dir='.'):
+
+    requirements = {}
+    for r in requirements_files:
+        filepath = os.path.join(base_dir, r)
+        logg.debug('reading {}'.format(filepath))
+        f = open(filepath, 'r')
+        while True:
+            l = f.readline()
+            if l == '':
+                break
+            l = l.rstrip()
+            m = re.split(re_v, l)
+            k = m[0]
+            if k == None:
+                raise ValueError('invalid requirement line {}'.format(l))
+            if requirements.get(k) == None:
+                logg.info('adding {} -> {}'.format(k, l))
+                requirements[k] = l
+            else:
+                logg.debug('skipping {}'.format(l))
+        f.close()
+
+    return list(requirements.values())
+
+
+requirements = []
+f = open('requirements.txt', 'r')
+while True:
+    l = f.readline()
+    if l == '':
+        break
+    requirements.append(l.rstrip())
+f.close()
+
+
+
+setup(
+    install_requires=requirements,
+        )

apps/cic-base/tests/base.py

@@ -0,0 +1,22 @@
+# standard imports
+import logging
+import unittest
+
+# external imports
+from chainlib.chain import ChainSpec
+
+# local imports
+from cic_base.rpc import setup as rpc_setup
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+
+class TestBase(unittest.TestCase):
+
+    def setUp(self):
+        self.chain_spec = ChainSpec('evm', 'foo', 42)
+        rpc_setup(self.chain_spec, 'http://localhost:8545', signer_provider='ipc://tmp/foo')
+
+    def tearDown(self):
+        pass

apps/cic-base/tests/test_basic.py

@@ -0,0 +1,14 @@
+# standard imports
+import unittest
+
+# local imports
+from tests.base import TestBase
+
+
+class TestBasic(TestBase):
+    
+    def test_basic(self):
+        pass
+
+if __name__ == '__main__':
+    unittest.main()

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -16,6 +16,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -28,10 +29,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -113,10 +112,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,12 +1,14 @@
-cic-base~=0.1.2b10
+cic-base==0.1.3a3+build.984b5cff
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.6a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+rlp==2.0.1
+chainsyncer[sql]~=0.0.3a3
+erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-cache/test_requirements.txt

@@ -6,6 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-web3==5.12.2
-cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8
+cic_base[full]==0.1.3a3+build.984b5cff
+sarafu-faucet~=0.0.4a1

apps/cic-eth/MANIFEST.in

@@ -0,0 +1,2 @@
+include *requirements.txt
+

apps/cic-eth/cic_eth/api/api_admin.py

@@ -562,13 +562,13 @@ class AdminApi:
             tx['source_token_symbol'] = source_token.symbol
             o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['sender_token_balance'] = erc20_c.parse_balance(r)
 
         if destination_token != None:
             tx['destination_token_symbol'] = destination_token.symbol
             o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
             r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance_of(r)
+            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
             #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
 
         # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which

apps/cic-eth/cic_eth/api/api_task.py

@@ -204,6 +204,82 @@ class Api:
 #        return t
 
 
+    def transfer_from(self, from_address, to_address, value, token_symbol, spender_address):
+        """Executes a chain of celery tasks that performs a transfer of ERC20 tokens by one address on behalf of another address to a third party.
+
+        :param from_address: Ethereum address of sender
+        :type from_address: str, 0x-hex
+        :param to_address: Ethereum address of recipient
+        :type to_address: str, 0x-hex
+        :param value: Estimated return from conversion
+        :type  value: int
+        :param token_symbol: ERC20 token symbol of token to send
+        :type token_symbol: str
+        :param spender_address: Ethereum address of recipient
+        :type spender_address: str, 0x-hex
+        :returns: uuid of root task
+        :rtype: celery.Task
+        """
+        s_check = celery.signature(
+                'cic_eth.admin.ctrl.check_lock',
+                [
+                    [token_symbol],
+                    self.chain_spec.asdict(),
+                    LockEnum.QUEUE,
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_nonce = celery.signature(
+                'cic_eth.eth.nonce.reserve_nonce',
+                [
+                    self.chain_spec.asdict(),
+                    from_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens = celery.signature(
+                'cic_eth.eth.erc20.resolve_tokens_by_symbol',
+                [
+                    self.chain_spec.asdict(),
+                    ],
+                queue=self.queue,
+                )
+        s_allow = celery.signature(
+                'cic_eth.eth.erc20.check_allowance',
+                [
+                    from_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_transfer = celery.signature(
+                'cic_eth.eth.erc20.transfer_from',
+                [
+                    from_address,
+                    to_address,
+                    value,
+                    self.chain_spec.asdict(),
+                    spender_address,
+                    ],
+                queue=self.queue,
+                )
+        s_tokens.link(s_allow)
+        s_nonce.link(s_tokens)
+        s_check.link(s_nonce)
+        if self.callback_param != None:
+            s_transfer.link(self.callback_success)
+            s_allow.link(s_transfer).on_error(self.callback_error)
+        else:
+            s_allow.link(s_transfer)
+
+        t = s_check.apply_async(queue=self.queue)
+        return t
+
+
+
     def transfer(self, from_address, to_address, value, token_symbol):
         """Executes a chain of celery tasks that performs a transfer of ERC20 tokens from one address to another.
 

apps/cic-eth/cic_eth/error.py

@@ -80,3 +80,8 @@ class SignerError(SeppukuError):
 class RoleAgencyError(SeppukuError):
     """Exception raise when a role cannot perform its function. This is a critical exception
     """
+
+
+class YouAreBrokeError(Exception):
+    """Exception raised when a value transfer is attempted without access to sufficient funds
+    """

apps/cic-eth/cic_eth/eth/erc20.py

@@ -24,6 +24,7 @@ from cic_eth.error import (
         TokenCountError,
         PermanentTxError,
         OutOfGasError,
+        YouAreBrokeError,
         )
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import (
@@ -71,6 +72,117 @@ def balance(tokens, holder_address, chain_spec_dict):
     return tokens
 
 
+@celery_app.task(bind=True)
+def check_allowance(self, tokens, holder_address, value, chain_spec_dict, spender_address):
+    """Best-effort verification that the allowance for a transfer from spend is sufficient.
+
+    :raises YouAreBrokeError: If allowance is insufficient
+    
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :return: Token list as passed to task
+    :rtype: dict
+    """
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+
+    caller_address = ERC20Token.caller_address 
+    c = ERC20(chain_spec)
+    o = c.allowance(t['address'], holder_address, spender_address, sender_address=caller_address)
+    r = rpc.do(o)
+    allowance = c.parse_allowance(r)
+    if allowance < value:
+        errstr = 'allowance {} insufficent to transfer {} {} by {} on behalf of {}'.format(allowance, value, t['symbol'], spender_address, holder_address)
+        logg.error(errstr)
+        raise YouAreBrokeError(errstr)
+
+    return tokens
+
+
+@celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
+def transfer_from(self, tokens, holder_address, receiver_address, value, chain_spec_dict, spender_address):
+    """Transfer ERC20 tokens between addresses
+
+    First argument is a list of tokens, to enable the task to be chained to the symbol to token address resolver function. However, it accepts only one token as argument.
+
+    :param tokens: Token addresses 
+    :type tokens: list of str, 0x-hex
+    :param holder_address: Token holder address
+    :type holder_address: str, 0x-hex
+    :param receiver_address: Token receiver address
+    :type receiver_address: str, 0x-hex
+    :param value: Amount of token, in 'wei'
+    :type value: int
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    :param spender_address: Address of account spending on behalf of holder
+    :type spender_address: str, 0x-hex
+    :raises TokenCountError: Either none or more then one tokens have been passed as tokens argument
+    :return: Transaction hash for tranfer operation
+    :rtype: str, 0x-hex
+    """
+    # we only allow one token, one transfer
+    logg.debug('tokens {}'.format(tokens))
+    if len(tokens) != 1:
+        raise TokenCountError
+    t = tokens[0]
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    queue = self.request.delivery_info.get('routing_key')
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
+
+    session = self.create_session()
+    nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
+    c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
+    try:
+        (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
+    except FileNotFoundError as e:
+        raise SignerError(e)
+    except ConnectionError as e:
+        raise SignerError(e)
+    
+
+    rpc_signer.disconnect()
+    rpc.disconnect()
+
+    cache_task = 'cic_eth.eth.erc20.cache_transfer_from_data'
+
+    register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
+    session.commit()
+    session.close()
+    
+    gas_pair = gas_oracle.get_gas(tx_signed_raw_hex)
+    gas_budget = gas_pair[0] * gas_pair[1]
+    logg.debug('transfer tx {} {} {}'.format(tx_hash_hex, queue, gas_budget))
+
+    s = create_check_gas_task(
+             [tx_signed_raw_hex],
+             chain_spec,
+             holder_address,
+             gas_budget,
+             [tx_hash_hex],
+             queue,
+            )
+    s.apply_async()
+    return tx_hash_hex
+
+
+
 @celery_app.task(bind=True, base=CriticalSQLAlchemyAndSignerTask)
 def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_dict):
     """Transfer ERC20 tokens between addresses
@@ -232,6 +344,7 @@ def resolve_tokens_by_symbol(self, token_symbols, chain_spec_dict):
         logg.debug('token {}'.format(token_address))
         tokens.append({
             'address': token_address,
+            'symbol': token_symbol,
             'converters': [],
             })
     rpc.disconnect()
@@ -279,6 +392,48 @@ def cache_transfer_data(
     return (tx_hash_hex, cache_id)
 
 
+@celery_app.task(base=CriticalSQLAlchemyTask)
+def cache_transfer_from_data(
+    tx_hash_hex,
+    tx_signed_raw_hex,
+    chain_spec_dict,
+        ):
+    """Helper function for otx_cache_transfer_from
+
+    :param tx_hash_hex: Transaction hash
+    :type tx_hash_hex: str, 0x-hex
+    :param tx: Signed raw transaction
+    :type tx: str, 0x-hex
+    :returns: Transaction hash and id of cache element in storage backend, respectively
+    :rtype: tuple
+    """
+    chain_spec = ChainSpec.from_dict(chain_spec_dict)
+    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
+
+    tx_data = ERC20.parse_transfer_from_request(tx['data'])
+    spender_address = tx_data[0]
+    recipient_address = tx_data[1]
+    token_value = tx_data[2]
+
+    session = SessionBase.create_session()
+    tx_cache = TxCache(
+        tx_hash_hex,
+        tx['from'],
+        recipient_address,
+        tx['to'],
+        tx['to'],
+        token_value,
+        token_value,
+        session=session,
+            )
+    session.add(tx_cache)
+    session.commit()
+    cache_id = tx_cache.id
+    session.close()
+    return (tx_hash_hex, cache_id)
+
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def cache_approve_data(
     tx_hash_hex,

apps/cic-eth/cic_eth/pytest/fixtures_config.py

@@ -2,13 +2,13 @@
 import os
 import logging
 
-# third-party imports
+# external imports
 import pytest
 import confini
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-logg = logging.getLogger(__file__)
+root_dir = os.path.dirname(os.path.dirname(script_dir))
+logg = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')

apps/cic-eth/cic_eth/pytest/fixtures_database.py

@@ -37,7 +37,8 @@ def init_database(
         database_engine,
         ):
 
-    rootdir = os.path.dirname(os.path.dirname(__file__))
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    rootdir = os.path.dirname(os.path.dirname(script_dir))
     dbdir = os.path.join(rootdir, 'cic_eth', 'db')
     migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
     if not os.path.isdir(migrationsdir):

apps/cic-eth/cic_eth/runnable/daemons/server.py

@@ -1,136 +0,0 @@
-# standard imports
-import os
-import re
-import logging
-import argparse
-import json
-
-# third-party imports
-import web3
-import confini
-import celery
-from json.decoder import JSONDecodeError
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db import dsn_from_config
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_eth', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-queue = args.q
-
-re_something = r'^/something/?'
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-def process_something(session, env):
-    r = re.match(re_something, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    #if env.get('CONTENT_TYPE') != 'application/json':
-    #    raise AttributeError('content type')
-
-    #if env.get('REQUEST_METHOD') != 'POST':
-    #    raise AttributeError('method')
-
-    #post_data = json.load(env.get('wsgi.input'))
-    
-    #return ('text/plain', 'foo'.encode('utf-8'),)
-
-
-# uwsgi application
-def application(env, start_response):
-    
-    for k in env.keys():
-        logg.debug('env {} {}'.format(k, env[k]))
-
-    headers = []
-    content = b''
-    err = None
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_something,
-            ]:
-        try:
-            r = handler(session, env)
-        except AttributeError as e:
-            logg.error('handler fail attribute {}'.format(e))
-            err = '400 Impertinent request'
-            break
-        except JSONDecodeError as e:
-            logg.error('handler fail json {}'.format(e))
-            err = '400 Invalid data format'
-            break
-        except KeyError as e:
-            logg.error('handler fail key {}'.format(e))
-            err = '400 Invalid JSON'
-            break
-        except ValueError as e:
-            logg.error('handler fail value {}'.format(e))
-            err = '400 Invalid data'
-            break
-        except RuntimeError as e:
-            logg.error('task fail value {}'.format(e))
-            err = '500 Task failed, sorry I cannot tell you more'
-            break
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    if err != None:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response(err, headers)
-        session.close()
-        return [content]
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,6 +194,7 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
+    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -15,6 +15,7 @@ import cic_base.config
 import cic_base.log
 import cic_base.argparse
 import cic_base.rpc
+from cic_base.eth.syncer import chain_interface
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -26,10 +27,8 @@ from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
-        HeadSyncer,
-        HistorySyncer,
-        )
+from chainsyncer.driver.head import HeadSyncer
+from chainsyncer.driver.history import HistorySyncer
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -80,6 +79,7 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -121,11 +121,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         11,
-        0,
-        'beta.16',
+        1,
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,25 +1,25 @@
-cic-base~=0.1.2b15
-celery==4.4.7
-crypto-dev-signer~=0.4.14b3
-confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a7
-redis==3.5.3
 alembic==1.4.2
-websockets==8.1
-requests~=2.24.0
-eth_accounts_index~=0.0.11a12
-erc20-transfer-authorization~=0.3.1a7
-uWSGI==2.0.19.1
-semver==2.13.0
-websocket-client==0.57.0
-moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3rc2
-hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a5
-chainqueue~=0.0.2b3
-sarafu-faucet==0.0.3a3
-erc20-faucet==0.2.1a4
+celery==4.4.7
+chainlib-eth~=0.0.5a1
+chainqueue~=0.0.2b5
+chainsyncer[sql]~=0.0.3a3
+cic-base==0.1.3a3+build.984b5cff
+cic-eth-registry~=0.5.6a1
 coincurve==15.0.0
+confini~=0.3.6rc3
+crypto-dev-signer~=0.4.14b6
+erc20-faucet~=0.2.2a1
+erc20-transfer-authorization~=0.3.2a1
+eth-address-index~=0.1.2a1
+eth_accounts_index~=0.0.12a1
+hexathon~=0.0.1a7
+moolb~=0.1.1b2
 potaahto~=0.0.1a2
 pycryptodome==3.10.1
+redis==3.5.3
+requests~=2.24.0
+sarafu-faucet~=0.0.4a1
+semver==2.13.0
+uWSGI==2.0.19.1
+websocket-client==0.57.0
+websockets==8.1

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/tests/conftest.py

@@ -17,11 +17,11 @@ root_dir = os.path.dirname(script_dir)
 sys.path.insert(0, root_dir)
 
 # assemble fixtures
-from tests.fixtures_config import *
-from tests.fixtures_database import *
-from tests.fixtures_celery import *
-from tests.fixtures_role import *
-from tests.fixtures_contract import *
+from cic_eth.pytest.fixtures_config import *
+from cic_eth.pytest.fixtures_celery import *
+from cic_eth.pytest.fixtures_database import *
+from cic_eth.pytest.fixtures_role import *
+from cic_eth.pytest.fixtures_contract import *
 from chainlib.eth.pytest import *
 from eth_contract_registry.pytest import *
 from cic_eth_registry.pytest.fixtures_contracts import *

apps/cic-eth/tests/task/conftest.py

@@ -1 +1 @@
-from tests.fixtures_celery import *
+from cic_eth.pytest.fixtures_celery import *

apps/cic-eth/tests/task/test_task_erc20.py

@@ -13,6 +13,7 @@ from chainlib.eth.tx import (
 
 # local imports
 from cic_eth.queue.tx import register_tx
+from cic_eth.error import YouAreBrokeError
 
 logg = logging.getLogger()
 
@@ -167,3 +168,101 @@ def test_erc20_approve_task(
     r = t.get_leaf()
 
     logg.debug('result {}'.format(r))
+
+
+def test_erc20_transfer_from_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    s_nonce = celery.signature(
+        'cic_eth.eth.nonce.reserve_nonce',
+        [
+            [token_object],
+            default_chain_spec.asdict(),
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            ],
+        queue=None,
+            )
+    s_transfer = celery.signature(
+            'cic_eth.eth.erc20.transfer_from',
+            [
+                custodial_roles['FOO_TOKEN_GIFTER'],
+                agent_roles['BOB'],
+                transfer_value,
+                default_chain_spec.asdict(),
+                agent_roles['ALICE'],
+                ],
+            queue=None,
+            )
+    s_nonce.link(s_transfer)
+    t = s_nonce.apply_async()
+    r = t.get_leaf()
+
+    logg.debug('result {}'.format(r))
+
+
+def test_erc20_allowance_check_task(
+        default_chain_spec,
+        foo_token,
+        agent_roles,
+        custodial_roles,
+        eth_signer,
+        eth_rpc,
+        init_database,
+        celery_session_worker,
+        token_roles,
+        ):
+
+    token_object = {
+        'address': foo_token,
+        'symbol': 'FOO',
+            }
+    transfer_value = 100 * (10 ** 6)
+
+    s_check = celery.signature(
+        'cic_eth.eth.erc20.check_allowance',
+        [
+            [token_object],
+            custodial_roles['FOO_TOKEN_GIFTER'],
+            transfer_value,
+            default_chain_spec.asdict(),
+            agent_roles['ALICE']
+            ],
+        queue=None,
+            )
+    t = s_check.apply_async()
+    with pytest.raises(YouAreBrokeError):
+        t.get() 
+
+    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], conn=eth_rpc)
+    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) 
+    (tx_hash, o) = c.approve(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], transfer_value)
+    r = eth_rpc.do(o)
+    o = receipt(tx_hash)
+    r = eth_rpc.do(o)
+    assert r['status'] == 1
+
+    t = s_check.apply_async()
+    t.get() 
+    assert t.successful()

apps/cic-notify/cic_notify/api.py

@@ -26,7 +26,7 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
         for q in qs[host]:
             if re.match(re_q, q['name']):
                 host_queues.append((host, q['name'],))
-   
+
     task_prefix_len = len(task_prefix)
     queue_tasks = []
     for (host, queue) in host_queues:
@@ -35,17 +35,18 @@ def get_sms_queue_tasks(app, task_prefix='cic_notify.tasks.sms.'):
             for task in tasks:
                 if len(task) >= task_prefix_len and task[:task_prefix_len] == task_prefix:
                     queue_tasks.append((queue, task,))
-    
+
     return queue_tasks
 
 
 class Api:
     # TODO: Implement callback strategy
-    def __init__(self, queue='cic-notify'):
+    def __init__(self, queue=None):
         """
         :param queue: The queue on which to execute notification tasks
         :type queue: str
         """
+        self.queue = queue
         self.sms_tasks = get_sms_queue_tasks(app)
         logg.debug('sms tasks {}'.format(self.sms_tasks))
 
@@ -61,13 +62,19 @@ class Api:
         """
         signatures = []
         for q in self.sms_tasks:
+
+            if not self.queue:
+                queue = q[0]
+            else:
+                queue = self.queue
+
             signature = celery.signature(
                     q[1],
                     [
                         message,
                         recipient,
                         ],
-                    queue=q[0],
+                    queue=queue,
                     )
             signatures.append(signature)
 

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -87,10 +87,18 @@ for key in config.store.keys():
         module = importlib.import_module(config.store[key])
         if key == 'TASKS_AFRICASTALKING':
             africastalking_notifier = module.AfricasTalkingNotifier
+
+            api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID')
+            logg.debug(f'SENDER ID VALUE IS: {api_sender_id}')
+
+            if not api_sender_id:
+                api_sender_id = None
+                logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}')
+
             africastalking_notifier.initialize(
                 config.get('AFRICASTALKING_API_USERNAME'),
                 config.get('AFRICASTALKING_API_KEY'),
-                config.get('AFRICASTALKING_API_SENDER_ID')
+                api_sender_id
             )
 
 

apps/cic-notify/cic_notify/version.py

@@ -9,7 +9,7 @@ import semver
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.5')
+version = (0, 4, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-notify/requirements.txt

@@ -1 +1,6 @@
-cic_base[full_graph]~=0.1.2a61
+cic_base==0.1.3a3+build.984b5cff
+africastalking==1.2.3
+celery==4.4.7
+confini==0.3.6rc3
+semver==2.13.0
+SQLAlchemy==1.3.20

apps/cic-notify/test_requirements.txt

@@ -2,4 +2,3 @@ pytest~=6.0.1
 pytest-celery~=0.0.0a1
 pytest-mock~=3.3.1
 pysqlite3~=0.4.3
-

apps/cic-ussd/cic_ussd/db/enum.py

@@ -0,0 +1,9 @@
+# standard import
+from enum import IntEnum
+
+
+class AccountStatus(IntEnum):
+    PENDING = 1
+    ACTIVE = 2
+    LOCKED = 3
+    RESET = 4

apps/cic-ussd/cic_ussd/db/models/account.py

@@ -1,19 +1,13 @@
 # standard imports
-from enum import IntEnum
-
-# third party imports
-from sqlalchemy import Column, Integer, String
 
 # local imports
+from cic_ussd.db.enum import AccountStatus
 from cic_ussd.db.models.base import SessionBase
 from cic_ussd.encoder import check_password_hash, create_password_hash
 
-
-class AccountStatus(IntEnum):
-    PENDING = 1
-    ACTIVE = 2
-    LOCKED = 3
-    RESET = 4
+# third party imports
+from sqlalchemy import Column, Integer, String
+from sqlalchemy.orm.session import Session
 
 
 class Account(SessionBase):
@@ -30,6 +24,21 @@ class Account(SessionBase):
     account_status = Column(Integer)
     preferred_language = Column(String)
 
+    @staticmethod
+    def get_by_phone_number(phone_number: str, session: Session):
+        """Retrieves an account from a phone number.
+        :param phone_number: The E164 format of a phone number.
+        :type phone_number:str
+        :param session:
+        :type session:
+        :return: An account object.
+        :rtype: Account
+        """
+        session = SessionBase.bind_session(session=session)
+        account = session.query(Account).filter_by(phone_number=phone_number).first()
+        SessionBase.release_session(session=session)
+        return account
+
     def __init__(self, blockchain_address, phone_number):
         self.blockchain_address = blockchain_address
         self.phone_number = phone_number

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -275,6 +275,18 @@
             "display_key": "ussd.kenya.new_pin_confirmation",
             "name": "new_pin_confirmation",
             "parent": "metadata_management"
+        },
+        "47": {
+            "description": "Year of birth entry menu.",
+            "display_key": "ussd.kenya.enter_date_of_birth",
+            "name": "enter_date_of_birth",
+            "parent": "metadata_management"
+        },
+        "48": {
+            "description": "Pin entry menu for changing year of birth data.",
+            "display_key": "ussd.kenya.dob_edit_pin_authorization",
+            "name": "dob_edit_pin_authorization",
+            "parent": "metadata_management"
         }
 
     }

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class CustomMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.custom', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PreferencesMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.preferences', identifier=identifier)

apps/cic-ussd/cic_ussd/operations.py

@@ -6,11 +6,13 @@ import logging
 import celery
 import i18n
 from cic_eth.api.api_task import Api
+from sqlalchemy.orm.session import Session
 from tinydb.table import Document
 from typing import Optional
 
 # local imports
 from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.db.models.task_tracker import TaskTracker
 from cic_ussd.menu.ussd_menu import UssdMenu
@@ -22,15 +24,18 @@ from cic_ussd.validator import check_known_user, validate_response_type
 logg = logging.getLogger()
 
 
-def add_tasks_to_tracker(task_uuid):
-    """
-    This function takes tasks spawned over api interfaces and records their creation time for tracking.
+def add_tasks_to_tracker(session, task_uuid: str):
+    """This function takes tasks spawned over api interfaces and records their creation time for tracking.
+    :param session:
+    :type session:
     :param task_uuid: The uuid for an initiated task.
     :type task_uuid: str
     """
+    session = SessionBase.bind_session(session=session)
     task_record = TaskTracker(task_uuid=task_uuid)
-    TaskTracker.session.add(task_record)
-    TaskTracker.session.commit()
+    session.add(task_record)
+    session.flush()
+    SessionBase.release_session(session=session)
 
 
 def define_response_with_content(headers: list, response: str) -> tuple:
@@ -95,6 +100,7 @@ def create_or_update_session(
         service_code: str,
         user_input: str,
         current_menu: str,
+        session,
         session_data: Optional[dict] = None) -> InMemoryUssdSession:
     """
     Handles the creation or updating of session as necessary.
@@ -108,12 +114,15 @@ def create_or_update_session(
     :type user_input: str
     :param current_menu: Menu name that is currently being displayed on the ussd session
     :type current_menu: str
+    :param session:
+    :type session:
     :param session_data: Any additional data that was persisted during the user's interaction with the system.
     :type session_data: dict.
     :return: ussd session object
     :rtype: InMemoryUssdSession
     """
-    existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
+    session = SessionBase.bind_session(session=session)
+    existing_ussd_session = session.query(UssdSession).filter_by(
         external_session_id=external_session_id).first()
 
     if existing_ussd_session:
@@ -132,20 +141,25 @@ def create_or_update_session(
             current_menu=current_menu,
             session_data=session_data
         )
+    SessionBase.release_session(session=session)
     return ussd_session
 
 
-def get_account_status(phone_number) -> str:
+def get_account_status(phone_number, session: Session) -> str:
     """Get the status of a user's account.
     :param phone_number: The phone number to be checked.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: The user account status.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    status = user.get_account_status()
-    Account.session.add(user)
-    Account.session.commit()
+    session = SessionBase.bind_session(session=session)
+    account = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    status = account.get_account_status()
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
 
     return status
 
@@ -165,6 +179,7 @@ def initiate_account_creation_request(chain_str: str,
                                       external_session_id: str,
                                       phone_number: str,
                                       service_code: str,
+                                      session,
                                       user_input: str) -> str:
     """This function issues a task to create a blockchain account on cic-eth. It then creates a record of the ussd
     session corresponding to the creation of the account and returns a response denoting that the user's account is
@@ -177,6 +192,8 @@ def initiate_account_creation_request(chain_str: str,
     :type phone_number: str
     :param service_code: The service code dialed.
     :type service_code: str
+    :param session:
+    :type session:
     :param user_input: The input entered by the user.
     :type user_input: str
     :return: A response denoting that the account is being created.
@@ -190,7 +207,7 @@ def initiate_account_creation_request(chain_str: str,
     creation_task_id = cic_eth_api.create_account().id
 
     # record task initiation time
-    add_tasks_to_tracker(task_uuid=creation_task_id)
+    add_tasks_to_tracker(task_uuid=creation_task_id, session=session)
 
     # cache account creation data
     cache_account_creation_task_id(phone_number=phone_number, task_id=creation_task_id)
@@ -204,6 +221,7 @@ def initiate_account_creation_request(chain_str: str,
         phone=phone_number,
         service_code=service_code,
         current_menu=current_menu.get('name'),
+        session=session,
         user_input=user_input)
 
     # define response to relay to user
@@ -268,12 +286,14 @@ def cache_account_creation_task_id(phone_number: str, task_id: str):
     redis_cache.persist(name=task_id)
 
 
-def process_current_menu(ussd_session: Optional[dict], user: Account, user_input: str) -> Document:
+def process_current_menu(account: Account, session: Session, ussd_session: Optional[dict], user_input: str) -> Document:
     """This function checks user input and returns a corresponding ussd menu
     :param ussd_session: An in db ussd session object.
     :type ussd_session: UssdSession
-    :param user: A user object.
-    :type user: Account
+    :param account: A account object.
+    :type account: Account
+    :param session:
+    :type session:
     :param user_input: The user's input.
     :type user_input: str
     :return: An in memory ussd menu object.
@@ -285,7 +305,13 @@ def process_current_menu(ussd_session: Optional[dict], user: Account, user_input
     else:
         # get current state
         latest_input = get_latest_input(user_input=user_input)
-        current_menu = process_request(ussd_session=ussd_session, user_input=latest_input, user=user)
+        session = SessionBase.bind_session(session=session)
+        current_menu = process_request(
+            account=account,
+            session=session,
+            ussd_session=ussd_session,
+            user_input=latest_input)
+        SessionBase.release_session(session=session)
     return current_menu
 
 
@@ -294,6 +320,7 @@ def process_menu_interaction_requests(chain_str: str,
                                       phone_number: str,
                                       queue: str,
                                       service_code: str,
+                                      session,
                                       user_input: str) -> str:
     """This function handles requests intended for interaction with ussd menu, it checks whether a user matching the
     provided phone number exists and in the absence of which it creates an account for the user.
@@ -308,25 +335,29 @@ def process_menu_interaction_requests(chain_str: str,
     :type queue: str
     :param service_code: The service dialed by the user making the request.
     :type service_code: str
+    :param session:
+    :type session:
     :param user_input: The inputs entered by the user.
     :type user_input: str
     :return: A response based on the request received.
     :rtype: str
     """
     # check whether the user exists
-    if not check_known_user(phone=phone_number):
+    if not check_known_user(phone_number=phone_number, session=session):
         response = initiate_account_creation_request(chain_str=chain_str,
                                                      external_session_id=external_session_id,
                                                      phone_number=phone_number,
                                                      service_code=service_code,
+                                                     session=session,
                                                      user_input=user_input)
 
     else:
-        # get user
-        user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
+        # get account
+        session = SessionBase.bind_session(session=session)
+        account = Account.get_by_phone_number(phone_number=phone_number, session=session)
 
         # retrieve and cache user's metadata
-        blockchain_address = user.blockchain_address
+        blockchain_address = account.blockchain_address
         s_query_person_metadata = celery.signature(
             'cic_ussd.tasks.metadata.query_person_metadata',
             [blockchain_address]
@@ -334,24 +365,25 @@ def process_menu_interaction_requests(chain_str: str,
         s_query_person_metadata.apply_async(queue='cic-ussd')
 
         # find any existing ussd session
-        existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
-            external_session_id=external_session_id).first()
+        existing_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first()
 
         # validate user inputs
         if existing_ussd_session:
             current_menu = process_current_menu(
+                account=account,
+                session=session,
                 ussd_session=existing_ussd_session.to_json(),
-                user=user,
                 user_input=user_input
             )
         else:
             current_menu = process_current_menu(
+                account=account,
+                session=session,
                 ussd_session=None,
-                user=user,
                 user_input=user_input
             )
 
-        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+        last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session)
 
         if last_ussd_session:
             # create or update the ussd session as appropriate
@@ -361,6 +393,7 @@ def process_menu_interaction_requests(chain_str: str,
                 service_code=service_code,
                 user_input=user_input,
                 current_menu=current_menu.get('name'),
+                session=session,
                 session_data=last_ussd_session.session_data
             )
         else:
@@ -369,15 +402,17 @@ def process_menu_interaction_requests(chain_str: str,
                 phone=phone_number,
                 service_code=service_code,
                 user_input=user_input,
-                current_menu=current_menu.get('name')
+                current_menu=current_menu.get('name'),
+                session=session
             )
 
         # define appropriate response
         response = custom_display_text(
+            account=account,
             display_key=current_menu.get('display_key'),
             menu_name=current_menu.get('name'),
+            session=session,
             ussd_session=ussd_session.to_json(),
-            user=user
         )
 
     # check that the response from the processor is valid
@@ -386,21 +421,26 @@ def process_menu_interaction_requests(chain_str: str,
 
     # persist session to db
     persist_session_to_db_task(external_session_id=external_session_id, queue=queue)
+    SessionBase.release_session(session=session)
 
     return response
 
 
-def reset_pin(phone_number: str) -> str:
+def reset_pin(phone_number: str, session: Session) -> str:
     """Reset account status from Locked to Pending.
     :param phone_number: The phone number belonging to the account to be unlocked.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: The status of the pin reset.
     :rtype: str
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    user.reset_account_pin()
-    Account.session.add(user)
-    Account.session.commit()
+    session = SessionBase.bind_session(session=session)
+    account = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    account.reset_account_pin()
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
 
     response = f'Pin reset for user {phone_number} is successful!'
     return response
@@ -438,11 +478,13 @@ def update_ussd_session(
     return session
 
 
-def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_session: dict):
+def save_to_in_memory_ussd_session_data(queue: str, session: Session, session_data: dict, ussd_session: dict):
     """This function is used to save information to the session data attribute of a ussd session object in the redis
     cache.
     :param queue: The queue on which the celery task should run.
     :type queue: str
+    :param session:
+    :type session:
     :param session_data: A dictionary containing data for a specific ussd session in redis that needs to be saved
     temporarily.
     :type session_data: dict
@@ -473,7 +515,7 @@ def save_to_in_memory_ussd_session_data(queue: str, session_data: dict, ussd_ses
         service_code=in_redis_ussd_session.get('service_code'),
         user_input=in_redis_ussd_session.get('user_input'),
         current_menu=in_redis_ussd_session.get('state'),
+        session=session,
         session_data=session_data
     )
     persist_session_to_db_task(external_session_id=external_session_id, queue=queue)
-

apps/cic-ussd/cic_ussd/phone_number.py

@@ -8,6 +8,10 @@ import phonenumbers
 from cic_ussd.db.models.account import Account
 
 
+class E164Format:
+    region = None
+
+
 def process_phone_number(phone_number: str, region: str):
     """This function parses any phone number for the provided region
     :param phone_number: A string with a phone number.
@@ -29,19 +33,5 @@ def process_phone_number(phone_number: str, region: str):
 
     return parsed_phone_number
 
-
-def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
-    """This function queries the database for a user based on the provided phone number.
-    :param phone_number: A valid phone number.
-    :type phone_number: str
-    :return: A user object matching a given phone number
-    :rtype: Account|None
-    """
-    # consider adding region to user's metadata
-    phone_number = process_phone_number(phone_number=phone_number, region='KE')
-    user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
-    return user
-
-
 class Support:
     phone_number = None

apps/cic-ussd/cic_ussd/processor.py

@@ -1,25 +1,27 @@
 # standard imports
+import datetime
 import logging
 import json
-import re
 from typing import Optional
 
 # third party imports
-import celery
 from sqlalchemy import desc
 from cic_eth.api import Api
+from sqlalchemy.orm.session import Session
 from tinydb.table import Document
 
 # local imports
 from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
 from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 from cic_ussd.db.models.ussd_session import UssdSession
-from cic_ussd.error import MetadataNotFoundError, SeppukuError
+from cic_ussd.db.enum import AccountStatus
+from cic_ussd.error import SeppukuError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
-from cic_ussd.phone_number import get_user_by_phone_number, Support
+from cic_ussd.phone_number import Support
 from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.conversions import to_wei, from_wei
@@ -61,47 +63,48 @@ def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()):
             raise SeppukuError(f'Could not retrieve default token for: {chain_str}')
 
 
-def process_pin_authorization(display_key: str, user: Account, **kwargs) -> str:
-    """
-    This method provides translation for all ussd menu entries that follow the pin authorization pattern.
+def process_pin_authorization(account: Account, display_key: str, **kwargs) -> str:
+    """This method provides translation for all ussd menu entries that follow the pin authorization pattern.
+    :param account: The account in a running USSD session.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user in a running USSD session.
-    :type user: Account
     :param kwargs: Any additional information required by the text values in the internationalization files.
     :type kwargs
     :return: A string value corresponding the ussd menu's text value.
     :rtype: str
     """
     remaining_attempts = 3
-    if user.failed_pin_attempts > 0:
+    if account.failed_pin_attempts > 0:
         return translation_for(
             key=f'{display_key}.retry',
-            preferred_language=user.preferred_language,
-            remaining_attempts=(remaining_attempts - user.failed_pin_attempts)
+            preferred_language=account.preferred_language,
+            remaining_attempts=(remaining_attempts - account.failed_pin_attempts)
         )
     else:
         return translation_for(
             key=f'{display_key}.first',
-            preferred_language=user.preferred_language,
+            preferred_language=account.preferred_language,
             **kwargs
         )
 
 
-def process_exit_insufficient_balance(display_key: str, user: Account, ussd_session: dict):
+def process_exit_insufficient_balance(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes the exit menu letting users their account balance is insufficient to perform a specific
     transaction.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
     :rtype: str
     """
     # get account balance
-    operational_balance = get_cached_operational_balance(blockchain_address=user.blockchain_address)
+    operational_balance = get_cached_operational_balance(blockchain_address=account.blockchain_address)
 
     # compile response data
     user_input = ussd_session.get('user_input').split('*')[-1]
@@ -111,13 +114,13 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     token_symbol = retrieve_token_symbol()
 
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
 
     tx_recipient_information = define_account_tx_metadata(user=recipient)
 
     return translation_for(
         key=display_key,
-        preferred_language=user.preferred_language,
+        preferred_language=account.preferred_language,
         amount=from_wei(transaction_amount),
         token_symbol=token_symbol,
         recipient_information=tx_recipient_information,
@@ -125,12 +128,14 @@ def process_exit_insufficient_balance(display_key: str, user: Account, ussd_sess
     )
 
 
-def process_exit_successful_transaction(display_key: str, user: Account, ussd_session: dict):
+def process_exit_successful_transaction(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes the exit menu after a successful initiation for a transfer of tokens.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: Corresponding translation text response
@@ -139,13 +144,13 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount')))
     token_symbol = retrieve_token_symbol()
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
-    tx_sender_information = define_account_tx_metadata(user=user)
+    tx_sender_information = define_account_tx_metadata(user=account)
 
     return translation_for(
         key=display_key,
-        preferred_language=user.preferred_language,
+        preferred_language=account.preferred_language,
         transaction_amount=from_wei(transaction_amount),
         token_symbol=token_symbol,
         recipient_information=tx_recipient_information,
@@ -153,13 +158,15 @@ def process_exit_successful_transaction(display_key: str, user: Account, ussd_se
     )
 
 
-def process_transaction_pin_authorization(user: Account, display_key: str, ussd_session: dict):
+def process_transaction_pin_authorization(account: Account, display_key: str, session: Session, ussd_session: dict):
     """This function processes pin authorization where making a transaction is concerned. It constructs a
     pre-transaction response menu that shows the details of the transaction.
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
+    :param session:
+    :type session:
     :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's
     text values.
     :type ussd_session: UssdSession
@@ -168,16 +175,16 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     """
     # compile response data
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     tx_recipient_information = define_account_tx_metadata(user=recipient)
-    tx_sender_information = define_account_tx_metadata(user=user)
+    tx_sender_information = define_account_tx_metadata(user=account)
 
     token_symbol = retrieve_token_symbol()
     user_input = ussd_session.get('session_data').get('transaction_amount')
     transaction_amount = to_wei(value=int(user_input))
     logg.debug('Requires integration to determine user tokens.')
     return process_pin_authorization(
-        user=user,
+        account=account,
         display_key=display_key,
         recipient_information=tx_recipient_information,
         transaction_amount=from_wei(transaction_amount),
@@ -186,14 +193,12 @@ def process_transaction_pin_authorization(user: Account, display_key: str, ussd_
     )
 
 
-def process_account_balances(user: Account, display_key: str, ussd_session: dict):
+def process_account_balances(user: Account, display_key: str):
     """
     :param user:
     :type user:
     :param display_key:
     :type display_key:
-    :param ussd_session:
-    :type ussd_session:
     :return:
     :rtype:
     """
@@ -257,6 +262,10 @@ def process_display_user_metadata(user: Account, display_key: str):
         contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard'))
         logg.debug(f'{contact_data}')
         full_name = f'{contact_data.get("given")} {contact_data.get("family")}'
+        date_of_birth = user_metadata.get('date_of_birth')
+        year_of_birth = date_of_birth.get('year')
+        present_year = datetime.datetime.now().year
+        age = present_year - year_of_birth
         gender = user_metadata.get('gender')
         products = ', '.join(user_metadata.get('products'))
         location = user_metadata.get('location').get('area_name')
@@ -265,6 +274,7 @@ def process_display_user_metadata(user: Account, display_key: str):
             key=display_key,
             preferred_language=user.preferred_language,
             full_name=full_name,
+            age=age,
             gender=gender,
             location=location,
             products=products
@@ -284,20 +294,18 @@ def process_display_user_metadata(user: Account, display_key: str):
             preferred_language=user.preferred_language,
             full_name=absent,
             gender=absent,
+            age=absent,
             location=absent,
             products=absent
         )
 
 
-
-def process_account_statement(user: Account, display_key: str, ussd_session: dict):
+def process_account_statement(user: Account, display_key: str):
     """
     :param user:
     :type user:
     :param display_key:
     :type display_key:
-    :param ussd_session:
-    :type ussd_session:
     :return:
     :rtype:
     """
@@ -399,23 +407,26 @@ def process_start_menu(display_key: str, user: Account):
     )
 
 
-def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
+def retrieve_most_recent_ussd_session(phone_number: str, session: Session) -> UssdSession:
     # get last ussd session based on user phone number
-    last_ussd_session = UssdSession.session\
-        .query(UssdSession)\
+    session = SessionBase.bind_session(session=session)
+    last_ussd_session = session.query(UssdSession)\
         .filter_by(msisdn=phone_number)\
         .order_by(desc(UssdSession.created))\
         .first()
+    SessionBase.release_session(session=session)
     return last_ussd_session
 
 
-def process_request(user_input: str, user: Account, ussd_session: Optional[dict] = None) -> Document:
+def process_request(account: Account, session, user_input: str, ussd_session: Optional[dict] = None) -> Document:
     """This function assesses a request based on the user from the request comes, the session_id and the user's
     input. It determines whether the request translates to a return to an existing session by checking whether the
     provided  session id exists in the database or whether the creation of a new ussd session object is warranted.
     It then returns the appropriate ussd menu text values.
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
+    :param session:
+    :type session:
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :param ussd_session: A JSON serialized in-memory ussd session object
@@ -423,22 +434,20 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
-    # retrieve metadata before any transition
-    key = generate_metadata_pointer(
-        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
-    )
-    person_metadata = get_cached_data(key=key)
 
     if ussd_session:
         if user_input == "0":
             return UssdMenu.parent_menu(menu_name=ussd_session.get('state'))
         else:
-            successive_state = next_state(ussd_session=ussd_session, user=user, user_input=user_input)
+            successive_state = next_state(
+                account=account,
+                session=session,
+                ussd_session=ussd_session,
+                user_input=user_input)
             return UssdMenu.find_by_name(name=successive_state)
     else:
-        if user.has_valid_pin():
-            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
+        if account.has_valid_pin():
+            last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session)
 
             if last_ussd_session:
                 # get last state
@@ -452,33 +461,35 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                     'exit_pin_mismatch',
                     'exit_invalid_request',
                     'exit_successful_transaction'
-                ] and person_metadata is not None:
+                ]:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)
         else:
-            if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name:
+            if account.failed_pin_attempts >= 3 and account.get_account_status() == AccountStatus.LOCKED.name:
                 return UssdMenu.find_by_name(name='exit_pin_blocked')
-            elif user.preferred_language is None:
+            elif account.preferred_language is None:
                 return UssdMenu.find_by_name(name='initial_language_selection')
             else:
                 return UssdMenu.find_by_name(name='initial_pin_entry')
 
 
-def next_state(ussd_session: dict, user: Account, user_input: str) -> str:
+def next_state(account: Account, session, ussd_session: dict, user_input: str) -> str:
     """This function navigates the state machine based on the ussd session object and user inputs it receives.
     It checks the user input and provides the successive state in the state machine. It then updates the session's
     state attribute with the new state.
+    :param account: The account requesting access to the ussd menu.
+    :type account: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
-    :param user: The user requesting access to the ussd menu.
-    :type user: Account
     :param user_input: The value a user enters in the ussd menu.
     :type user_input: str
     :return: A string value corresponding the successive give a specific state in the state machine.
     """
     state_machine = UssdStateMachine(ussd_session=ussd_session)
-    state_machine.scan_data((user_input, ussd_session, user))
+    state_machine.scan_data((user_input, ussd_session, account, session))
     new_state = state_machine.state
 
     return new_state
@@ -493,42 +504,63 @@ def process_exit_invalid_menu_option(display_key: str, preferred_language: str):
 
 
 def custom_display_text(
+        account: Account,
         display_key: str,
         menu_name: str,
-        ussd_session: dict,
-        user: Account) -> str:
+        session: Session,
+        ussd_session: dict) -> str:
     """This function extracts the appropriate session data based on the current menu name. It then inserts them as
     keywords in the i18n function.
+    :param account: The account in a running USSD session.
+    :type account: Account
     :param display_key: The path in the translation files defining an appropriate ussd response
     :type display_key: str
     :param menu_name: The name by which a specific menu can be identified.
     :type menu_name: str
-    :param user: The user in a running USSD session.
-    :type user: Account
+    :param session:
+    :type session:
     :param ussd_session: A JSON serialized in-memory ussd session object
     :type ussd_session: dict
     :return: A string value corresponding the ussd menu's text value.
     :rtype: str
     """
     if menu_name == 'transaction_pin_authorization':
-        return process_transaction_pin_authorization(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_transaction_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'exit_insufficient_balance':
-        return process_exit_insufficient_balance(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_exit_insufficient_balance(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'exit_successful_transaction':
-        return process_exit_successful_transaction(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_exit_successful_transaction(
+            account=account,
+            display_key=display_key,
+            session=session,
+            ussd_session=ussd_session)
     elif menu_name == 'start':
-        return process_start_menu(display_key=display_key, user=user)
+        return process_start_menu(display_key=display_key, user=account)
     elif 'pin_authorization' in menu_name:
-        return process_pin_authorization(display_key=display_key, user=user)
+        return process_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session)
     elif 'enter_current_pin' in menu_name:
-        return process_pin_authorization(display_key=display_key, user=user)
+        return process_pin_authorization(
+            account=account,
+            display_key=display_key,
+            session=session)
     elif menu_name == 'account_balances':
-        return process_account_balances(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_account_balances(display_key=display_key, user=account)
     elif 'transaction_set' in menu_name:
-        return process_account_statement(display_key=display_key, user=user, ussd_session=ussd_session)
+        return process_account_statement(display_key=display_key, user=account)
     elif menu_name == 'display_user_metadata':
-        return process_display_user_metadata(display_key=display_key, user=user)
+        return process_display_user_metadata(display_key=display_key, user=account)
     elif menu_name == 'exit_invalid_menu_option':
-        return process_exit_invalid_menu_option(display_key=display_key, preferred_language=user.preferred_language)
+        return process_exit_invalid_menu_option(display_key=display_key, preferred_language=account.preferred_language)
     else:
-        return translation_for(key=display_key, preferred_language=user.preferred_language)
+        return translation_for(key=display_key, preferred_language=account.preferred_language)

apps/cic-ussd/cic_ussd/requests.py

@@ -8,9 +8,12 @@ from urllib.parse import urlparse, parse_qs
 
 # third-party imports
 from sqlalchemy import desc
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.db.enum import AccountStatus
 from cic_ussd.operations import get_account_status, reset_pin
 from cic_ussd.validator import check_known_user
 
@@ -72,24 +75,26 @@ def get_account_creation_callback_request_data(env: dict) -> tuple:
     return status, task_id, result
 
 
-def process_pin_reset_requests(env: dict, phone_number: str):
+def process_pin_reset_requests(env: dict, phone_number: str, session: Session):
     """This function processes requests that are responsible for the pin reset functionality. It processes GET and PUT
     requests responsible for returning an account's status and
     :param env: A dictionary of values representing data sent on the api.
     :type env: dict
     :param phone_number: The phone of the user whose pin is being reset.
     :type phone_number: str
+    :param session:
+    :type session:
     :return: A response denoting the result of the request to reset the user's pin.
     :rtype: str
     """
-    if not check_known_user(phone=phone_number):
+    if not check_known_user(phone_number=phone_number, session=session):
         return f'No user matching {phone_number} was found.', '404 Not Found'
 
     if get_request_method(env) == 'PUT':
-        return reset_pin(phone_number=phone_number), '200 OK'
+        return reset_pin(phone_number=phone_number, session=session), '200 OK'
 
     if get_request_method(env) == 'GET':
-        status = get_account_status(phone_number=phone_number)
+        status = get_account_status(phone_number=phone_number, session=session)
         response = {
             'status': f'{status}'
         }
@@ -97,16 +102,18 @@ def process_pin_reset_requests(env: dict, phone_number: str):
         return response, '200 OK'
 
 
-def process_locked_accounts_requests(env: dict) -> tuple:
+def process_locked_accounts_requests(env: dict, session: Session) -> tuple:
     """This function authenticates staff requests and returns a serialized JSON formatted list of blockchain addresses
     of accounts for which the PIN has been locked due to too many failed attempts.
     :param env: A dictionary of values representing data sent on the api.
     :type env: dict
+    :param session:
+    :type session:
     :return: A tuple containing a serialized list of blockchain addresses for locked accounts and corresponding message
     for the response.
     :rtype: tuple
     """
-    logg.debug('Authentication requires integration with cic-auth')
+    session = SessionBase.bind_session(session=session)
     response = ''
 
     if get_request_method(env) == 'GET':
@@ -123,12 +130,14 @@ def process_locked_accounts_requests(env: dict) -> tuple:
             else:
                 limit = r[1]
 
-        locked_accounts = Account.session.query(Account.blockchain_address).filter(
+        locked_accounts = session.query(Account.blockchain_address).filter(
             Account.account_status == AccountStatus.LOCKED.value,
             Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all()
 
         # convert lists to scalar blockchain addresses
         locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts]
+
+        SessionBase.release_session(session=session)
         response = json.dumps(locked_accounts)
         return response, '200 OK'
     return response, '405 Play by the rules'

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py

@@ -36,11 +36,8 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 # set up db
 data_source_name = dsn_from_config(config)
 SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
 
 
-# handle requests from CICADA
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
     :param env: Object containing server and request information
@@ -55,19 +52,24 @@ def application(env, start_response):
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
     headers = [('Content-Type', 'text/plain')]
 
+    # create session for the life time of http request
+    session = SessionBase.create_session()
+
     if get_request_endpoint(env) == '/pin':
         phone_number = get_query_parameters(env=env, query_name='phoneNumber')
         phone_number = quote_plus(phone_number)
-        response, message = process_pin_reset_requests(env=env, phone_number=phone_number)
+        response, message = process_pin_reset_requests(env=env, phone_number=phone_number, session=session)
         response_bytes, headers = define_response_with_content(headers=errors_headers, response=response)
-        SessionBase.session.close()
+        session.commit()
+        session.close()
         start_response(message, headers)
         return [response_bytes]
 
     # handle requests for locked accounts
-    response, message = process_locked_accounts_requests(env=env)
+    response, message = process_locked_accounts_requests(env=env, session=session)
     response_bytes, headers = define_response_with_content(headers=headers, response=response)
     start_response(message, headers)
-    SessionBase.session.close()
+    session.commit()
+    session.close()
     return [response_bytes]
 

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -26,7 +26,7 @@ from cic_ussd.metadata.base import Metadata
 from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
-from cic_ussd.phone_number import process_phone_number, Support
+from cic_ussd.phone_number import process_phone_number, Support, E164Format
 from cic_ussd.processor import get_default_token_data
 from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
@@ -55,8 +55,6 @@ data_source_name = dsn_from_config(config)
 SessionBase.connect(data_source_name,
                     pool_size=int(config.get('DATABASE_POOL_SIZE')),
                     debug=config.true('DATABASE_DEBUG'))
-# create session for the life time of http request
-SessionBase.session = SessionBase.create_session()
 
 # set up translations
 i18n.load_path.append(config.get('APP_LOCALE_PATH'))
@@ -126,6 +124,7 @@ else:
 
 valid_service_codes = config.get('APP_SERVICE_CODE').split(",")
 
+E164Format.region = config.get('PHONE_NUMBER_REGION')
 Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
 
 
@@ -142,10 +141,13 @@ def application(env, start_response):
     errors_headers = [('Content-Type', 'text/plain'), ('Content-Length', '0')]
     headers = [('Content-Type', 'text/plain')]
 
+    # create session for the life time of http request
+    session = SessionBase.create_session()
+
     if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
 
         if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
-            start_response('405 Play by the rules', errors_headers)
+            start_response('405 Urlencoded, please', errors_headers)
             return []
 
         post_data = env.get('wsgi.input').read()
@@ -168,7 +170,7 @@ def application(env, start_response):
 
         # add validation for phone number
         if phone_number:
-            phone_number = process_phone_number(phone_number=phone_number, region=config.get('PHONE_NUMBER_REGION'))
+            phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region)
 
         # validate ip address
         if not check_ip(config=config, env=env):
@@ -205,14 +207,20 @@ def application(env, start_response):
                                                      phone_number=phone_number,
                                                      queue=args.q,
                                                      service_code=service_code,
+                                                     session=session,
                                                      user_input=user_input)
 
         response_bytes, headers = define_response_with_content(headers=headers, response=response)
         start_response('200 OK,', headers)
-        SessionBase.session.close()
+        session.commit()
+        session.close()
         return [response_bytes]
 
     else:
+        logg.error('invalid query {}'.format(env))
+        for r in env:
+            logg.debug('{}: {}'.format(r, env))
+        session.close()
         start_response('405 Play by the rules', errors_headers)
         return []
 

apps/cic-ussd/cic_ussd/state_machine/logic/balance.py

@@ -3,6 +3,7 @@ import logging
 from typing import Tuple
 
 # third-party imports
+from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.db.models.account import Account
@@ -10,11 +11,11 @@ from cic_ussd.db.models.account import Account
 logg = logging.getLogger(__file__)
 
 
-def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]):
+def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function compiles a brief statement of a user's last three inbound and outbound transactions and send the
     same as a message on their selected avenue for notification.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     logg.debug('This section requires integration with cic-eth. (The last 6 transactions would be sent as an sms.)')

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -16,7 +16,7 @@ def menu_one_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user input's match with '1'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '1'
 
 
@@ -27,7 +27,7 @@ def menu_two_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user input's match with '2'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '2'
 
 
@@ -38,7 +38,7 @@ def menu_three_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user input's match with '3'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '3'
 
 
@@ -50,7 +50,7 @@ def menu_four_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user input's match with '4'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '4'
 
 
@@ -62,10 +62,22 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user input's match with '5'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '5'
 
 
+def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
+    """
+    This function checks that user input matches a string with value '6'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '6'
+    :rtype: bool
+    """
+    user_input, ussd_session, user, session = state_machine_data
+    return user_input == '6'
+
+
 def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '00'
@@ -74,7 +86,7 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bo
     :return: A user input's match with '00'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '00'
 
 
@@ -86,5 +98,5 @@ def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account]) ->
     :return: A user input's match with '99'
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user_input == '99'

apps/cic-ussd/cic_ussd/state_machine/logic/pin.py

@@ -9,11 +9,13 @@ import re
 from typing import Tuple
 
 # third party imports
-import bcrypt
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.db.models.account import AccountStatus, Account
-from cic_ussd.encoder import PasswordEncoder, create_password_hash, check_password_hash
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.db.enum import AccountStatus
+from cic_ussd.encoder import create_password_hash, check_password_hash
 from cic_ussd.operations import persist_session_to_db_task, create_or_update_session
 from cic_ussd.redis import InMemoryStore
 
@@ -21,7 +23,7 @@ from cic_ussd.redis import InMemoryStore
 logg = logging.getLogger(__file__)
 
 
-def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_valid_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks a pin's validity by ensuring it has a length of for characters and the characters are
     numeric.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -29,7 +31,7 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A pin's validity
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     pin_is_valid = False
     matcher = r'^\d{4}$'
     if re.match(matcher, user_input):
@@ -37,34 +39,34 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
     return pin_is_valid
 
 
-def is_authorized_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_authorized_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user.verify_password(password=user_input)
 
 
-def is_locked_account(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_locked_account(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks whether a user's account is locked due to too many failed attempts.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user.get_account_status() == AccountStatus.LOCKED.name
 
 
-def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account]):
+def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function hashes a pin and stores it in session data.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
     # define redis cache entry point
     cache = InMemoryStore.cache
@@ -93,54 +95,56 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Accoun
         service_code=in_redis_ussd_session.get('service_code'),
         user_input=user_input,
         current_menu=in_redis_ussd_session.get('state'),
+        session=session,
         session_data=session_data
     )
     persist_session_to_db_task(external_session_id=external_session_id, queue='cic-ussd')
 
 
-def pins_match(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     initial_pin = ussd_session.get('session_data').get('initial_pin')
-    logg.debug(f'USSD SESSION:  {ussd_session}')
     return check_password_hash(user_input, initial_pin)
 
 
-def complete_pin_change(state_machine_data: Tuple[str, dict, Account]):
+def complete_pin_change(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function persists the user's pin to the database
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
+    session = SessionBase.bind_session(session=session)
     password_hash = ussd_session.get('session_data').get('initial_pin')
     user.password_hash = password_hash
-    Account.session.add(user)
-    Account.session.commit()
+    session.add(user)
+    session.flush()
+    SessionBase.release_session(session=session)
 
 
-def is_blocked_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_blocked_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks whether the user input confirming a specific pin matches the initial pin entered.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     return user.get_account_status() == AccountStatus.LOCKED.name
 
 
-def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks whether the user's new pin is a valid pin and that it isn't the same as the old one.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     :return: A match between two pin values.
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     is_old_pin = user.verify_password(password=user_input)
     return is_valid_pin(state_machine_data=state_machine_data) and not is_old_pin

apps/cic-ussd/cic_ussd/state_machine/logic/sms.py

@@ -9,15 +9,15 @@ logg = logging.getLogger()
 
 
 def send_terms_to_user_if_required(state_machine_data: Tuple[str, dict, Account]):
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     logg.debug('Requires integration to cic-notify.')
 
 
 def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]):
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     logg.debug('Requires integration to cic-notify.')
 
 
 def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account]):
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     logg.debug('Requires integration to cic-notify.')

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -5,13 +5,16 @@ from typing import Tuple
 
 # third party imports
 import celery
+from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.balance import BalanceManager, compute_operational_balance
+from cic_ussd.balance import compute_operational_balance
 from cic_ussd.chain import Chain
-from cic_ussd.db.models.account import AccountStatus, Account
+from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
+from cic_ussd.db.enum import AccountStatus
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
-from cic_ussd.phone_number import get_user_by_phone_number, process_phone_number
+from cic_ussd.phone_number import process_phone_number, E164Format
 from cic_ussd.processor import retrieve_token_symbol
 from cic_ussd.redis import create_cached_data_key, get_cached_data
 from cic_ussd.transactions import OutgoingTransactionProcessor
@@ -20,7 +23,7 @@ from cic_ussd.transactions import OutgoingTransactionProcessor
 logg = logging.getLogger(__file__)
 
 
-def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks that a user exists, is not the initiator of the transaction, has an active account status
     and is authorized to perform  standard transactions.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -28,14 +31,17 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :return: A user's validity
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
-    recipient = get_user_by_phone_number(phone_number=user_input)
-    is_not_initiator = process_phone_number(user_input, 'KE') != user.phone_number
+    user_input, ussd_session, user, session = state_machine_data
+    phone_number = process_phone_number(user_input, E164Format.region)
+    session = SessionBase.bind_session(session=session)
+    recipient = Account.get_by_phone_number(phone_number=phone_number, session=session)
+    SessionBase.release_session(session=session)
+    is_not_initiator = phone_number != user.phone_number
     has_active_account_status = user.get_account_status() == AccountStatus.ACTIVE.name
     return is_not_initiator and has_active_account_status and recipient is not None
 
 
-def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks that the transaction amount provided is valid as per the criteria for the transaction
     being attempted.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -43,14 +49,14 @@ def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account]) -
     :return: A transaction amount's validity
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     try:
         return int(user_input) > 0
     except ValueError:
         return False
 
 
-def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> bool:
+def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
     """This function checks that the transaction amount provided is valid as per the criteria for the transaction
     being attempted.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
@@ -58,10 +64,7 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo
     :return: An account balance's validity
     :rtype: bool
     """
-    user_input, ussd_session, user = state_machine_data
-    balance_manager = BalanceManager(address=user.blockchain_address,
-                                     chain_str=Chain.spec.__str__(),
-                                     token_symbol='SRF')
+    user_input, ussd_session, user, session = state_machine_data
     # get cached balance
     key = create_cached_data_key(
         identifier=bytes.fromhex(user.blockchain_address[2:]),
@@ -73,30 +76,37 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account]) -> boo
     return int(user_input) <= operational_balance
 
 
-def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account]):
+def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
     session_data = ussd_session.get('session_data') or {}
-    session_data['recipient_phone_number'] = user_input
+    recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    session_data['recipient_phone_number'] = recipient_phone_number
 
-    save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
+    save_to_in_memory_ussd_session_data(
+        queue='cic-ussd',
+        session=session,
+        session_data=session_data,
+        ussd_session=ussd_session)
 
 
-def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account]):
+def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account, Session]):
     """
     :param state_machine_data:
     :type state_machine_data:
     :return:
     :rtype:
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
-    recipient = get_user_by_phone_number(phone_number=user_input)
+    recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     blockchain_address = recipient.blockchain_address
+
     # retrieve and cache account's metadata
     s_query_person_metadata = celery.signature(
         'cic_ussd.tasks.metadata.query_person_metadata',
@@ -105,32 +115,36 @@ def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account]):
     s_query_person_metadata.apply_async(queue='cic-ussd')
 
 
-def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account]):
+def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
     session_data = ussd_session.get('session_data') or {}
     session_data['transaction_amount'] = user_input
 
-    save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
+    save_to_in_memory_ussd_session_data(
+        queue='cic-ussd',
+        session=session,
+        session_data=session_data,
+        ussd_session=ussd_session)
 
 
-def process_transaction_request(state_machine_data: Tuple[str, dict, Account]):
+def process_transaction_request(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function saves the phone number corresponding the intended recipients blockchain account.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
     # retrieve token symbol
     chain_str = Chain.spec.__str__()
 
     # get user from phone number
     recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number')
-    recipient = get_user_by_phone_number(phone_number=recipient_phone_number)
+    recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session)
     to_address = recipient.blockchain_address
     from_address = user.blockchain_address
     amount = int(ussd_session.get('session_data').get('transaction_amount'))

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -5,12 +5,14 @@ from typing import Tuple
 
 # third-party imports
 import celery
-from cic_types.models.person import Person, generate_metadata_pointer
+from cic_types.models.person import generate_metadata_pointer
 from cic_types.models.person import generate_vcard_from_contact_data, manage_identity_data
+from sqlalchemy.orm.session import Session
 
 # local imports
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 from cic_ussd.error import MetadataNotFoundError
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
@@ -19,37 +21,63 @@ from cic_ussd.redis import get_cached_data
 logg = logging.getLogger(__file__)
 
 
-def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account]):
+def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function changes the user's preferred language to english.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
+    session = SessionBase.bind_session(session=session)
     user.preferred_language = 'en'
-    Account.session.add(user)
-    Account.session.commit()
+    session.add(user)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+    preferences_data = {
+        'preferred_language': 'en'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [user.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
 
 
-def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account]):
+def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function changes the user's preferred language to swahili.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
-    user.preferred_language = 'sw'
-    Account.session.add(user)
-    Account.session.commit()
+    user_input, ussd_session, account, session = state_machine_data
+    session = SessionBase.bind_session(session=session)
+    account.preferred_language = 'sw'
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
+
+    preferences_data = {
+        'preferred_language': 'sw'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [account.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
 
 
-def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account]):
+def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function sets user's account to active.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
-    user.activate_account()
-    Account.session.add(user)
-    Account.session.commit()
+    user_input, ussd_session, account, session = state_machine_data
+    session = SessionBase.bind_session(session=session)
+    account.activate_account()
+    session.add(account)
+    session.flush()
+    SessionBase.release_session(session=session)
 
 
 def process_gender_user_input(user: Account, user_input: str):
@@ -61,6 +89,7 @@ def process_gender_user_input(user: Account, user_input: str):
     :return:
     :rtype:
     """
+    gender = ""
     if user.preferred_language == 'en':
         if user_input == '1':
             gender = 'Male'
@@ -78,13 +107,13 @@ def process_gender_user_input(user: Account, user_input: str):
     return gender
 
 
-def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account]):
+def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function saves first name data to the ussd session in the redis cache.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
-
+    user_input, ussd_session, user, session = state_machine_data
+    session = SessionBase.bind_session(session=session)
     # get current menu
     current_state = ussd_session.get('state')
 
@@ -93,6 +122,9 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict,
     if 'given_name' in current_state:
         key = 'given_name'
 
+    if 'date_of_birth' in current_state:
+        key = 'date_of_birth'
+
     if 'family_name' in current_state:
         key = 'family_name'
 
@@ -114,7 +146,11 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict,
         session_data = {
             key: user_input
         }
-    save_to_in_memory_ussd_session_data(queue='cic-ussd', session_data=session_data, ussd_session=ussd_session)
+    save_to_in_memory_ussd_session_data(
+        queue='cic-ussd',
+        session=session,
+        session_data=session_data,
+        ussd_session=ussd_session)
 
 
 def format_user_metadata(metadata: dict, user: Account):
@@ -130,6 +166,13 @@ def format_user_metadata(metadata: dict, user: Account):
     given_name = metadata.get('given_name')
     family_name = metadata.get('family_name')
 
+    if isinstance(metadata.get('date_of_birth'), dict):
+        date_of_birth = metadata.get('date_of_birth')
+    else:
+        date_of_birth = {
+            "year": int(metadata.get('date_of_birth')[:4])
+        }
+
     # check whether there's existing location data
     if isinstance(metadata.get('location'), dict):
         location = metadata.get('location')
@@ -154,6 +197,7 @@ def format_user_metadata(metadata: dict, user: Account):
     )
     return {
         "date_registered": date_registered,
+        "date_of_birth": date_of_birth,
         "gender": gender,
         "identities": identities,
         "location": location,
@@ -166,12 +210,12 @@ def format_user_metadata(metadata: dict, user: Account):
     }
 
 
-def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account]):
+def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]):
     """This function persists elements of the user metadata stored in session data
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
 
     # get session data
     metadata = ussd_session.get('session_data')
@@ -187,8 +231,8 @@ def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     s_create_person_metadata.apply_async(queue='cic-ussd')
 
 
-def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
-    user_input, ussd_session, user = state_machine_data
+def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, user, session = state_machine_data
     blockchain_address = user.blockchain_address
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
@@ -201,12 +245,12 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
 
     given_name = ussd_session.get('session_data').get('given_name')
     family_name = ussd_session.get('session_data').get('family_name')
+    date_of_birth = ussd_session.get('session_data').get('date_of_birth')
     gender = ussd_session.get('session_data').get('gender')
     location = ussd_session.get('session_data').get('location')
     products = ussd_session.get('session_data').get('products')
 
     # validate user metadata
-    person = Person()
     user_metadata = json.loads(user_metadata)
 
     # edit specific metadata attribute
@@ -214,6 +258,11 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
         user_metadata['given_name'] = given_name
     if family_name:
         user_metadata['family_name'] = family_name
+    if date_of_birth and len(date_of_birth) == 4:
+        year = int(date_of_birth[:4])
+        user_metadata['date_of_birth'] = {
+            'year': year
+        }
     if gender:
         user_metadata['gender'] = gender
     if location:
@@ -233,8 +282,8 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
     s_edit_person_metadata.apply_async(queue='cic-ussd')
 
 
-def get_user_metadata(state_machine_data: Tuple[str, dict, Account]):
-    user_input, ussd_session, user = state_machine_data
+def get_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]):
+    user_input, ussd_session, user, session = state_machine_data
     blockchain_address = user.blockchain_address
     s_get_user_metadata = celery.signature(
         'cic_ussd.tasks.metadata.query_person_metadata',

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -19,7 +19,7 @@ def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]):
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     # check for user metadata in cache
     key = generate_metadata_pointer(
         identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
@@ -34,7 +34,7 @@ def is_valid_name(state_machine_data: Tuple[str, dict, Account]):
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: str
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     name_matcher = "^[a-zA-Z]+$"
     valid_name = re.match(name_matcher, user_input)
     if valid_name:
@@ -50,9 +50,21 @@ def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]):
     :return:
     :rtype:
     """
-    user_input, ussd_session, user = state_machine_data
+    user_input, ussd_session, user, session = state_machine_data
     selection_matcher = "^[1-2]$"
     if re.match(selection_matcher, user_input):
         return True
     else:
         return False
+
+
+def is_valid_date(state_machine_data: Tuple[str, dict, Account]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, user, session = state_machine_data
+    # For MVP this value is defaulting to year
+    return len(user_input) == 4 and int(user_input) >= 1900

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -53,13 +53,25 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
             session.add(user)
             session.commit()
             session.close()
-    
+
             queue = self.request.delivery_info.get('routing_key')
-            s = celery.signature(
+
+            # add phone number metadata lookup
+            s_phone_pointer = celery.signature(
                     'cic_ussd.tasks.metadata.add_phone_pointer',
                     [result, phone_number]
                     )
-            s.apply_async(queue=queue)
+            s_phone_pointer.apply_async(queue=queue)
+
+            # add custom metadata tags
+            custom_metadata = {
+                "tags": ["ussd", "individual"]
+            }
+            s_custom_metadata = celery.signature(
+                'cic_ussd.tasks.metadata.add_custom_metadata',
+                [result, custom_metadata]
+            )
+            s_custom_metadata.apply_async(queue=queue)
 
             # expire cache
             cache.expire(task_id, timedelta(seconds=180))

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -7,8 +7,10 @@ from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
+from cic_ussd.metadata.custom import CustomMetadata
 from cic_ussd.metadata.person import PersonMetadata
 from cic_ussd.metadata.phone import PhonePointerMetadata
+from cic_ussd.metadata.preferences import PreferencesMetadata
 from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
@@ -44,7 +46,7 @@ def create_person_metadata(blockchain_address: str, data: dict):
 
 
 @celery_app.task
-def edit_person_metadata(blockchain_address: str, data: bytes):
+def edit_person_metadata(blockchain_address: str, data: dict):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.edit(data=data)
@@ -56,3 +58,17 @@ def add_phone_pointer(self, blockchain_address: str, phone_number: str):
     stripped_address = strip_0x(blockchain_address)
     phone_metadata_client = PhonePointerMetadata(identifier=identifier)
     phone_metadata_client.create(data=stripped_address)
+
+
+@celery_app.task()
+def add_custom_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = CustomMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)
+
+
+@celery_app.task()
+def add_preferences_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = PreferencesMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)

apps/cic-ussd/cic_ussd/validator.py

@@ -9,6 +9,7 @@ from confini import Config
 
 # local imports
 from cic_ussd.db.models.account import Account
+from cic_ussd.db.models.base import SessionBase
 
 logg = logging.getLogger(__file__)
 
@@ -45,29 +46,21 @@ def check_request_content_length(config: Config, env: dict):
         config.get('APP_MAX_BODY_LENGTH'))
 
 
-def check_known_user(phone: str):
-    """
-    This method attempts to ascertain whether the user already exists and is known to the system.
+def check_known_user(phone_number: str, session):
+    """This method attempts to ascertain whether the user already exists and is known to the system.
     It sends a get request to the platform application and attempts to retrieve the user's data which it persists in
     memory.
-    :param phone: A valid phone number
-    :type phone: str
+    :param phone_number: A valid phone number
+    :type phone_number: str
+    :param session:
+    :type session:
     :return: Is known phone number
     :rtype: boolean
     """
-    user = Account.session.query(Account).filter_by(phone_number=phone).first()
-    return user is not None
-
-
-def check_phone_number(number: str):
-    """
-    Checks whether phone number is present
-    :param number: A valid phone number
-    :type number: str
-    :return: Phone number presence
-    :rtype: boolean
-    """
-    return number is not None
+    session = SessionBase.bind_session(session=session)
+    account = session.query(Account).filter_by(phone_number=phone_number).first()
+    SessionBase.release_session(session=session)
+    return account is not None
 
 
 def check_request_method(env: dict):

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b15
-cic-eth~=0.11.0b16
-cic-notify~=0.4.0a5
-cic-types~=0.1.0a10
+cic_base[full_graph]==0.1.3a3+build.984b5cff
+cic-eth~=0.11.1a3
+cic-notify~=0.4.0a7
+cic-types~=0.1.0a11

apps/cic-ussd/states/account_management_states.json

@@ -7,6 +7,7 @@
   "new_pin_confirmation",
   "display_user_metadata",
   "name_edit_pin_authorization",
+  "dob_edit_pin_authorization",
   "gender_edit_pin_authorization",
   "location_edit_pin_authorization",
   "products_edit_pin_authorization",

apps/cic-ussd/states/user_metadata_states.json

@@ -5,5 +5,6 @@
   "enter_age",
   "enter_location",
   "enter_products",
+  "enter_date_of_birth",
   "display_metadata_pin_authorization"
 ]

apps/cic-ussd/test_requirements.txt

@@ -8,4 +8,4 @@ pytest-mock==3.3.1
 pytest-ordering==0.6
 pytest-redis==2.0.0
 requests-mock==1.8.0
-tavern==1.14.2
+tavern==1.14.2

apps/cic-ussd/tests/fixtures/integration.py

@@ -124,46 +124,6 @@ def second_profile_management_session_id() -> str:
     return session_id()
 
 
-@pytest.fixture(scope='session')
-def first_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_given_name() -> str:
-    return fake.first_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_family_name() -> str:
-    return fake.last_name()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_location() -> str:
-    return fake.city()
-
-
-@pytest.fixture(scope='session')
-def first_account_change_product() -> str:
-    return fake.color_name()
-
-
-@pytest.fixture(scope='session')
-def second_account_change_product() -> str:
-    return fake.color_name()
-
-
 @pytest.fixture(scope='session')
 def first_profile_management_session_id_1() -> str:
     return session_id()

apps/cic-ussd/tests/integration/README.md

@@ -0,0 +1,25 @@
+# INTEGRATION TESTING
+
+This folder contains integration tests.
+
+## OVERVIEW
+
+There are four files defining the integration tests.
+
+* **test_account_creation**: Tests account sign up process.
+* **test_transactions**: Tests transactions between two accounts.
+* **test_profile_management**: Tests that account metadata can be edited.
+* **test_account_management**: Tests that account management functionalities are intact.
+
+## REQUIREMENTS
+
+In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000`
+which is the value set in the config file `.config/test/integration.ini`.
+
+This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image:
+
+```shell
+export DEV_FAUCET_AMOUNT=50000000
+RUN_MASK=1 docker-compose up contract-migration
+RUN_MASK=2 docker-compose up contract-migration
+```

apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml

@@ -214,12 +214,13 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '28'
+        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Enter first name\n0. Back"
+          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
+    delay_before: 10
 
   - name: Pin number confirmation [{second_account_pin_number} - second account]
     request:
@@ -232,227 +233,6 @@ stages:
       headers:
         content-type: "application/x-www-form-urlencoded"
       method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la kwanza\n0. Nyuma"
-
-  - name: Enter first name [first_account_given_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '29'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter family name\n0. Back"
-
-  - name: Enter first name [second_account_given_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '37'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jina lako la mwisho\n0. Nyuma"
-
-  - name: Enter last name [first_account_family_name - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter gender\n1. Male\n2. Female\n3. Other\n0. Back"
-
-  - name: Enter last name [second_account_family_name - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '64'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka jinsia yako\n1. Mwanaume\n2. Mwanamke\n3. Nyngine\n0. Nyuma"
-
-  - name: Select gender [Male - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '31'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Enter your location\n0. Back"
-
-  - name: Select gender [Female - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '27'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka eneo lako\n0. Nyuma"
-
-  - name: Enter location [first_account_location - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '55'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Please enter a product or service you offer\n0. Back"
-
-  - name: Enter location [second_account_location - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '42'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Weka bidhaa ama huduma unauza\n0. Nyuma"
-
-  - name: Enter product [first_account_product - first account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{first_metadata_entry_session_id}"
-        phoneNumber: "{first_account_phone_number}"
-        text: "1*{first_account_pin_number}*{first_account_pin_number}*{first_account_given_name}*{first_account_family_name}*1*{first_account_location}*{first_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
-    response:
-      status_code:
-        - 200
-      headers:
-        Content-Length: '51'
-        Content-Type: "text/plain"
-      verify_response_with:
-        function: ext.validator:validate_response
-        extra_kwargs:
-          expected_response: "CON Balance {gift_value} {token_symbol}\n1. Send\n2. My Account\n3. Help"
-    delay_before: 10
-
-  - name: Enter product [second_account_product - second account]
-    request:
-      url: "{server_url}"
-      data:
-        serviceCode: "*483*46#"
-        sessionId: "{second_metadata_entry_session_id}"
-        phoneNumber: "{second_account_phone_number}"
-        text: "2*{second_account_pin_number}*{second_account_pin_number}*{second_account_given_name}*{second_account_family_name}*2*{second_account_location}*{second_account_product}"
-      headers:
-        content-type: "application/x-www-form-urlencoded"
-      method: POST
     response:
       status_code:
         - 200

apps/cic-ussd/tests/integration/test_account_management.tavern.yaml

@@ -31,7 +31,6 @@ stages:
       status_code:
         - 200
       headers:
-        Content-Length: '51'
         Content-Type: "text/plain"
       verify_response_with:
         function: ext.validator:validate_response

apps/cic-ussd/tests/integration/test_transactions.tavern.yaml

@@ -170,7 +170,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
+          expected_response: "CON {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\nPlease enter your PIN to confirm.\n0. Back"
 
   - name: Enter transcation amount [second account]
     request:
@@ -191,7 +191,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
+          expected_response: "CON {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\nTafadhali weka nambari yako ya siri kudhibitisha.\n0. Nyuma"
 
   - name: Pin to authorize transaction [first account]
     request:
@@ -212,7 +212,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Your request has been sent. {second_account_given_name} {second_account_family_name} {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_given_name} {first_account_family_name} {first_account_phone_number}.\n00. Back\n99. Exit"
+          expected_response: "CON Your request has been sent. {second_account_phone_number} will receive 17.00 {token_symbol} from {first_account_phone_number}.\n00. Back\n99. Exit"
 
   - name: Pin to authorize transaction [second account]
     request:
@@ -233,7 +233,7 @@ stages:
       verify_response_with:
         function: ext.validator:validate_response
         extra_kwargs:
-          expected_response: "CON Ombi lako limetumwa. {first_account_given_name} {first_account_family_name} {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_given_name} {second_account_family_name} {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
+          expected_response: "CON Ombi lako limetumwa. {first_account_phone_number} atapokea 25.00 {token_symbol} kutoka kwa {second_account_phone_number}.\n00. Nyuma\n99. Ondoka"
 
   - name: Verify balance changes [first account]
     delay_before: 10

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -0,0 +1,39 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "dob_edit_pin_authorization",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "conditions": [
+      "cic_ussd.state_machine.logic.validator.has_cached_user_metadata",
+      "cic_ussd.state_machine.logic.validator.is_valid_date"
+    ]
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "enter_location",
+    "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "exit_invalid_input",
+    "unless": "cic_ussd.state_machine.logic.validator.is_valid_date"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin",
+    "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  }
+]

apps/cic-ussd/transitions/gender_setting_transitions.json

@@ -2,7 +2,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_gender",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"

apps/cic-ussd/transitions/user_metadata_transitions.json

@@ -14,21 +14,27 @@
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_products",
+    "dest": "enter_location",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_four_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "display_metadata_pin_authorization",
+    "dest": "enter_products",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected"
   },
+  {
+    "trigger": "scan_data",
+    "source": "metadata_management",
+    "dest": "display_metadata_pin_authorization",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_six_selected"
+  },
   {
     "trigger": "scan_data",
     "source": "display_metadata_pin_authorization",

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -17,6 +17,9 @@ en:
     enter_family_name: |-
       CON Enter family name
       0. Back
+    enter_date_of_birth: |-
+      CON Enter year of birth
+      0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
@@ -52,14 +55,16 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit location
-      4. Edit products
-      5. View my profile
+      3. Edit age
+      4. Edit location
+      5. Edit products
+      6. View my profile
       0. Back
     display_user_metadata: |-
       CON Your details are:
         Name: %{full_name}
         Gender: %{gender}
+        Age: %{age}
         Location: %{location}
         You sell: %{products}
       0. Back
@@ -117,6 +122,13 @@ en:
       retry: |-
         CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
+    dob_edit_pin_authorization:
+      first: |-
+        CON Please enter your PIN
+        0. Back
+      retry: |-
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
+        0. Back
     gender_edit_pin_authorization:
       first: |-
         CON Please enter your PIN

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -17,6 +17,9 @@ sw:
     enter_family_name: |-
       CON Weka jina lako la mwisho
       0. Nyuma
+    enter_date_of_birth: |-
+      CON Weka mwaka wa kuzaliwa
+      0. Nyuma
     enter_gender: |-
       CON Weka jinsia yako
       1. Mwanaume
@@ -52,14 +55,16 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3. Weka eneo
-      4. Weka bidhaa
-      5. Angalia wasifu wako
+      3. Weka umri
+      4. Weka eneo
+      5. Weka bidhaa
+      6. Angalia wasifu wako
       0. Nyuma
     display_user_metadata: |-
       CON Wasifu wako una maelezo yafuatayo:
         Jina: %{full_name}
         Jinsia: %{gender}
+        Umri: %{age}
         Eneo: %{location}
         Unauza: %{products}
       0. Nyuma
@@ -117,6 +122,13 @@ sw:
       retry: |-
         CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
         0. Nyuma
+    dob_edit_pin_authorization:
+      first: |-
+        CON Tafadhali weka PIN yako
+        0. Nyuma
+      retry: |-
+        CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
+        0. Nyuma
     gender_edit_pin_authorization:
       first: |-
         CON Tafadhali weka PIN yako

apps/contract-migration/requirements.txt

@@ -1,4 +1,4 @@
-	cic-base[full_graph]==0.1.2b15
+	cic-base==0.1.2b15
 	sarafu-faucet==0.0.3a3
 	sarafu-token==0.0.1a8
 	cic-eth==0.11.0b16 

apps/contract-migration/reset.sh

@@ -76,6 +76,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	>&2 echo "add deployer address as account index writer"
 	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
+	>&2 echo "deploy contract registry contract"
 	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
@@ -87,7 +88,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
-	>&2 echo "deploy address declarator contract"
+	>&2 echo "deploy transfer auth contract"
 	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 

apps/data-seeding/README.md

@@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything:
 
 To import, run to _completion_:
 
-`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
+`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
 
 After the script completes, keystore files for all generated accouts will be found in `<datadir>/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one).
 
@@ -150,7 +150,7 @@ Then run:
 
 Run in sequence, in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
 
 In another terminal:
 
@@ -171,7 +171,11 @@ Then, in sequence, run in first terminal:
 
 In second terminal:
 
-`python cic_ussd/import_users.py -v -c config out`
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> -c config out`
+
+In the event that you are running the command in a local environment you may want to consider passing the `--ussd-no-ssl` flag i.e:
+
+`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
 
 
 
@@ -199,6 +203,13 @@ If _number of users_ is omitted the script will run until manually interrupted.
 If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
 
 
+### Importing preferences metadata
+
+`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
+
+If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
+
+
 ##### Importing pins and ussd data (optional)
 
 Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
@@ -226,7 +237,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in
 
 ### Step 5 - Verify
 
-`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>` 
+`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>` 
 
 Included checks:
 * Private key is in cic-eth keystore
@@ -262,3 +273,5 @@ Should exit with code 0 if all input data is found in the respective services.
 - MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953
 
 - `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it.
+
+- A strict constraint is maintained insistin the use of postgresql-12.

apps/data-seeding/cic_eth/import_balance.py

@@ -18,19 +18,17 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
-        block_by_number,
-        Block,
         )
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.jsonrpc import jsonrpc_template
+from chainlib.jsonrpc import JSONRPCRequest
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from chainlib.eth.constant import ZERO_ADDRESS
@@ -38,6 +36,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+from cic_base.eth.syncer import chain_interface
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -70,12 +69,14 @@ elif args.vv == True:
 config_dir = os.path.join(args.c)
 os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
-config.process()
 # override args
+config.process()
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'y'),
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -184,27 +185,6 @@ class Handler:
         #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
 
 
-#class BlockGetter:
-#
-#    def __init__(self, conn, gas_oracle, nonce_oracle, chain_spec):
-#        self.conn = conn
-#        self.tx_factory = ERC20(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
-#
-#
-#    def get(self, n):
-#        o = block_by_number(n)
-#        r = self.conn.do(o)
-#        b = None
-#        try:
-#            b = Block(r)
-#        except TypeError as e:
-#            if r == None:
-#                logg.debug('block not found {}'.format(n))
-#            else:
-#                logg.error('block retrieve error {}'.format(e))
-#        return b
-
-
 def progress_callback(block_number, tx_index):
     sys.stdout.write(str(block_number).ljust(200) + "\n")
 
@@ -225,11 +205,13 @@ def main():
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
     txf.set_code(tx, data)
-    
-    o = jsonrpc_template()
+   
+    j = JSONRPCRequest()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found token index address {}'.format(token_index_address))
@@ -243,10 +225,11 @@ def main():
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)
-    o = jsonrpc_template()
+    o = j.template()
     o['method'] = 'eth_call'
     o['params'].append(txf.normalize(tx))
     o['params'].append('latest')
+    o = j.finalize(o)
     r = conn.do(o)
     try:
         sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
@@ -304,7 +287,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/cic_eth/import_users.py

@@ -194,8 +194,7 @@ if __name__ == '__main__':
             f.write(json.dumps(o))
             f.close()
 
-            #fi.write('{},{}\n'.format(new_address, old_address))
-            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
             meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
             os.symlink(os.path.realpath(filepath), meta_filepath)
 
@@ -221,7 +220,7 @@ if __name__ == '__main__':
 
 
             # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom')
             custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
 
             filepath = os.path.join(

apps/data-seeding/cic_meta/import_meta.js

@@ -31,13 +31,16 @@ function sendit(uid, envelope) {
 	const req = http.request(url + uid, opts, (res) => {
 		res.on('data', process.stdout.write);
 		res.on('end', () => {
+        if (!res.complete) {
+            console.log('The connection was terminated while the message was being sent.')
+        }
 			console.log('result', res.statusCode, res.headers);
 		});
 	});
-	if (!req.write(d)) {
-		console.error('foo', d);
-		process.exit(1);
-	}
+  req.on('error', (err) => {
+      console.log('ERROR when talking to meta', err)
+  })
+  req.write(d)
 	req.end();
 }
 
@@ -55,6 +58,7 @@ function doOne(keystore, filePath) {
 		const s = new crdt.Syncable(uid, o);
 		s.setSigner(signer);
 		s.onwrap = (env) => {
+      console.log(`Sending uid: ${uid} and env: ${env} to meta`)
 			sendit(uid, env);
 		};
 		s.sign();
@@ -84,6 +88,7 @@ let batchCount = 0;
 
 
 function importMeta(keystore) {
+  console.log('Running importMeta....')
 	let err;
 	let files;
 
@@ -94,6 +99,11 @@ function importMeta(keystore) {
 		setTimeout(importMeta, batchDelay, keystore);
 		return;
 	}
+  console.log(`Trying to read ${files.length} files`)
+  if (files === 0) {
+    console.log(`ERROR did not find any files under ${workDir}. \nLooks like there is no work for me, bailing!`)
+    process.exit(1)
+  }
 	let limit = batchSize;
 	if (files.length < limit) {
 		limit = files.length;
@@ -108,6 +118,7 @@ function importMeta(keystore) {
 		doOne(keystore, filePath);
 		count++;
 		batchCount++;
+    //console.log('done one', count, batchCount)
 		if (batchCount == batchSize) {
 			console.debug('reached batch size, breathing');
 			batchCount=0;