Correct check test chainspec var

.gitignore

@@ -14,4 +14,3 @@ build/
 **/.venv
 .idea
 **/.vim
-**/*secret.yaml

.gitlab-ci.yml

@@ -1,67 +1,14 @@
 include:
-  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
+  - local: 'ci_templates/.cic-template.yml'
-  # these includes are app specific unit tests
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-    #- local: 'apps/contract-migration/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
-    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - deploy 
+  - release
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest  
-
-variables:
-  DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1" 
-  CI_DEBUG_TRACE: "true"
-  TAG: $CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-
-     
-# runs on protected branches and pushes to repo
-build-push:
-  stage: build
-  tags:
-    - integration
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  script:
-    - TAG=$TAG sh ./scripts/build-push.sh
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-
-deploy-k8s-dev:
-  stage: deploy
-  image: line/kubectl-kustomize
-  variables:
-    CI_DEBUG_TRACE: "true"
-  script:
-    - kubectl config set-cluster k8s --server="${K8S_DEV_SERVER?dev server missing}"
-    - kubectl config set clusters.k8s.certificate-authority-data ${K8S_DEV_CERTIFICATE_AUTHORITY_DATA}
-    - kubectl config set-credentials gitlab --token="${K8S_DEV_USER_TOKEN}"
-    - kubectl config set-context grassroots --cluster=k8s --user=gitlab --namespace grassroots
-    - kubectl config use-context grassroots  
-      #- sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}/g" deployment.yaml
-      #- kubectl apply -f deployment.yaml
-    - echo "Wiping state..."
-    - kubectl delete jobs.batch --all 
-    - kubectl delete hr postgresql && kubectl delete pvc -l 'app.kubernetes.io/name=postgresql'
-    - kubectl delete sts,pvc -l 'app=bloxberg-validator'
-    - kubectl delete hr redis && kubectl delete pvc -l 'app=redis'
-    - kubectl apply -f kubernetes/eth-node/ -f kubernetes/postgresql/ -f kubernetes/redis/
-    - echo "deploy and run database migrations..."
-      # set image based on deploy tag
-    - bash ./scripts/set-image.sh
-    - kubectl apply -f . 
-    - echo "run contract migrations..."
-    - kubectl apply -f kubernetes/contract-migration/contract-migration-job.yaml
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-

apps/cic-base-os/Dockerfile

@@ -0,0 +1,34 @@
+# The solc image messes up the alpine environment, so we have to go all over again
+FROM python:3.8.6-slim-buster
+
+LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
+LABEL spdx-license-identifier="GPL-3.0-or-later"
+LABEL description="Base layer for buiding development images for the cic component suite"
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
+
+
+RUN echo installing nodejs tooling
+
+COPY ./dev/nvm.sh /root/
+
+# Install nvm with node and npm
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
+ENV NVM_DIR /root/.nvm
+ENV NODE_VERSION 15.3.0
+ENV BANCOR_NODE_VERSION 10.16.0
+
+RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
+	&& . $NVM_DIR/nvm.sh \
+	&& nvm install $NODE_VERSION \
+	&& nvm alias default $NODE_VERSION \
+	&& nvm use $NODE_VERSION \
+# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
+# owner of these files is "1001" by default - wtf
+	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
+

apps/cic-base-os/README.md

@@ -0,0 +1 @@
+## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-cache/.gitlab-ci.yml

@@ -1,17 +1,52 @@
-build-test-cic-cache:
+.cic_cache_variables:
-  stage: test 
+    variables:
-  tags:
+        APP_NAME: cic-cache
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-cache
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-cache:
-  script:
+    extends:
-    - cd apps/cic-cache
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_cache_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  rules:
+      changes:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+          - apps/cic-cache/**/*
-    changes:
+      when: always
-        - apps/$APP_NAME/**/*
+
-    when: always
+test-mr-cic-cache:
+    stage: test
+    extends:
+        - .cic_cache_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-cache"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-cache:
+    extends:
+        - .py_build_push
+        - .cic_cache_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-cache/**/*
+      when: always
+        
+

apps/cic-cache/README.md

@@ -1 +0,0 @@
-# CIC-CACHE

apps/cic-cache/cic_cache/api.py

@@ -55,37 +55,15 @@ class Api:
                     queue=callback_queue,
                     )       
 
-    def list(self, offset=0, limit=100, address=None, oldest=False):
+    def list(self, offset, limit, address=None):
         s = celery.signature(
         'cic_cache.tasks.tx.tx_filter',
         [
-            offset,
+            0,
-            limit,
+            100,
             address,
-            oldest,
             ],
-            queue=self.queue,
+            queue=None
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t
-
-
-    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter_content',
-        [
-            offset,
-            limit,
-            address,
-            block_offset,
-            block_limit,
-            oldest,
-            ],
-            queue=self.queue,
         )
         if self.callback_param != None:
             s.link(self.callback_success).on_error(self.callback_error)

apps/cic-cache/cic_cache/cache.py

@@ -10,16 +10,12 @@ from cic_cache.db.list import (
         list_transactions_mined,
         list_transactions_account_mined,
         list_transactions_mined_with_data,
-        list_transactions_mined_with_data_index,
-        list_transactions_account_mined_with_data_index,
-        list_transactions_account_mined_with_data,
         )
 
 logg = logging.getLogger()
 
 
 DEFAULT_FILTER_SIZE = 8192 * 8
-DEFAULT_LIMIT = 100
 
 class Cache:
 
@@ -36,7 +32,7 @@ class BloomCache(Cache):
         return n
 
 
-    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions(self, offset, limit):
         """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
 
         Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -53,7 +49,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+        rows = list_transactions_mined(self.session, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -62,12 +58,7 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
+            lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -76,7 +67,7 @@ class BloomCache(Cache):
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
 
 
-    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions_account(self, address, offset, limit):
         """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
 
         :param address: Address to retrieve transactions for.
@@ -88,7 +79,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -97,12 +88,7 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
+            lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -113,21 +99,8 @@ class BloomCache(Cache):
 
 class DataCache(Cache):
 
-    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions_with_data(self, offset, end):
-        if limit == 0:
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def __process_rows(self, rows, oldest):
         tx_cache = []
         highest_block = -1;
         lowest_block = -1;
@@ -135,12 +108,7 @@ class DataCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r['block_number']
-                lowest_block = r['block_number']
+            lowest_block = r['block_number']
-            else:
-                if oldest:
-                    highest_block = r['block_number']
-                else:
-                    lowest_block = r['block_number']
             tx_type = 'unknown'
 
             if r['value'] != None:

apps/cic-cache/cic_cache/cli/arg.py

@@ -12,7 +12,7 @@ class ArgumentParser(BaseArgumentParser):
 
     def process_local_flags(self, local_arg_flags):
         if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
             self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -1,5 +1,5 @@
 [celery]
 broker_url = redis://localhost:6379
 result_url = 
-queue = cic-cache
+queue = cic-eth
 debug = 0

apps/cic-cache/cic_cache/db/list.py

@@ -13,9 +13,6 @@ def list_transactions_mined(
         session,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -26,62 +23,15 @@ def list_transactions_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    order_by = 'DESC'
+    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
     r = session.execute(s)
     return r
 
 
 def list_transactions_mined_with_data(
-        session,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
-
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data_index(
         session,
         offset,
         end,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -92,87 +42,7 @@ def list_transactions_mined_with_data_index(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined_with_data_index(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
-    r = session.execute(s)
-    return r
-
-def list_transactions_account_mined_with_data(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
 
     r = session.execute(s)
     return r
@@ -183,9 +53,6 @@ def list_transactions_account_mined(
         address,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
 
@@ -198,20 +65,7 @@ def list_transactions_account_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-
+    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-
-    else:
-        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
     r = session.execute(s)
     return r
 

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -91,14 +91,13 @@ def process_transactions_all_data(session, env):
     if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
         return None
 
-    logg.debug('got data request {}'.format(env))
+    offset = r[1]
-    block_offset = r[1]
+    end = r[2]
-    block_end = r[2]
     if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()

apps/cic-cache/cic_cache/tasks/tx.py

@@ -2,17 +2,14 @@
 import celery
 
 # local imports
-from cic_cache.cache import (
+from cic_cache.cache import BloomCache
-        BloomCache,
-        DataCache,
-        )
 from cic_cache.db.models.base import SessionBase
 
 celery_app = celery.current_app
 
 
 @celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, encoding='hex'):
     queue = self.request.delivery_info.get('routing_key')
 
     session = SessionBase.create_session()
@@ -20,9 +17,9 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     c = BloomCache(session)
     b = None
     if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
     else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
 
     session.close()
 
@@ -38,17 +35,4 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     return o
 
 
-@celery_app.task(bind=True)
-def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
-    session = SessionBase.create_session()
 
-    c = DataCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
-    else:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
-
-    session.close()
-
-    return (lowest_block, highest_block, tx_cache,)

apps/cic-cache/cic_cache/version.py

@@ -4,7 +4,7 @@ import semver
 version = (
         0,
         2,
-        1,
+        0,
         'alpha.2',
         )
 

apps/cic-cache/docker/Dockerfile

@@ -10,10 +10,9 @@ COPY requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt
 
 COPY . . 

apps/cic-cache/docker/Dockerfile_ci

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+ 
+# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
+
+COPY requirements.txt .
+#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+#RUN pip install $pip_extra_index_url_flag .
+#RUN pip install .[server]
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . . 
+
+RUN python setup.py install
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-cache/
+
+# for db migrations
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+COPY /docker/start_tracker.sh ./start_tracker.sh
+COPY /docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
+# Tracker
+# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
+# Server
+# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
+ENTRYPOINT []

apps/cic-cache/docker/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-

apps/cic-cache/requirements.txt

@@ -2,14 +2,13 @@ alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a1
+cic-eth-registry~=0.5.8a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]>=0.0.6a3,<0.1.0
+chainsyncer[sql]>=0.0.6a1,<0.1.0
-erc20-faucet>=0.3.2a1, <0.4.0
+erc20-faucet~=0.2.4a1
-chainlib-eth>=0.0.9a7,<0.1.0
+chainlib>=0.0.7a1,<0.1.0
-chainlib>=0.0.9a3,<0.1.0
+eth-address-index>=0.1.4a1,<0.2.0
-eth-address-index>=0.2.3a1,<0.3.0

apps/cic-cache/setup.cfg

@@ -41,4 +41,3 @@ console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
-	cic-cache-list = cic_cache.runable.list:main

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.7a1
+sarafu-faucet~=0.0.5a2
-erc20-transfer-authorization>=0.3.5a1,<0.4.0
+erc20-transfer-authorization>=0.3.4a1,<0.4.0

apps/cic-cache/tests/cli/test_cli_args.py

@@ -1,40 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_cache.cli
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
-
-
-def test_argumentparserto_config():
-
-    argparser = cic_cache.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-

apps/cic-cache/tests/cli/test_cli_celery.py

@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_cache.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_cache.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_cache.cli.CeleryApp.from_config(config)

apps/cic-cache/tests/cli/test_cli_chain.py

@@ -1,68 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-from chainlib.eth.pytest.fixtures_chain import default_chain_spec
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from hexathon import add_0x
-
-# local imports
-import cic_cache.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        ):
-    ifc = cic_cache.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    recipient = add_0x(os.urandom(20).hex())
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-

apps/cic-cache/tests/conftest.py

@@ -64,6 +64,7 @@ def txs(
         dt.timestamp(),
             )
 
+
     tx_number = 42
     tx_hash_second = '0x' + os.urandom(32).hex()
     tx_signed_second = '0x' + os.urandom(128).hex()
@@ -92,44 +93,6 @@ def txs(
             ]
 
 
-@pytest.fixture(scope='function')
-def more_txs(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        ):
-
-    session = init_database
-
-    tx_number = 666
-    tx_hash = '0x' + os.urandom(32).hex()
-    tx_signed = '0x' + os.urandom(128).hex()
-    nonce = 3
-
-    dt = datetime.datetime.utcnow()
-    dt += datetime.timedelta(hours=1)
-    db.add_transaction(
-        session,
-        tx_hash,
-        list_defaults['block']+2,
-        tx_number,
-        list_actors['alice'],
-        list_actors['diane'],
-        list_tokens['bar'],
-        list_tokens['bar'],
-        2048,
-        4096,
-        False,
-        dt.timestamp(),
-        )
-
-    session.commit()
-
-    return [tx_hash] + txs
-
-
 @pytest.fixture(scope='function')
 def tag_txs(
         init_database,

apps/cic-cache/tests/test_cache.py

@@ -8,7 +8,6 @@ import json
 import pytest
 
 # local imports
-from cic_cache import db
 from cic_cache import BloomCache
 from cic_cache.cache import DataCache
 
@@ -19,6 +18,7 @@ def test_cache(
         init_database,
         list_defaults,
         list_actors,
+        list_tokens,
         txs,
         ):
 
@@ -37,6 +37,9 @@ def test_cache(
 
 def test_cache_data(
         init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
         txs,
         tag_txs,
         ):
@@ -44,209 +47,10 @@ def test_cache_data(
     session = init_database
 
     c = DataCache(session)
-    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
+    b = c.load_transactions_with_data(410000, 420000)
 
     assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[0]
+    assert b[2][0]['tx_hash'] == txs[1]
-    assert b[2][0]['tx_type'] == 'unknown'
+    assert b[2][1]['tx_type'] == 'unknown'
-    assert b[2][1]['tx_type'] == 'test.taag'
+    assert b[2][0]['tx_type'] == 'test.taag'
-
-
-def test_cache_ranges(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = BloomCache(session)
-    b = c.load_transactions(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    # now check when supplying account
-    b = c.load_transactions_account(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    # add block filter to the mix
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
     
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-
-
-def test_cache_ranges_data( 
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = DataCache(session)
-
-    b = c.load_transactions_with_data(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    # now check when supplying account
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    # add block filter to the mix
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[2]

apps/cic-cache/tests/testdata/config/test.ini

@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy

apps/cic-eth/.gitlab-ci.yml

@@ -1,16 +1,52 @@
-build-test-cic-eth:
+.cic_eth_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-eth
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-eth
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-eth:
-  script:
+    extends:
-    - cd apps/cic-eth
+        - .cic_eth_variables
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .py_build_target_dev
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-      #rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
-      #  changes:
+          - apps/cic-eth/**/*
-      #      - apps/$APP_NAME/**/*
+      when: always
-      #  when: always
+
+test-mr-cic-eth:
+    stage: test
+    extends:
+        - .cic_eth_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r admin_requirements.txt
+          -r services_requirements.txt
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-eth"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-eth/**/*
+      when: always
+
+build-push-cic-eth:
+    extends:
+        - .py_build_push
+        - .cic_eth_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-eth/**/*
+      when: always

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.5.6a2,<0.6.0
-hexathon~=0.0.1a8
+hexathon~=0.0.1a7
-chainqueue>=0.0.4a6,<0.1.0
+chainqueue>=0.0.3a1,<0.1.0
-eth-erc20>=0.1.2a2,<0.2.0
+eth-erc20>=0.0.10a3,<0.1.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -4,6 +4,7 @@ import logging
 
 # external imports
 import celery
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from hexathon import (
         add_0x,
@@ -19,17 +20,18 @@ from cic_eth.task import (
         CriticalSQLAlchemyTask,
         )
 from cic_eth.error import LockedError
-from cic_eth.encode import (
-        tx_normalize,
-        ZERO_ADDRESS_NORMAL,
-        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+def normalize_address(a):
+    if a == None:
+        return None
+    return add_0x(hex_uniform(strip_0x(a)))
+
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
+def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -41,7 +43,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -51,7 +53,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
+def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -63,7 +65,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
@@ -73,7 +75,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
+def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -83,7 +85,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
@@ -91,7 +93,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
+def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -101,7 +103,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
@@ -109,7 +111,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
+def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -119,7 +121,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
@@ -127,7 +129,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
+def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -137,7 +139,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
     :returns: New lock state for address
     :rtype: number
     """
-    address = tx_normalize.wallet_address(address)
+    address = normalize_address(address)
     chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
@@ -146,13 +148,12 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    if address != None:
+    address = normalize_address(address)
-        address = tx_normalize.wallet_address(address)
     chain_str = '::'
     if chain_spec_dict != None:
         chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
+    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
     if address != None:
         r |= Lock.check(chain_str, lock_flags, address=address, session=session)
     if r > 0:

apps/cic-eth/cic_eth/admin/nonce.py

@@ -33,7 +33,6 @@ from cic_eth.admin.ctrl import (
 from cic_eth.queue.tx import queue_create
 from cic_eth.eth.gas import create_check_gas_task
 from cic_eth.task import BaseTask
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -74,7 +73,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
 
     set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
 
-    query_address = tx_normalize.wallet_address(address)
+    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.sender==query_address)

apps/cic-eth/cic_eth/api/admin.py

@@ -32,6 +32,7 @@ from chainqueue.db.enum import (
         status_str,
     )
 from chainqueue.error import TxStateChangeError
+from chainqueue.sql.query import get_tx
 from eth_erc20 import ERC20
 
 # local imports
@@ -39,7 +40,6 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.error import InitializationError
-from cic_eth.queue.query import get_tx_local
 
 app = celery.current_app
 
@@ -284,7 +284,7 @@ class AdminApi:
         tx_hash_hex = None
         session = SessionBase.create_session()
         for k in txs.keys():
-            tx_dict = get_tx_local(chain_spec, k, session=session)
+            tx_dict = get_tx(chain_spec, k, session=session)
             if tx_dict['nonce'] == nonce:
                 tx_hash_hex = k
         session.close()

apps/cic-eth/cic_eth/api/api_task.py

@@ -520,9 +520,9 @@ class Api(ApiBase):
             s_external_get = celery.signature(
                 external_task,
                 [
+                    address,
                     offset,
                     limit,
-                    address,
                     ],
                 queue=external_queue,
                 )

apps/cic-eth/cic_eth/cli/rpc.py

@@ -24,24 +24,20 @@ class RPC:
 
 
     def get_default(self):
-        return self.get_by_label('default')
+        return RPCConnection.connect(self.chain_spec, 'default')
-
-
-    def get_by_label(self, label):
-        return RPCConnection.connect(self.chain_spec, label)
 
 
     @staticmethod
-    def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
+    def from_config(config, use_signer=False):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
         if use_signer:
 
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
+            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
         rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc

apps/cic-eth/cic_eth/data/config/dispatcher.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 1

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,2 +1,2 @@
 [eth]
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_gifter_minimum_balance = 10000000000000000000

apps/cic-eth/cic_eth/data/config/retry.ini

@@ -1,3 +0,0 @@
-[retry]
-delay =
-batch_size =

apps/cic-eth/cic_eth/db/models/lock.py

@@ -4,12 +4,12 @@ import logging
 
 # third-party imports
 from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode  import ZERO_ADDRESS_NORMAL
 
 logg = logging.getLogger()
 
@@ -37,7 +37,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
+    def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
         """Sets flags associated with the given address and chain.
 
         If a flags entry does not exist it is created.
@@ -90,7 +90,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
+    def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
         """Resets flags associated with the given address and chain.
 
         If the resulting flags entry value is 0, the entry will be deleted.
@@ -134,7 +134,7 @@ class Lock(SessionBase):
 
 
     @staticmethod
-    def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
+    def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
         """Checks whether all given flags are set for given address and chain. 
 
         Does not validate the address against any other tables or components.

apps/cic-eth/cic_eth/encode.py

@@ -1,16 +0,0 @@
-# external imports
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainqueue.encode import TxHexNormalizer
-from chainlib.eth.tx import unpack
-
-tx_normalize = TxHexNormalizer()
-
-ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
-
-
-def unpack_normal(signed_tx_bytes, chain_spec):
-    tx = unpack(signed_tx_bytes, chain_spec)
-    tx['hash'] = tx_normalize.tx_hash(tx['hash'])
-    tx['from'] = tx_normalize.wallet_address(tx['from'])
-    tx['to'] = tx_normalize.wallet_address(tx['to'])
-    return tx

apps/cic-eth/cic_eth/eth/account.py

@@ -14,7 +14,10 @@ from chainlib.eth.sign import (
         sign_message,
         )
 from chainlib.eth.address import to_checksum_address
-from chainlib.eth.tx import TxFormat
+from chainlib.eth.tx import (
+        TxFormat,
+        unpack,
+        )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
 from eth_accounts_index.registry import AccountRegistry
@@ -46,10 +49,6 @@ from cic_eth.eth.nonce import (
 from cic_eth.queue.tx import (
         register_tx,
         )
-from cic_eth.encode import (
-        unpack_normal,
-        ZERO_ADDRESS_NORMAL,
-        )
 
 logg = logging.getLogger()
 celery_app = celery.current_app 
@@ -296,17 +295,17 @@ def cache_gift_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
 
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
     tx_data = Faucet.parse_give_to_request(tx['data'])
 
     session = self.create_session()
 
     tx_dict = {
-            'hash': tx['hash'],
+            'hash': tx_hash_hex,
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS_NORMAL,
+            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS,
             'from_value': 0,
             'to_value': 0,
             }
@@ -335,17 +334,17 @@ def cache_account_data(
     :rtype: tuple
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
+    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
     tx_data = AccountsIndex.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_dict = {
-            'hash': tx['hash'],
+            'hash': tx_hash_hex,
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS_NORMAL,
+            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS,
             'from_value': 0,
             'to_value': 0,
             }

apps/cic-eth/cic_eth/eth/gas.py

@@ -4,7 +4,7 @@ import logging
 # external imports
 import celery
 from hexathon import strip_0x
-#from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.eth.address import is_checksum_address
 from chainlib.connection import RPCConnection
@@ -21,6 +21,7 @@ from chainlib.eth.error import (
 from chainlib.eth.tx import (
         TxFactory,
         TxFormat,
+        unpack,
         )
 from chainlib.eth.contract import (
         abi_decode_single,
@@ -44,7 +45,6 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
 from cic_eth.queue.tx import (
         queue_create,
         register_tx,
-        unpack,
         )
 from cic_eth.queue.query import get_tx
 from cic_eth.task import (
@@ -53,11 +53,6 @@ from cic_eth.task import (
         CriticalSQLAlchemyAndSignerTask,
         CriticalWeb3AndSignerTask,
         )
-from cic_eth.encode import (
-        tx_normalize,
-        ZERO_ADDRESS_NORMAL,
-        unpack_normal,
-        )
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -71,7 +66,6 @@ class MaxGasOracle:
         return MAXIMUM_FEE_UNITS
 
 
-#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
     """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher.
 
@@ -136,16 +130,16 @@ def cache_gas_data(
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
+    tx = unpack(tx_signed_raw_bytes, chain_spec)
 
     session = SessionBase.create_session()
 
     tx_dict = {
-            'hash': tx['hash'],
+            'hash': tx_hash_hex,
             'from': tx['from'],
             'to': tx['to'],
-            'source_token': ZERO_ADDRESS_NORMAL,
+            'source_token': ZERO_ADDRESS,
-            'destination_token': ZERO_ADDRESS_NORMAL,
+            'destination_token': ZERO_ADDRESS,
             'from_value': tx['value'],
             'to_value': tx['value'],
             }
@@ -156,7 +150,7 @@ def cache_gas_data(
 
 
 @celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
-def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
+def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
     """Check the gas level of the sender address of a transaction.
 
     If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
@@ -176,20 +170,6 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
     :return: Signed raw transaction data list
     :rtype: param txs, unchanged
     """
-    if address != None:
-        if not is_checksum_address(address):
-            raise ValueError('invalid address {}'.format(address))
-        address = tx_normalize.wallet_address(address)
-
-    tx_hashes = []
-    txs = []
-    for tx_hash in tx_hashes_hex:
-        tx_hash = tx_normalize.tx_hash(tx_hash)
-        tx_hashes.append(tx_hash)
-    for tx in txs_hex:
-        tx = tx_normalize.tx_wire(tx)
-        txs.append(tx)
-
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
 
@@ -207,6 +187,9 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
                 raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
             addresspass.append(address)
 
+    if not is_checksum_address(address):
+        raise ValueError('invalid address {}'.format(address))
+
     queue = self.request.delivery_info.get('routing_key')
 
     conn = RPCConnection.connect(chain_spec)
@@ -321,7 +304,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
     # Determine value of gas tokens to send
     # if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
     # however, we will perform a 0-value transaction instead
-    recipient_address = tx_normalize.wallet_address(recipient_address)
     zero_amount = False
     session = SessionBase.create_session()
     status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
@@ -396,7 +378,6 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     :returns: Transaction hash
     :rtype: str, 0x-hex
     """
-    txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
     session = SessionBase.create_session()
 
     otx = Otx.load(txold_hash_hex, session)

apps/cic-eth/cic_eth/ext/tx.py

@@ -12,11 +12,9 @@ from chainlib.eth.tx import (
         transaction_by_block,
         receipt,
         )
-from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.tx import Tx
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
@@ -25,8 +23,6 @@ from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
 from eth_erc20 import ERC20
-from erc20_faucet import Faucet
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.queue.time import tx_times
@@ -39,32 +35,6 @@ logg = logging.getLogger()
 MAX_BLOCK_TX = 250
 
 
-def parse_transaction(chain_spec, rpc, tx, sender_address=None):
-    try:
-        transfer_data = ERC20.parse_transfer_request(tx['input'])
-        tx_address = transfer_data[0]
-        tx_token_value = transfer_data[1]
-        logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
-        return (tx_address, tx_token_value)
-    except RequestMismatchException:
-        pass
-
-    try:
-        transfer_data = Faucet.parse_give_to_request(tx['input'])
-        tx_address = transfer_data[0]
-        c = Faucet(chain_spec)
-        o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number'])
-        r = rpc.do(o)
-        tx_token_value = Faucet.parse_token_amount(r)
-        logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
-        return (tx_address, tx_token_value)
-
-    except RequestMismatchException:
-        pass
-
-    return None
-
-
 # TODO: Make this method easier to read
 @celery_app.task(bind=True, base=BaseTask)
 def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
@@ -101,39 +71,36 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data)
 
     txs = {}
-    logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high']))
     for block_height in range(bloomspec['low'], bloomspec['high']):
         block_height_bytes = block_height.to_bytes(4, 'big')
         if block_filter.check(block_height_bytes):
             logg.debug('filter matched block {}'.format(block_height))
             o = block_by_number(block_height)
             block = rpc.do(o)
+            logg.debug('block {}'.format(block))
 
             for tx_index in range(0, len(block['transactions'])):
-                tx_index_bytes = tx_index.to_bytes(4, 'big')
+                composite = tx_index + block_height
-                composite = block_height_bytes + tx_index_bytes
+                tx_index_bytes = composite.to_bytes(4, 'big')
-                if tx_filter.check(composite):
+                if tx_filter.check(tx_index_bytes):
                     logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
 
-                    o = transaction_by_block(block['hash'], tx_index)
                     try:
+                        #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
+                        o = transaction_by_block(block['hash'], tx_index)
                         tx = rpc.do(o)
                     except Exception as e:
                         logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e))
                         continue
-
-                    tx = Tx(tx).src()
-
-                    logg.debug('got tx {}'.format(tx))
                     tx_address = None
                     tx_token_value = 0
-
+                    try:
-                    transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address)
+                        transfer_data = ERC20.parse_transfer_request(tx['data'])
-                    if transfer_data == None:
+                        tx_address = transfer_data[0]
+                        tx_token_value = transfer_data[1]
+                    except ValueError:
+                        logg.debug('not a transfer transaction, skipping {}'.format(tx))
                         continue
-                    tx_address = transfer_data[0]
-                    tx_token_value = transfer_data[1]
-                    
                     if address == tx_address:
                         status = StatusEnum.SENT
                         try:
@@ -169,7 +136,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     return txs
 
 
-
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)

apps/cic-eth/cic_eth/queue/balance.py

@@ -15,7 +15,6 @@ from chainqueue.db.enum import (
 # local imports
 from cic_eth.db import SessionBase
 from cic_eth.task import CriticalSQLAlchemyTask
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -23,9 +22,6 @@ logg = logging.getLogger()
 
 
 def __balance_outgoing_compatible(token_address, holder_address):
-    token_address = tx_normalize.executable_address(token_address)
-    holder_address = tx_normalize.wallet_address(holder_address)
-
     session = SessionBase.create_session()
     q = session.query(TxCache.from_value)
     q = q.join(Otx)
@@ -62,9 +58,6 @@ def balance_outgoing(tokens, holder_address, chain_spec_dict):
 
 
 def __balance_incoming_compatible(token_address, receiver_address):
-    token_address = tx_normalize.executable_address(token_address)
-    receiver_address = tx_normalize.wallet_address(receiver_address)
-
     session = SessionBase.create_session()
     q = session.query(TxCache.to_value)
     q = q.join(Otx)
@@ -117,7 +110,7 @@ def assemble_balances(balances_collection):
     logg.debug('received collection {}'.format(balances_collection))
     for c in balances_collection:
         for b in c:
-            address = tx_normalize.executable_address(b['address'])
+            address = b['address']
             if tokens.get(address) == None:
                 tokens[address] = {
                     'address': address,

apps/cic-eth/cic_eth/queue/lock.py

@@ -6,7 +6,6 @@ import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db import SessionBase
 from cic_eth.db.models.lock import Lock
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -22,9 +21,6 @@ def get_lock(address=None):
     :returns: List of locks
     :rtype: list of dicts
     """
-    if address != None:
-        address = tx_normalize.wallet_address(address)
-
     session = SessionBase.create_session()
     q = session.query(
             Lock.date_created,

apps/cic-eth/cic_eth/queue/query.py

@@ -4,8 +4,8 @@ import datetime
 # external imports
 import celery
 from chainlib.chain import ChainSpec
-import chainqueue.sql.query
 from chainlib.eth.tx import unpack
+import chainqueue.sql.query
 from chainqueue.db.enum import (
         StatusEnum,
         is_alive,
@@ -20,10 +20,6 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.lock import Lock
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import (
-        tx_normalize,
-        unpack_normal,
-        )
 
 celery_app = celery.current_app
 
@@ -31,76 +27,49 @@ celery_app = celery.current_app
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx_cache(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    return get_tx_cache_local(chain_spec, tx_hash)
+    session = SessionBase.create_session()
-
-
-def get_tx_cache_local(chain_spec, tx_hash, session=None):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
-    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_tx_cache(chain_spec, tx_hash, session=session)
-    SessionBase.release_session(session)
+    session.close()
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_tx(chain_spec_dict, tx_hash):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    return get_tx_local(chain_spec, tx_hash)
+    session = SessionBase.create_session()
-
-
-def get_tx_local(chain_spec, tx_hash, session=None):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
-    session = SessionBase.bind_session(session)
     r =  chainqueue.sql.query.get_tx(chain_spec, tx_hash, session=session)
-    SessionBase.release_session(session)
+    session.close()
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart)
+    session = SessionBase.create_session()
-
-
-def get_account_tx_local(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=None):
-    address = tx_normalize.wallet_address(address)
-    session = SessionBase.bind_session(session)
     r = chainqueue.sql.query.get_account_tx(chain_spec, address, as_sender=True, as_recipient=True, counterpart=None, session=session)
-    SessionBase.release_session(session)
+    session.close()
     return r
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
-def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0):
+def get_upcoming_tx_nolock(chain_spec_dict, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    return get_upcoming_tx_nolock_local(chain_spec, status=status, not_status=not_status, recipient=recipient, before=before, limit=limit)
-
-
-def get_upcoming_tx_nolock_local(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
-    recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.create_session()
-    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack_normal)
+    r = chainqueue.sql.query.get_upcoming_tx(chain_spec, status, not_status=not_status, recipient=recipient, before=before, limit=limit, session=session, decoder=unpack)
     session.close()
     return r
 
 
 def get_status_tx(chain_spec, status, not_status=None, before=None, exact=False, limit=0, session=None):
-    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack_normal)
+    return chainqueue.sql.query.get_status_tx_cache(chain_spec, status, not_status=not_status, before=before, exact=exact, limit=limit, session=session, decoder=unpack)
 
 
 def get_paused_tx(chain_spec, status=None, sender=None, session=None, decoder=None):
-    sender = tx_normalize.wallet_address(sender)
+    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack)
-    return chainqueue.sql.query.get_paused_tx_cache(chain_spec, status=status, sender=sender, session=session, decoder=unpack_normal)
 
 
 def get_nonce_tx(chain_spec, nonce, sender):
-    sender = tx_normalize.wallet_address(sender)
+    return get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack)
-    return get_nonce_tx_local(chain_spec, nonce, sender)
-
-
-def get_nonce_tx_local(chain_spec, nonce, sender, session=None):
-    sender = tx_normalize.wallet_address(sender)
-    return chainqueue.sql.query.get_nonce_tx_cache(chain_spec, nonce, sender, decoder=unpack_normal, session=session)
 
 
 def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, recipient=None, before=None, limit=0, session=None):
@@ -122,8 +91,6 @@ def get_upcoming_tx(chain_spec, status=StatusEnum.READYSEND, not_status=None, re
     :returns: Transactions
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
-    if recipient != None:
-        recipient = tx_normalize.wallet_address(recipient)
     session = SessionBase.bind_session(session)
     q_outer = session.query(
             TxCache.sender,

apps/cic-eth/cic_eth/queue/state.py

@@ -6,14 +6,12 @@ import chainqueue.sql.state
 import celery
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_sent(chain_spec_dict, tx_hash, fail=False):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_sent(chain_spec, tx_hash, fail, session=session)
@@ -23,7 +21,6 @@ def set_sent(chain_spec_dict, tx_hash, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_final(chain_spec, tx_hash, block=block, tx_index=tx_index, fail=fail, session=session)
@@ -33,7 +30,6 @@ def set_final(chain_spec_dict, tx_hash, block=None, tx_index=None, fail=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_cancel(chain_spec_dict, tx_hash, manual=False):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_cancel(chain_spec, tx_hash, manual, session=session)
@@ -43,7 +39,6 @@ def set_cancel(chain_spec_dict, tx_hash, manual=False):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_rejected(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_rejected(chain_spec, tx_hash, session=session)
@@ -53,7 +48,6 @@ def set_rejected(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_fubar(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_fubar(chain_spec, tx_hash, session=session)
@@ -63,7 +57,6 @@ def set_fubar(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_manual(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_manual(chain_spec, tx_hash, session=session)
@@ -73,7 +66,6 @@ def set_manual(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_ready(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_ready(chain_spec, tx_hash, session=session)
@@ -83,7 +75,6 @@ def set_ready(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_reserved(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_reserved(chain_spec, tx_hash, session=session)
@@ -93,7 +84,6 @@ def set_reserved(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def set_waitforgas(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.set_waitforgas(chain_spec, tx_hash, session=session)
@@ -103,7 +93,6 @@ def set_waitforgas(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def get_state_log(chain_spec_dict, tx_hash):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.get_state_log(chain_spec, tx_hash, session=session)
@@ -113,7 +102,6 @@ def get_state_log(chain_spec_dict, tx_hash):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def obsolete(chain_spec_dict, tx_hash, final):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     session = SessionBase.create_session()
     r = chainqueue.sql.state.obsolete_by_cache(chain_spec, tx_hash, final, session=session)

apps/cic-eth/cic_eth/queue/time.py

@@ -13,7 +13,6 @@ from chainqueue.error import NotLocalTxError
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
 from cic_eth.db.models.base import SessionBase
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 
@@ -21,7 +20,6 @@ logg = logging.getLogger()
 
 
 def tx_times(tx_hash, chain_spec, session=None):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
 
     session = SessionBase.bind_session(session)
 

apps/cic-eth/cic_eth/queue/tx.py

@@ -32,16 +32,12 @@ from cic_eth.db import SessionBase
 from cic_eth.db.enum import LockEnum
 from cic_eth.task import CriticalSQLAlchemyTask
 from cic_eth.error import LockedError
-from cic_eth.encode import tx_normalize
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
 def queue_create(chain_spec, nonce, holder_address, tx_hash, signed_tx, session=None):
-    tx_hash = tx_normalize.tx_hash(tx_hash)
-    signed_tx = tx_normalize.tx_hash(signed_tx)
-    holder_address = tx_normalize.wallet_address(holder_address)
     session = SessionBase.bind_session(session)
 
     lock = Lock.check_aggregate(str(chain_spec), LockEnum.QUEUE, holder_address, session=session) 
@@ -71,8 +67,6 @@ def register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=No
     :returns: Tuple; Transaction hash, signed raw transaction data
     :rtype: tuple
     """
-    tx_hash_hex = tx_normalize.tx_hash(tx_hash_hex)
-    tx_signed_raw_hex = tx_normalize.tx_hash(tx_signed_raw_hex)
     logg.debug('adding queue tx {}:{} -> {}'.format(chain_spec, tx_hash_hex, tx_signed_raw_hex))
     tx_signed_raw = bytes.fromhex(strip_0x(tx_signed_raw_hex))
     tx = unpack(tx_signed_raw, chain_spec)

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -101,14 +101,14 @@ class DispatchSyncer:
                     LockEnum.QUEUE,
                     tx['from'],
                     ],
-                queue=config.get('CELERY_QUEUE'),
+                queue=queue,
                 )
             s_send = celery.signature(
                     'cic_eth.eth.tx.send',
                     [
                         self.chain_spec.asdict(),
                         ], 
-                    queue=config.get('CELERY_QUEUE'),
+                    queue=queue,
                     )
             s_check.link(s_send)
             t = s_check.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -10,14 +10,15 @@ from chainlib.eth.tx import unpack
 from chainqueue.db.enum import StatusBits
 from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
+from chainqueue.sql.query import get_paused_tx_cache as get_paused_tx
 from chainlib.eth.address import to_checksum_address
 
 # local imports
 from cic_eth.db.models.base import SessionBase
 from cic_eth.eth.gas import create_check_gas_task
-from cic_eth.queue.query import get_paused_tx
 from .base import SyncFilter
 
+#logg = logging.getLogger().getChild(__name__)
 logg = logging.getLogger()
 
 

apps/cic-eth/cic_eth/runnable/tag.py

@@ -18,7 +18,7 @@ from cic_eth.db.models.base import SessionBase
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
+arg_flags = cic_eth.cli.argflag_std_base
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_positional('tag', type=str, help='address tag')

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        4,
+        2,
-        'alpha.7',
+        'alpha.4',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+NAME=cic_eth
+USER=postgres
+PASSWORD=tralala
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+POOL_SIZE=50
+DEBUG=0

apps/cic-eth/config/docker/chain.ini

@@ -1,2 +0,0 @@
-[chain]
-spec = evm:bloxberg:8996

apps/cic-eth/config/docker/database.ini

@@ -0,0 +1,10 @@
+[database]
+NAME=cic_eth
+USER=postgres
+PASSWORD=tralala
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+POOL_SIZE=50
+DEBUG=0

apps/cic-eth/docker/Dockerfile

@@ -8,7 +8,6 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
@@ -19,7 +18,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt
@@ -33,7 +31,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
     cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 COPY docker/entrypoints/* ./ 

apps/cic-eth/docker/Dockerfile_ci

@@ -0,0 +1,69 @@
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+WORKDIR /usr/src/cic-eth
+
+# Copy just the requirements and install....this _might_ give docker a hint on caching but we 
+# do load these all into setup.py later
+# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
+#COPY cic-eth/requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+#    pip install --index-url https://pypi.org/simple \
+#    --force-reinstall \
+#	--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+#    -r requirements.txt 
+COPY *requirements.txt . 
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt \
+    -r services_requirements.txt \
+    -r admin_requirements.txt
+    
+COPY . .
+RUN python setup.py install
+
+COPY docker/entrypoints/* ./ 
+RUN chmod 755 *.sh
+
+# # ini files in config directory defines the configurable parameters for the application
+# # they can all be overridden by environment variables
+# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY config/ /usr/local/etc/cic-eth/
+COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+#COPY util/liveness/health.sh /usr/local/bin/health.sh
+ENTRYPOINT []
+
+# ------------------ PRODUCTION CONTAINER ----------------------
+#FROM python:3.8.6-slim-buster as prod 
+#
+#RUN apt-get update && \
+#	apt install -y gnupg libpq-dev procps
+#
+#WORKDIR /root
+#
+#COPY --from=dev /usr/local/bin/ /usr/local/bin/
+#COPY --from=dev /usr/local/lib/python3.8/site-packages/ \
+#                          /usr/local/lib/python3.8/site-packages/
+#
+#COPY docker/entrypoints/* ./ 
+#RUN chmod 755 *.sh
+#
+## # ini files in config directory defines the configurable parameters for the application
+## # they can all be overridden by environment variables
+## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+#COPY config/ /usr/local/etc/cic-eth/
+#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
+#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+#COPY scripts/ scripts/ 
+#
+## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
+##COPY util/liveness/health.sh /usr/local/bin/health.sh
+#
+#ENTRYPOINT []
+#

apps/cic-eth/docker/run_tests.sh

@@ -1,11 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r admin_requirements.txt \
--r services_requirements.txt \
--r test_requirements.txt  
-
-export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a7,<0.1.0
+chainlib-eth>=0.0.7a1,<0.1.0
 semver==2.13.0

apps/cic-eth/scripts/migrate.py

@@ -10,7 +10,6 @@ from alembic.config import Config as AlembicConfig
 import confini
 
 from cic_eth.db import dsn_from_config
-import cic_eth.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -20,20 +19,25 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_eth', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
 
-arg_flags = cic_eth.cli.argflag_std_base
+config_dir = os.path.join('/usr/local/etc/cic-eth')
 
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
 argparser.add_argument('-f', action='store_true', help='force action')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-extra_args = {
+if args.vv:
-    'migrations_dir': None,
+    logging.getLogger().setLevel(logging.DEBUG)
-    'reset': None,
+elif args.v:
-    'f': '_FORCE_ACTION',
+    logging.getLogger().setLevel(logging.INFO)
-        }
+
-config = cic_eth.cli.Config.from_args(args, arg_flags, 0, extra_args=extra_args)
+config = confini.Config(args.c, args.env_prefix)
+config.process()
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config:\n{}'.format(config))

apps/cic-eth/services_requirements.txt

@@ -1,15 +1,15 @@
-chainqueue>=0.0.5a1,<0.1.0
+chainqueue>=0.0.3a2,<0.1.0
-chainsyncer[sql]>=0.0.6a3,<0.1.0
+chainsyncer[sql]>=0.0.6a1,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
-hexathon~=0.0.1a8
+hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.2.3a4,<0.3.0
+eth-address-index>=0.1.4a1,<0.2.0
-eth-accounts-index>=0.1.2a3,<0.2.0
+eth-accounts-index>=0.0.14a1,<0.1.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.5.8a1,<0.6.0
-erc20-faucet>=0.3.2a2,<0.4.0
+erc20-faucet>=0.2.4a1,<0.3.0
-erc20-transfer-authorization>=0.3.5a2,<0.4.0
+erc20-transfer-authorization>=0.3.4a1,<0.4.0
-sarafu-faucet>=0.0.7a2,<0.1.0
+sarafu-faucet>=0.0.5a2,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.1.2a2
+eth-erc20~=0.0.12a1

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -200,7 +200,6 @@ def test_callback_filter(
     assert r['status'] == 1
 
     rcpt = snake_and_camel(r)
-    tx.block.hash = rcpt['block_hash']
     tx.apply_receipt(rcpt)
 
     fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])

apps/cic-eth/tests/filters/test_gas_filter.py

@@ -1,6 +1,7 @@
 # external imports
 from chainlib.connection import RPCConnection
 from chainlib.eth.nonce import OverrideNonceOracle
+from chainqueue.sql.tx import create as queue_create
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -25,8 +26,6 @@ from chainqueue.db.enum import StatusBits
 # local imports
 from cic_eth.runnable.daemons.filters.gas import GasFilter
 from cic_eth.eth.gas import cache_gas_data
-from cic_eth.encode import tx_normalize
-from cic_eth.queue.tx import queue_create
 
 
 def test_filter_gas(

apps/cic-eth/tests/filters/test_register_filter.py

@@ -22,11 +22,10 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
+from chainqueue.sql.query import get_account_tx
 
 # local imports
 from cic_eth.runnable.daemons.filters.register import RegistrationFilter
-from cic_eth.encode import tx_normalize
-from cic_eth.queue.query import get_account_tx_local
 
 logg = logging.getLogger()
 
@@ -80,7 +79,7 @@ def test_register_filter(
     t.get_leaf()
     assert t.successful()
 
-    gift_txs = get_account_tx_local(default_chain_spec, agent_roles['ALICE'], as_sender=True, session=init_database)
+    gift_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database)
     ks = list(gift_txs.keys())
     assert len(ks) == 1
 

apps/cic-eth/tests/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
--r admin_requirements.txt
--r services_requirements.txt
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests

apps/cic-eth/tests/task/api/test_admin.py

@@ -34,6 +34,10 @@ from chainqueue.sql.state import (
         set_ready,
         set_reserved,
         )
+from chainqueue.sql.query import (
+        get_tx,
+        get_nonce_tx_cache,
+        )
 
 # local imports
 from cic_eth.api.admin import AdminApi
@@ -42,11 +46,6 @@ from cic_eth.db.enum import LockEnum
 from cic_eth.error import InitializationError
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.queue.tx import queue_create
-from cic_eth.queue.query import (
-        get_tx,
-        get_nonce_tx_local,
-        )
-from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -287,15 +286,13 @@ def test_fix_nonce(
     assert t.successful()
 
     init_database.commit()
-   
+    
-    logg.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+    txs = get_nonce_tx_cache(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
-    txs = get_nonce_tx_local(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
     ks = txs.keys()
     assert len(ks) == 2
 
     for k in ks:
-        #hsh = add_0x(k)
+        hsh = add_0x(k)
-        hsh = tx_normalize.tx_hash(k)
         otx = Otx.load(hsh, session=init_database)
         init_database.refresh(otx)
         logg.debug('checking nonce {} tx {} status {}'.format(3, otx.tx_hash, otx.status))

apps/cic-eth/tests/task/api/test_admin_noncritical.py

@@ -30,6 +30,7 @@ from chainqueue.sql.state import (
         )
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusBits
+from chainqueue.sql.query import get_nonce_tx_cache
 from eth_erc20 import ERC20
 from cic_eth_registry import CICRegistry
 
@@ -37,7 +38,6 @@ from cic_eth_registry import CICRegistry
 from cic_eth.api.admin import AdminApi
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.eth.erc20 import cache_transfer_data
-from cic_eth.queue.query import get_nonce_tx_local
 
 logg = logging.getLogger()
 
@@ -312,7 +312,7 @@ def test_resend_inplace(
     otx = Otx.load(tx_hash_hex, session=init_database)
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 2
 
 
@@ -363,10 +363,10 @@ def test_resend_clone(
     assert otx.status & StatusBits.IN_NETWORK == StatusBits.IN_NETWORK
     assert otx.status & StatusBits.OBSOLETE == StatusBits.OBSOLETE
 
-    txs = get_nonce_tx_local(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
-    txs = get_nonce_tx_local(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
+    txs = get_nonce_tx_cache(default_chain_spec, otx.nonce + 1, agent_roles['ALICE'], session=init_database)
     assert len(txs) == 1
 
     otx = Otx.load(txs[0], session=init_database)

apps/cic-eth/tests/task/test_task_gas.py

@@ -21,6 +21,7 @@ from chainlib.eth.constant import (
         MINIMUM_FEE_UNITS,
         MINIMUM_FEE_PRICE,
         )
+from chainqueue.sql.tx import create as queue_create
 from chainqueue.sql.query import get_tx
 from chainqueue.db.enum import StatusBits
 from chainqueue.sql.state import (
@@ -34,7 +35,6 @@ from hexathon import strip_0x
 # local imports
 from cic_eth.eth.gas import cache_gas_data
 from cic_eth.error import OutOfGasError
-from cic_eth.queue.tx import queue_create
 
 logg = logging.getLogger()
 

apps/cic-eth/tests/task/test_task_list.py

@@ -51,7 +51,6 @@ def test_ext_tx_collate(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
-    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 

apps/cic-eth/tests/testdata/config/test.ini

@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy

apps/cic-eth/tests/unit/cli/test_cli_args.py

@@ -1,56 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_eth.cli
-
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-#config_dir = os.path.join(script_dir, '..', '..', 'testdata', 'config')
-
-
-def test_argumentparser_to_config():
-
-    argparser = cic_eth.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '--redis-host', 'foo',
-        '--redis-port', '123',
-        '--redis-db', '0',
-        '--redis-host-callback', 'bar',
-        '--redis-port-callback', '456',
-        '--redis-timeout', '10.0',
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    #config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-    config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('REDIS_HOST') == 'foo'
-    assert config.get('REDIS_PORT') == 123
-    assert config.get('REDIS_DB') == 0
-    assert config.get('_REDIS_HOST_CALLBACK') == 'bar'
-    assert config.get('_REDIS_PORT_CALLBACK') == 456
-    assert config.get('REDIS_TIMEOUT') == 10.0
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-

apps/cic-eth/tests/unit/cli/test_cli_celery.py

@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_eth.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_eth.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_eth.cli.CeleryApp.from_config(config)

apps/cic-eth/tests/unit/cli/test_cli_chain.py

@@ -1,64 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-
-# local imports
-import cic_eth.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_eth.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        agent_roles,
-        ):
-    ifc = cic_eth.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-

apps/cic-eth/tests/unit/db/test_tx.py

@@ -46,7 +46,6 @@ def test_set(
         tx_hash_hex,
         tx_signed_raw_hex,
         )
-    otx.block = 666
     init_database.add(otx)
     init_database.commit()
 
@@ -75,6 +74,7 @@ def test_set(
     assert (tx_stored.destination_token_address == ZERO_ADDRESS)
     assert (tx_stored.from_value == tx['value'])
     assert (tx_stored.to_value == to_value)
+    assert (tx_stored.block_number == 666)
     assert (tx_stored.tx_index == 13)
 
 

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -20,6 +20,7 @@ from cic_eth.db.models.nonce import (
 logg = logging.getLogger()
 
 
+# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state
 def test_filter_process(
         init_database,
         default_chain_spec,
@@ -47,10 +48,10 @@ def test_filter_process(
     eth_rpc.do(o)
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
-    block_bytes = r['block_number'].to_bytes(4, 'big')
+    a = r['block_number']
-    b.add(block_bytes)
+    b.add(a.to_bytes(4, 'big'))
-    tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
+    a = r['block_number'] + r['transaction_index']
-    t.add(block_bytes + tx_index_bytes)
+    t.add(a.to_bytes(4, 'big'))
     tx_hashes.append(tx_hash_hex)
 
     # external tx
@@ -60,10 +61,10 @@ def test_filter_process(
     eth_rpc.do(o)
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
-    block_bytes = r['block_number'].to_bytes(4, 'big')
+    a = r['block_number']
-    b.add(block_bytes)
+    b.add(a.to_bytes(4, 'big'))
-    tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
+    a = r['block_number'] + r['transaction_index']
-    t.add(block_bytes + tx_index_bytes)
+    t.add(a.to_bytes(4, 'big'))
     tx_hashes.append(tx_hash_hex)
 
     init_eth_tester.mine_blocks(10)

apps/cic-eth/tests/unit/queue/test_balances.py

@@ -13,7 +13,6 @@ from cic_eth.queue.balance import (
         balance_incoming,
         assemble_balances,
         )
-from cic_eth.encode import tx_normalize
 
 logg = logging.getLogger()
 
@@ -52,8 +51,8 @@ def test_assemble():
     r = assemble_balances(b)
     logg.debug('r {}'.format(r))
 
-    assert r[0]['address'] == tx_normalize.executable_address(token_foo)
+    assert r[0]['address'] == token_foo
-    assert r[1]['address'] == tx_normalize.executable_address(token_bar)
+    assert r[1]['address'] == token_bar
     assert r[0].get('balance_foo') != None
     assert r[0].get('balance_bar') != None
     assert r[1].get('balance_baz') != None
@@ -75,11 +74,11 @@ def test_outgoing_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_normalize.tx_hash(tx_hash),
+            tx_hash,
-            tx_normalize.wallet_address(sender),
+            sender,
-            tx_normalize.wallet_address(recipient),
+            recipient,
-            tx_normalize.executable_address(token_address),
+            token_address,
-            tx_normalize.executable_address(token_address),
+            token_address,
             1000,
             1000,
             session=init_database,
@@ -126,11 +125,11 @@ def test_incoming_balance(
     token_address = '0x' + os.urandom(20).hex()
     sender = '0x' + os.urandom(20).hex()
     txc = TxCache(
-            tx_normalize.tx_hash(tx_hash),
+            tx_hash,
-            tx_normalize.wallet_address(sender),
+            sender,
-            tx_normalize.wallet_address(recipient),
+            recipient,
-            tx_normalize.executable_address(token_address),
+            token_address,
-            tx_normalize.executable_address(token_address),
+            token_address,
             1000,
             1000,
             session=init_database,

apps/cic-eth/tests/unit/queue/test_query.py

@@ -21,7 +21,6 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.queue.query import get_upcoming_tx
 from cic_eth.queue.tx import register_tx
 from cic_eth.eth.gas import cache_gas_data
-from cic_eth.encode import tx_normalize
 
 # test imports
 from tests.util.nonce import StaticNonceOracle
@@ -40,8 +39,8 @@ def test_upcoming_with_lock(
     gas_oracle = RPCGasOracle(eth_rpc)
     c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
 
-    alice_normal = tx_normalize.wallet_address(agent_roles['ALICE'])
+    alice_normal = add_0x(hex_uniform(strip_0x(agent_roles['ALICE'])))
-    bob_normal = tx_normalize.wallet_address(agent_roles['BOB'])
+    bob_normal = add_0x(hex_uniform(strip_0x(agent_roles['BOB'])))
 
     (tx_hash_hex, tx_rpc) = c.create(alice_normal, bob_normal, 100 * (10 ** 6))
     tx_signed_raw_hex = tx_rpc['params'][0]

apps/cic-eth/tests/unit/queue/test_queue_lock.py

@@ -9,7 +9,7 @@ from cic_eth.db.models.lock import Lock
 from cic_eth.db.enum import LockEnum
 from cic_eth.error import LockedError
 from cic_eth.queue.tx import queue_create
-from cic_eth.encode import tx_normalize
+
 
 def test_queue_lock(
     init_database,
@@ -21,8 +21,6 @@ def test_queue_lock(
     address = '0x' + os.urandom(20).hex()
     tx_hash = '0x' + os.urandom(32).hex()
     tx_raw = '0x' + os.urandom(128).hex()
-    address_normal = tx_normalize.wallet_address(address)
-    tx_hash_normal = tx_normalize.tx_hash(tx_hash)
 
     Lock.set(chain_str, LockEnum.QUEUE)
     with pytest.raises(LockedError):
@@ -34,7 +32,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -54,7 +52,7 @@ def test_queue_lock(
                 tx_raw,
                 )
 
-    Lock.set(chain_str, LockEnum.QUEUE, address=address_normal, tx_hash=tx_hash_normal)
+    Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     with pytest.raises(LockedError):
         queue_create(
                 default_chain_spec,
@@ -63,3 +61,5 @@ def test_queue_lock(
                 tx_hash,
                 tx_raw,
                 )
+
+

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,8 @@
-crypto-dev-signer>=0.4.15a1,<=0.4.15
+crypto-dev-signer>=0.4.14b7,<=0.4.14
-chainqueue>=0.0.5a1,<0.1.0
+chainqueue>=0.0.3a1,<0.1.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+confini>=0.3.6rc4,<0.5.0
+cic-eth-registry>=0.5.8a1,<0.6.0
 redis==3.5.3
-hexathon~=0.0.1a8
+hexathon~=0.0.1a7
 pycryptodome==3.10.1
 pyxdg==0.27

apps/cic-meta/.gitlab-ci.yml

@@ -1,16 +1,43 @@
-build-test-cic-meta:
+
-  stage: test
+.cic_meta_variables:
-  tags:
+    variables:
-   - integration
+        APP_NAME: cic-meta
-  variables:
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-    APP_NAME: cic-meta
+        CONTEXT: apps/$APP_NAME
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+
-  script:
+build-mr-cic-meta:
-    - cd apps/cic-meta
+    extends:
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .py_build_merge_request
-    - docker run --entrypoint=sh $MR_IMAGE_TAG docker/run_tests.sh
+        - .cic_meta_variables
-     #rules:
+    rules:
-     #- if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-     #  changes:
+      changes:
-     #      - apps/$APP_NAME/**/*
+          - apps/cic-meta/**/*
-     #  when: always
+      when: always
+
+test-mr-cic-meta:
+    extends:
+        - .cic_meta_variables
+    stage: test
+    image: $MR_IMAGE_TAG 
+    script:
+        - cd /root
+        - npm install --dev 
+        - npm run test
+        - npm run test:coverage
+    needs: ["build-mr-cic-meta"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/cic-meta/**/*
+      when: always
+
+build-push-cic-meta:
+    extends:
+        - .py_build_push
+        - .cic_meta_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-meta/**/*
+      when: always

apps/cic-meta/docker/Dockerfile

@@ -15,10 +15,11 @@ RUN --mount=type=cache,mode=0755,target=/root/.npm \
 COPY webpack.config.js .
 COPY tsconfig.json . 
 ## required to build the cic-client-meta module
-COPY . .
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
 COPY tests/*.asc /root/pgp/
 
-
 ## copy runtime configs
 COPY .config/ /usr/local/etc/cic-meta/
 #

apps/cic-meta/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+#FROM node:15.3.0-alpine3.10
+FROM node:lts-alpine3.14
+
+WORKDIR /root
+
+RUN apk add --no-cache postgresql bash
+
+# copy the dependencies
+COPY package.json package-lock.json .
+RUN npm set cache /root/.npm && \
+    npm ci
+
+COPY webpack.config.js .
+COPY tsconfig.json . 
+## required to build the cic-client-meta module
+COPY src/ src/
+COPY scripts/ scripts/
+COPY tests/ tests/
+COPY tests/*.asc /root/pgp/
+
+## copy runtime configs
+COPY .config/ /usr/local/etc/cic-meta/
+#
+## db migrations
+COPY docker/db.sh ./db.sh
+RUN chmod 755 ./db.sh
+#
+RUN alias tsc=node_modules/typescript/bin/tsc 
+COPY docker/start_server.sh ./start_server.sh
+RUN chmod 755 ./start_server.sh
+ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/docker/run_tests.sh

@@ -1,7 +0,0 @@
-#! /bin/bash
-
-set -e
-
-npm install --dev 
-npm run test
-npm run test:coverage

apps/cic-notify/.gitlab-ci.yml

@@ -1,17 +1,52 @@
-build-test-cic-notify:
+.cic_notify_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-notify
-    - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-notify
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-notify:
-  script:
+    extends:
-    - cd apps/cic-notify
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_notify_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  rules:
+      changes:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+          - apps/cic-notify/**/*
-    changes:
+      when: always
-        - apps/$APP_NAME/**/*
+
-    when: always
+test-mr-cic-notify:
+    stage: test
+    extends:
+        - .cic_notify_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests
+    needs: ["build-mr-cic-notify"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-notify:
+    extends:
+        - .py_build_push
+        - .cic_notify_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-notify/**/*
+      when: always
+        
+

apps/cic-notify/docker/Dockerfile_ci

@@ -0,0 +1,27 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+
+#RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+COPY requirements.txt .
+
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+
+RUN python setup.py install
+
+COPY docker/*.sh .
+RUN chmod +x *.sh 
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY .config/ /usr/local/etc/cic-notify/
+COPY cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/
+
+ENTRYPOINT []

apps/cic-notify/docker/run_tests.sh

@@ -1,9 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_notify --cov-fail-under=90 --cov-report term-missing tests

apps/cic-ussd/.gitlab-ci.yml

@@ -1,16 +1,52 @@
-build-test-cic-ussd:
+.cic_ussd_variables:
-  stage: test
+    variables:
-  tags:
+        APP_NAME: cic-ussd
-  - integration
+        DOCKERFILE_PATH: docker/Dockerfile_ci
-  variables:
+        CONTEXT: apps/$APP_NAME
-    APP_NAME: cic-ussd
+
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
+build-mr-cic-ussd:
-  script:
+    extends:
-    - cd apps/cic-ussd
+        - .py_build_merge_request
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
+        - .cic_ussd_variables
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
+    rules:
-  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
-    changes:
+          - apps/cic-ussd/**/*
-        - apps/$APP_NAME/**/*
+      when: always
-    when: always
+
+test-mr-cic-ussd:
+    stage: test
+    extends:
+        - .cic_ussd_variables
+    cache:
+        key:
+            files:
+                - test_requirements.txt
+        paths:
+        - /root/.cache/pip
+    image: $MR_IMAGE_TAG
+    script:
+        - cd apps/$APP_NAME/
+        - >
+          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
+          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
+          -r test_requirements.txt 
+        - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd
+    needs: ["build-mr-cic-ussd"]
+    rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      changes:
+          - apps/$APP_NAME/**/*
+      when: always
+
+build-push-cic-ussd:
+    extends:
+        - .py_build_push
+        - .cic_ussd_variables
+    rules:
+    - if:  $CI_COMMIT_BRANCH == "master"
+      changes:
+          - apps/cic-ussd/**/*
+      when: always
+
+

apps/cic-ussd/cic_ussd/account/balance.py

@@ -20,7 +20,7 @@ def get_balances(address: str,
                  asynchronous: bool = False,
                  callback_param: any = None,
                  callback_queue='cic-ussd',
-                 callback_task='cic_ussd.tasks.callback_handler.balances_callback') -> Optional[list]:
+                 callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Optional[list]:
     """This function queries cic-eth for an account's balances, It provides a means to receive the balance either
     asynchronously or synchronously.. It returns a dictionary containing the network, outgoing and incoming balances.
     :param address: Ethereum address of an account.

apps/cic-ussd/cic_ussd/account/transaction.py

@@ -117,18 +117,18 @@ def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]:
     return recipient_transaction_data, sender_transaction_data
 
 
-def validate_transaction_account(blockchain_address: str, role: str, session: Session) -> Account:
+def validate_transaction_account(session: Session, transaction: dict) -> Account:
     """This function checks whether the blockchain address specified in a parsed transaction object resolves to an
     account object in the ussd system.
-    :param blockchain_address:
+    :param session: Database session object.
-    :type blockchain_address:
+    :type session: Session
-    :param role:
+    :param transaction: Parsed transaction data object.
-    :type role:
+    :type transaction: dict
-    :param session:
-    :type session:
     :return:
     :rtype:
     """
+    blockchain_address = transaction.get('blockchain_address')
+    role = transaction.get('role')
     session = SessionBase.bind_session(session)
     account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
     if not account:

apps/cic-ussd/cic_ussd/processor/util.py

@@ -67,7 +67,6 @@ def resume_last_ussd_session(last_state: str) -> Document:
         'exit',
         'exit_invalid_pin',
         'exit_invalid_new_pin',
-        'exit_invalid_recipient',
         'exit_invalid_request',
         'exit_pin_blocked',
         'exit_pin_mismatch',

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -4,7 +4,6 @@ from typing import Tuple
 
 # third party imports
 import celery
-from phonenumbers.phonenumberutil import NumberParseException
 
 # local imports
 from cic_ussd.account.balance import get_cached_available_balance
@@ -22,21 +21,23 @@ logg = logging.getLogger(__file__)
 
 
 def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:
-    """This function checks that a phone number provided as the recipient of a transaction does not match the sending
+    """This function checks that a user exists, is not the initiator of the transaction, has an active account status
-    party's own phone number.
+    and is authorized to perform  standard transactions.
     :param state_machine_data: A tuple containing user input, a ussd session and user object.
     :type state_machine_data: tuple
-    :return: A recipient account's validity for a transaction
+    :return: A user's validity
     :rtype: bool
     """
     user_input, ussd_session, account, session = state_machine_data
-    try:
+    phone_number = process_phone_number(user_input, E164Format.region)
-        phone_number = process_phone_number(user_input, E164Format.region)
+    session = SessionBase.bind_session(session=session)
-    except NumberParseException:
+    recipient = Account.get_by_phone_number(phone_number=phone_number, session=session)
-        phone_number = None
+    SessionBase.release_session(session=session)
     is_not_initiator = phone_number != account.phone_number
-    is_present = Account.get_by_phone_number(phone_number, session) is not None
+    has_active_account_status = False
-    return phone_number is not None and phone_number.startswith('+') and is_present and is_not_initiator
+    if recipient:
+        has_active_account_status = recipient.get_status(session) == AccountStatus.ACTIVE.name
+    return is_not_initiator and has_active_account_status and recipient is not None
 
 
 def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Session]) -> bool:

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -138,7 +138,7 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     balances_data = result[0]
     available_balance = calculate_available_balance(balances_data)
     transaction = param
-    blockchain_address = transaction.get('blockchain_address')
+    blockchain_address = param.get('blockchain_address')
     transaction['available_balance'] = available_balance
     queue = self.request.delivery_info.get('routing_key')
 
@@ -150,10 +150,10 @@ def transaction_balances_callback(self, result: list, param: dict, status_code:
     )
     s_notify_account = celery.signature('cic_ussd.tasks.notifications.transaction', queue=queue)
 
-    if transaction.get('transaction_type') == 'transfer':
+    if param.get('transaction_type') == 'transfer':
         celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
 
-    if transaction.get('transaction_type') == 'tokengift':
+    if param.get('transaction_type') == 'tokengift':
         s_process_account_metadata = celery.signature(
             'cic_ussd.tasks.processor.parse_transaction', [{}, transaction], queue=queue
         )
@@ -184,11 +184,10 @@ def transaction_callback(result: dict, param: str, status_code: int):
     source_token_value = result.get('source_token_value')
 
     recipient_metadata = {
-        "alt_blockchain_address": sender_blockchain_address,
-        "blockchain_address": recipient_blockchain_address,
-        "role": "recipient",
         "token_symbol": destination_token_symbol,
         "token_value": destination_token_value,
+        "blockchain_address": recipient_blockchain_address,
+        "role": "recipient",
         "transaction_type": param
     }
 
@@ -202,11 +201,10 @@ def transaction_callback(result: dict, param: str, status_code: int):
 
     if param == 'transfer':
         sender_metadata = {
-            "alt_blockchain_address": recipient_blockchain_address,
             "blockchain_address": sender_blockchain_address,
-            "role": "sender",
             "token_symbol": source_token_symbol,
             "token_value": source_token_value,
+            "role": "sender",
             "transaction_type": param
         }
 

apps/cic-ussd/cic_ussd/tasks/notifications.py

@@ -29,8 +29,7 @@ def transaction(notification_data: dict):
     phone_number = notification_data.get('phone_number')
     preferred_language = notification_data.get('preferred_language')
     token_symbol = notification_data.get('token_symbol')
-    alt_metadata_id = notification_data.get('alt_metadata_id')
+    transaction_account_metadata = notification_data.get('metadata_id')
-    metadata_id = notification_data.get('metadata_id')
     transaction_type = notification_data.get('transaction_type')
     timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
 
@@ -48,8 +47,7 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_recipient_information=metadata_id,
+                                           tx_sender_information=transaction_account_metadata,
-                                           tx_sender_information=alt_metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)
         if role == 'sender':
@@ -58,7 +56,6 @@ def transaction(notification_data: dict):
                                            preferred_language=preferred_language,
                                            amount=amount,
                                            token_symbol=token_symbol,
-                                           tx_recipient_information=alt_metadata_id,
+                                           tx_recipient_information=transaction_account_metadata,
-                                           tx_sender_information=metadata_id,
                                            timestamp=timestamp,
                                            balance=balance)

apps/cic-ussd/cic_ussd/tasks/processor.py

@@ -11,7 +11,6 @@ from chainlib.hash import strip_0x
 from cic_ussd.account.statement import get_cached_statement
 from cic_ussd.account.transaction import aux_transaction_data, validate_transaction_account
 from cic_ussd.cache import cache_data, cache_data_key
-from cic_ussd.db.models.account import Account
 from cic_ussd.db.models.base import SessionBase
 
 
@@ -73,17 +72,13 @@ def parse_transaction(preferences: dict, transaction: dict) -> dict:
     preferred_language = preferences.get('preferred_language')
     if not preferred_language:
         preferred_language = i18n.config.get('fallback')
-    transaction['preferred_language'] = preferred_language
+
     transaction = aux_transaction_data(preferred_language, transaction)
     session = SessionBase.create_session()
-    role = transaction.get('role')
+    account = validate_transaction_account(session, transaction)
-    alt_blockchain_address = transaction.get('alt_blockchain_address')
+    metadata_id = account.standard_metadata_id()
-    blockchain_address = transaction.get('blockchain_address')
+    transaction['metadata_id'] = metadata_id
-    account = validate_transaction_account(blockchain_address, role, session)
-    alt_account = session.query(Account).filter_by(blockchain_address=alt_blockchain_address).first()
-    if alt_account:
-        transaction['alt_metadata_id'] = alt_account.standard_metadata_id()
-    transaction['metadata_id'] = account.standard_metadata_id()
     transaction['phone_number'] = account.phone_number
+    session.commit()
     session.close()
     return transaction

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 1, 'alpha.4')
+version = (0, 3, 0, 'alpha.10')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/docker/Dockerfile_ci

@@ -0,0 +1,32 @@
+# syntax = docker/dockerfile:1.2
+FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
+RUN apt-get install -y redis-server
+
+# create secrets directory
+RUN mkdir -vp pgp/keys
+
+# create application directory
+RUN mkdir -vp cic-ussd
+RUN mkdir -vp data
+
+COPY requirements.txt .
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+RUN pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
+    -r requirements.txt
+
+COPY . .
+RUN python setup.py install
+
+COPY cic_ussd/db/ussd_menu.json data/
+
+COPY docker/*.sh .
+RUN chmod +x /root/*.sh
+
+# copy config and migration files to definitive file so they can be referenced in path definitions for running scripts
+COPY config/ /usr/local/etc/cic-ussd/
+COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic
+
+ENTRYPOINT []

apps/cic-ussd/docker/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-
-pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd

apps/cic-ussd/requirements.txt

@@ -1,10 +1,10 @@
 alembic==1.4.2
 bcrypt==3.2.0
 celery==4.4.7
-cic-eth[services]~=0.12.4a7
+cic-eth[services]==0.12.2a3
 cic-notify~=0.4.0a10
 cic-types~=0.1.0a14
-confini>=0.4.1a1,<0.5.0
+confini~=0.4.1a1
 phonenumbers==8.12.12
 psycopg2==2.8.6
 python-i18n[YAML]==0.3.9
@@ -13,5 +13,11 @@ redis==3.5.3
 semver==2.13.0
 SQLAlchemy==1.3.20
 tinydb==4.2.0
-transitions==0.8.4
+phonenumbers==8.12.12
+redis==3.5.3
+celery==4.4.7
+python-i18n[YAML]==0.3.9
+pyxdg==0.27
+bcrypt==3.2.0
 uWSGI==2.0.19.1
+transitions==0.8.4

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -75,21 +75,17 @@ def test_transaction_actors(activated_account, transaction_result, valid_recipie
 def test_validate_transaction_account(activated_account, init_database, transactions_list):
     sample_transaction = transactions_list[0]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
-    recipient_account = validate_transaction_account(
+    recipient_account = validate_transaction_account(init_database, recipient_transaction)
-        recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
+    sender_account = validate_transaction_account(init_database, sender_transaction)
-    sender_account = validate_transaction_account(
-        sender_transaction.get('blockchain_address'), sender_transaction.get('role'),  init_database)
     assert isinstance(recipient_account, Account)
     assert isinstance(sender_account, Account)
     sample_transaction = transactions_list[1]
     recipient_transaction, sender_transaction = transaction_actors(sample_transaction)
     with pytest.raises(UnknownUssdRecipient) as error:
-        validate_transaction_account(
+        validate_transaction_account(init_database, recipient_transaction)
-            recipient_transaction.get('blockchain_address'), recipient_transaction.get('role'), init_database)
     assert str(
         error.value) == f'Tx for recipient: {recipient_transaction.get("blockchain_address")} has no matching account in the system.'
-    validate_transaction_account(
+    validate_transaction_account(init_database, sender_transaction)
-        sender_transaction.get('blockchain_address'), sender_transaction.get('role'), init_database)
     assert f'Tx from sender: {sender_transaction.get("blockchain_address")} has no matching account in system.'
 
 

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -30,6 +30,8 @@ def test_is_valid_recipient(activated_account,
                             valid_recipient):
     state_machine = ('0112365478', generic_ussd_session, valid_recipient, init_database)
     assert is_valid_recipient(state_machine) is False
+    state_machine = (pending_account.phone_number, generic_ussd_session, valid_recipient, init_database)
+    assert is_valid_recipient(state_machine) is False
     state_machine = (valid_recipient.phone_number, generic_ussd_session, activated_account, init_database)
     assert is_valid_recipient(state_machine) is True