Merge branch 'master' into bvander/move-scripts-to-e2e-folder

.gitlab-ci.yml

@@ -6,6 +6,7 @@ include:
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
+  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[syncer]
-loop_interval = 1

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.14',
+        'beta.13',
         )
 
 version_object = semver.VersionInfo(

apps/contract-migration/docker/Dockerfile

@@ -87,7 +87,6 @@ COPY contract-migration/testdata/pgp testdata/pgp
 COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
 COPY contract-migration/keystore keystore
 COPY contract-migration/envlist .
-COPY contract-migration/scripts scripts/ 
 
 # A shared output dir for environment configs
 RUN mkdir -p /tmp/cic/config

apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py

@@ -1,76 +0,0 @@
-# external imports
-from chainlib.jsonrpc import JSONRPCException
-from eth_erc20 import ERC20
-from eth_accounts_index import AccountsIndex
-from eth_token_index import TokenUniqueSymbolIndex
-
-class ERC20Token:
-
-    def __init__(self, chain_spec, address, conn):
-        self.__address = address
-
-        c = ERC20(chain_spec)
-        o = c.symbol(address)
-        r = conn.do(o)
-        self.__symbol = c.parse_symbol(r)
-
-        o = c.decimals(address)
-        r = conn.do(o)
-        self.__decimals = c.parse_decimals(r)
-
-
-    def symbol(self):
-        return self.__symbol
-
-
-    def decimals(self):
-        return self.__decimals
-
-
-class IndexCache:
-
-    def __init__(self, chain_spec, address):
-        self.address = address
-        self.chain_spec = chain_spec
-
-    
-    def parse(self, r):
-        return r
-
-
-    def get(self, conn):
-        entries = []
-        i = 0
-        while True:
-            o = self.o.entry(self.address, i)
-            try:
-                r = conn.do(o)
-                entries.append(self.parse(r, conn))
-            except JSONRPCException:
-                return entries
-            i += 1
-
-
-class AccountRegistryCache(IndexCache):
-    
-    def __init__(self, chain_spec, address):
-        super(AccountRegistryCache, self).__init__(chain_spec, address)
-        self.o = AccountsIndex(chain_spec)
-        self.get_accounts = self.get
-
-
-    def parse(self, r, conn):
-        return self.o.parse_account(r)
-
-    
-class TokenRegistryCache(IndexCache):
-
-    def __init__(self, chain_spec, address):
-        super(TokenRegistryCache, self).__init__(chain_spec, address)
-        self.o = TokenUniqueSymbolIndex(chain_spec)
-        self.get_tokens = self.get
-
-
-    def parse(self, r, conn):
-        token_address = self.o.parse_entry(r)
-        return ERC20Token(self.chain_spec, token_address, conn)

apps/contract-migration/scripts/cic_eth/traffic/traffic.py

@@ -1,129 +0,0 @@
-# standard imports
-import os
-import logging
-import re
-import sys
-import json
-
-# external imports
-import redis
-import celery
-from cic_eth_registry.registry import CICRegistry
-from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
-from chainlib.eth.connection import EthHTTPConnection
-from chainlib.chain import ChainSpec
-from chainlib.eth.gas import RPCGasOracle
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import block_latest
-from hexathon import strip_0x
-from cic_base import (
-        argparse,
-        config,
-        log,
-        rpc,
-        signer as signer_funcs,
-        )
-
-# local imports
-#import common
-from cmd.traffic import (
-        TrafficItem,
-        TrafficRouter,
-        TrafficProvisioner,
-        TrafficSyncHandler,
-        )
-from cmd.traffic import add_args as add_traffic_args
-from cmd.cache import (
-        AccountRegistryCache,
-        TokenRegistryCache,
-        )
-
-
-# common basics
-script_dir = os.path.realpath(os.path.dirname(__file__))
-logg = log.create()
-argparser = argparse.create(script_dir, argparse.full_template)
-argparser = argparse.add(argparser, add_traffic_args, 'traffic')
-args = argparse.parse(argparser, logg)
-config = config.create(args.c, args, args.env_prefix)
-
-# map custom args to local config entries
-batchsize = args.batch_size
-if batchsize < 1:
-    batchsize = 1
-logg.info('batch size {}'.format(batchsize))
-config.add(batchsize, '_BATCH_SIZE', True)
-
-config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
-config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
-
-config.add(args.y, '_KEYSTORE_FILE', True)
-
-config.add(args.q, '_CELERY_QUEUE', True)
-
-logg.debug(config)
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-def main():
-    # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
-
-    # connect to celery
-    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-    # set up registry
-    rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
-    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
-    #registry = registry.init_legacy(config, w3)
-    CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
-    registry = CICRegistry(chain_spec, conn)
-
-    # Connect to blockchain with chainlib
-    gas_oracle = RPCGasOracle(conn)
-    nonce_oracle = RPCNonceOracle(signer_address, conn)
-
-    # Set up magic traffic handler
-    traffic_router = TrafficRouter()
-    traffic_router.apply_import_dict(config.all(), config)
-    handler = TrafficSyncHandler(config, traffic_router, conn)
-
-    # Set up syncer
-    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
-    o = block_latest()
-    r = conn.do(o)
-    block_offset = int(strip_0x(r), 16) + 1
-    syncer_backend.set(block_offset, 0)
-
-    # get relevant registry entries
-    token_registry = registry.lookup('TokenRegistry')
-    logg.info('using token registry {}'.format(token_registry))
-    token_cache = TokenRegistryCache(chain_spec, token_registry)
-
-    account_registry = registry.lookup('AccountRegistry')
-    logg.info('using account registry {}'.format(account_registry))
-    account_cache = AccountRegistryCache(chain_spec, account_registry)
-   
-    # Set up provisioner for common task input data
-    #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
-    TrafficProvisioner.oracles['token'] = token_cache
-    TrafficProvisioner.oracles['account'] = account_cache
-    
-    TrafficProvisioner.default_aux = {
-            'chain_spec': config.get('CIC_CHAIN_SPEC'),
-            'registry': registry,
-            'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
-            'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
-            'redis_db': config.get('REDIS_DB'),
-            'api_queue': config.get('_CELERY_QUEUE'),
-            }
-
-    syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
-    syncer.add_filter(handler)
-    syncer.loop(1, conn)
-
-
-if __name__ == '__main__':
-    main()

apps/data-seeding/.gitlab-ci.yml

@@ -0,0 +1,21 @@
+.data_seeding_variables:
+    variables:
+        APP_NAME: data-seeding
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.data_seeding_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
+
+build-mr-contract-migration:
+    extends:
+        - .data_seeding_changes_target
+        - .py_build_merge_request
+        - .data_seeding_variables
+
+build-push-contract-migration:
+    extends:
+        - .py_build_push
+        - .data_seeding_variables
+

apps/data-seeding/cic_eth/traffic/cmd/traffic.py

@@ -163,9 +163,9 @@ class TrafficProvisioner:
     """Aux parameter template to be passed to the traffic generator module"""
 
 
-    def __init__(self, conn):
-        self.tokens = self.oracles['token'].get_tokens(conn)
-        self.accounts = self.oracles['account'].get_accounts(conn)
+    def __init__(self):
+        self.tokens = self.oracles['token'].get_tokens()
+        self.accounts = self.oracles['account'].get_accounts()
         self.aux = copy.copy(self.default_aux)
         self.__balances = {}
         for a in self.accounts:
@@ -277,14 +277,13 @@ class TrafficSyncHandler:
     :type traffic_router: TrafficRouter
     :raises Exception: Any Exception redis may raise on connection attempt.
     """
-    def __init__(self, config, traffic_router, conn):
+    def __init__(self, config, traffic_router):
         self.traffic_router = traffic_router
         self.redis_channel = str(uuid.uuid4())
         self.pubsub = self.__connect_redis(self.redis_channel, config)
         self.traffic_items = {}
         self.config = config
         self.init = False
-        self.conn = conn
 
 
     # connects to redis
@@ -308,7 +307,7 @@ class TrafficSyncHandler:
         :param tx_index: Syncer block transaction index at time of call.
         :type tx_index: number
         """
-        traffic_provisioner = TrafficProvisioner(self.conn)
+        traffic_provisioner = TrafficProvisioner()
         traffic_provisioner.add_aux('redis_channel', self.redis_channel)
 
         refresh_accounts = None
@@ -344,7 +343,7 @@ class TrafficSyncHandler:
             sender = traffic_provisioner.accounts[sender_index]
             #balance_full = balances[sender][token_pair[0].symbol()]
             if len(sender_indices) == 1:
-                sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
+                sender_indices[m] = sender_sender_indices[len(senders)-1]
             sender_indices = sender_indices[:len(sender_indices)-1]
 
             balance_full = traffic_provisioner.balance(sender, token_pair[0])
@@ -352,14 +351,7 @@ class TrafficSyncHandler:
             recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
             recipient = traffic_provisioner.accounts[recipient_index]
           
-            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
-                traffic_item,
-                token_pair,
-                sender,
-                recipient,
-                balance_full,
-                )
-                )
+            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
             (e, t, balance_result,) = traffic_item.method(
                     token_pair,
                     sender,
@@ -367,6 +359,7 @@ class TrafficSyncHandler:
                     balance_full,
                     traffic_provisioner.aux,
                     block_number,
+                    tx_index,
                     )
             traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
             sender_indices.append(recipient_index)

apps/data-seeding/cic_eth/traffic/common/registry.py

@@ -3,7 +3,7 @@ import logging
 import copy
 
 # external imports
-from cic_registry.registry import Registry
+from cic_registry import CICRegistry
 from eth_token_index import TokenUniqueSymbolIndex
 from eth_accounts_index import AccountRegistry
 from chainlib.chain import ChainSpec

apps/data-seeding/cic_eth/traffic/common/signer.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports 
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
+from crypto_dev_signer.keystore import DictKeystore
 
 logg = logging.getLogger(__name__)
 

apps/data-seeding/cic_eth/traffic/local/account.py

@@ -11,7 +11,7 @@ queue = 'cic-eth'
 name = 'account'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Triggers creation and registration of new account through the custodial cic-eth component.
 
     It expects the following aux parameters to exist:

apps/data-seeding/cic_eth/traffic/local/noop.py

@@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
 
     If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@@ -26,10 +26,12 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     :type aux: dict
     :param block_number: Syncer block number position at time of method call
     :type block_number: number
+    :param tx_index: Syncer block transaction index position at time of method call
+    :type tx_index: number
     :raises KeyError: Missing required aux element
     :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
     :rtype: tuple
     """
-    logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
+    logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
 
     return (None, None, sender_balance, )

apps/data-seeding/cic_eth/traffic/local/transfer.py

@@ -12,7 +12,7 @@ queue = 'cic-eth'
 name = 'erc20_transfer'
 
 
-def do(token_pair, sender, recipient, sender_balance, aux, block_number):
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
     """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
 
     It expects the following aux parameters to exist:
@@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number):
     balance_units = int(sender_balance_value / decimals)
 
     if balance_units <= 0:
-        return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
+        return (AttributeError('sender {} has zero balance'), None, 0,)
 
     spend_units = random.randint(1, balance_units)
     spend_value = spend_units * decimals

apps/data-seeding/cic_eth/traffic/traffic.py

@@ -0,0 +1,101 @@
+# standard imports
+import os
+import logging
+import re
+import sys
+import json
+
+# external imports
+import redis
+import celery
+from chainsyncer.backend import MemBackend
+from chainsyncer.driver import HeadSyncer
+from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.gas import DefaultGasOracle
+from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.block import block_latest
+from hexathon import strip_0x
+
+# local imports
+import common
+from cmd.traffic import (
+        TrafficItem,
+        TrafficRouter,
+        TrafficProvisioner,
+        TrafficSyncHandler,
+        )
+from cmd.traffic import add_args as add_traffic_args
+
+
+# common basics
+script_dir = os.path.realpath(os.path.dirname(__file__))
+logg = common.log.create()
+argparser = common.argparse.create(script_dir, common.argparse.full_template)
+argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
+args = common.argparse.parse(argparser, logg)
+config = common.config.create(args.c, args, args.env_prefix)
+
+# map custom args to local config entries
+batchsize = args.batch_size
+if batchsize < 1:
+    batchsize = 1
+logg.info('batch size {}'.format(batchsize))
+config.add(batchsize, '_BATCH_SIZE', True)
+
+config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
+config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
+
+config.add(args.y, '_KEYSTORE_FILE', True)
+
+config.add(args.q, '_CELERY_QUEUE', True)
+
+common.config.log(config)
+
+
+def main():
+    # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
+    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
+
+    # connect to celery
+    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+    # set up registry
+    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    registry = common.registry.init_legacy(config, w3)
+
+    # Connect to blockchain with chainlib
+    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = DefaultGasOracle(conn)
+    nonce_oracle = DefaultNonceOracle(signer_address, conn)
+
+    # Set up magic traffic handler
+    traffic_router = TrafficRouter()
+    traffic_router.apply_import_dict(config.all(), config)
+    handler = TrafficSyncHandler(config, traffic_router)
+
+    # Set up syncer
+    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
+    o = block_latest()
+    r = conn.do(o)
+    block_offset = int(strip_0x(r), 16) + 1
+    syncer_backend.set(block_offset, 0)
+
+    # Set up provisioner for common task input data
+    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.default_aux = {
+            'chain_spec': config.get('CIC_CHAIN_SPEC'),
+            'registry': registry,
+            'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
+            'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
+            'redis_db': config.get('REDIS_DB'),
+            'api_queue': config.get('_CELERY_QUEUE'),
+            }
+
+    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
+    syncer.add_filter(handler)
+    syncer.loop(1, conn)
+
+
+if __name__ == '__main__':
+    main()

apps/data-seeding/config/traffic.ini

@@ -1,4 +1,4 @@
 [traffic]
 #local.noop_traffic = 2
-#local.account = 2
+local.account = 2
 local.transfer = 2

apps/data-seeding/docker/Dockerfile

@@ -0,0 +1,43 @@
+# syntax = docker/dockerfile:1.2
+FROM python:3.8.6-slim-buster as compile-image
+
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
+
+WORKDIR /root
+RUN mkdir -vp /usr/local/etc/cic
+
+COPY data-seeding/requirements.txt . 
+
+ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
+RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
+
+# -------------- begin runtime container ---------------- 
+FROM python:3.8.6-slim-buster as runtime-image
+
+RUN apt-get update
+RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
+RUN apt-get install -y jq bash iputils-ping socat telnet dnsutils
+
+COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
+COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
+COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
+                          /usr/local/lib/python3.8/site-packages/
+
+WORKDIR root/
+
+ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
+# RUN useradd -u 1001 --create-home grassroots
+# RUN adduser grassroots sudo && \
+#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+# WORKDIR /home/grassroots
+
+COPY data-seeding/ . 
+
+# we copied these from the root build container.
+# this is dumb though...I guess the compile image should have the same user
+# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
+
+# USER grassroots
+
+ENTRYPOINT [ ]

apps/data-seeding/verify.py

@@ -72,7 +72,6 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
-argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@@ -102,8 +101,6 @@ config.censor('PASSWORD', 'SSL')
 config.add(args.meta_provider, '_META_PROVIDER', True)
 config.add(args.ussd_provider, '_USSD_PROVIDER', True)
 
-token_symbol = args.token_symbol
-
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
@@ -276,10 +273,7 @@ class Verifier:
     def verify_balance(self, address, balance):
         o = self.erc20_tx_factory.balance(self.token_address, address)
         r = self.conn.do(o)
-        try:
-            actual_balance = int(strip_0x(r), 16)
-        except ValueError:
-            actual_balance = int(r)
+        actual_balance = int(strip_0x(r), 16)
         balance = int(balance / 1000000) * 1000000
         logg.debug('balance for {}: {}'.format(address, balance))
         if balance != actual_balance:
@@ -467,7 +461,7 @@ def main():
     tx = txf.template(ZERO_ADDRESS, token_index_address)
     data = add_0x(registry_addressof_method)
     h = hashlib.new('sha256')
-    h.update(token_symbol.encode('utf-8'))
+    h.update(b'SRF')
     z = h.digest()
     data += eth_abi.encode_single('bytes32', z).hex()
     txf.set_code(tx, data)