Merge remote-tracking branch 'origin/master' into lash/verify-details

apps/cic-cache/docker/Dockerfile

@@ -10,10 +10,9 @@ COPY requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt
 
 COPY . . 

apps/cic-cache/docker/Dockerfile_ci

@@ -10,9 +10,8 @@ COPY requirements.txt .
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt
 
 COPY . . 

apps/cic-cache/requirements.txt

@@ -10,6 +10,6 @@ celery==4.4.7
 redis==3.5.3
 chainsyncer[sql]>=0.0.6a1,<0.1.0
 erc20-faucet>=0.3.2a1, <0.4.0
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth==0.0.9a2
-chainlib>=0.0.9a2,<0.1.0
+chainlib==0.0.9a2
 eth-address-index>=0.2.3a1,<0.3.0

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a1,<0.7.0
 hexathon~=0.0.1a8
 chainqueue>=0.0.3a2,<0.1.0
-eth-erc20>=0.1.2a2,<0.2.0
+eth-erc20>=0.1.2a1,<0.2.0

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -101,14 +101,14 @@ class DispatchSyncer:
                     LockEnum.QUEUE,
                     tx['from'],
                     ],
-                queue=config.get('CELERY_QUEUE'),
+                queue=queue,
                 )
             s_send = celery.signature(
                     'cic_eth.eth.tx.send',
                     [
                         self.chain_spec.asdict(),
                         ], 
-                    queue=config.get('CELERY_QUEUE'),
+                    queue=queue,
                     )
             s_check.link(s_send)
             t = s_check.apply_async()

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         12,
         4,
-        'alpha.4',
+        'alpha.3',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -8,7 +8,6 @@ FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
@@ -19,7 +18,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt
@@ -33,7 +31,6 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
     cic-eth-aux-erc20-demurrage-token~=0.0.2a6
 
 COPY docker/entrypoints/* ./ 

apps/cic-eth/docker/Dockerfile_ci

@@ -9,7 +9,6 @@ WORKDIR /usr/src/cic-eth
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-ARG EXTRA_PIP_ARGS=""
 #RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
 #    pip install --index-url https://pypi.org/simple \
 #    --force-reinstall \
@@ -19,7 +18,6 @@ COPY *requirements.txt .
 RUN pip install --index-url https://pypi.org/simple  \
 	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
     --extra-index-url $EXTRA_INDEX_URL \
-    $EXTRA_PIP_ARGS \
     -r requirements.txt \
     -r services_requirements.txt \
     -r admin_requirements.txt

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth>=0.0.9a2,<0.1.0
 semver==2.13.0

apps/cic-eth/services_requirements.txt

@@ -6,10 +6,10 @@ redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.2.3a4,<0.3.0
+eth-address-index>=0.2.3a1,<0.3.0
-eth-accounts-index>=0.1.2a3,<0.2.0
+eth-accounts-index>=0.1.2a1,<0.2.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a1,<0.7.0
-erc20-faucet>=0.3.2a2,<0.4.0
+erc20-faucet>=0.3.2a1,<0.4.0
-erc20-transfer-authorization>=0.3.5a2,<0.4.0
+erc20-transfer-authorization>=0.3.5a1,<0.4.0
-sarafu-faucet>=0.0.7a2,<0.1.0
+sarafu-faucet>=0.0.7a1,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.1.2a2
+eth-erc20~=0.1.2a1

apps/cic-eth/tools_requirements.txt

@@ -1,6 +1,6 @@
 crypto-dev-signer>=0.4.15a1,<=0.4.15
 chainqueue>=0.0.3a1,<0.1.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a1,<0.7.0
 redis==3.5.3
 hexathon~=0.0.1a8
 pycryptodome==3.10.1

apps/contract-migration/config_template/redis.ini

@@ -1,4 +0,0 @@
-[redis]
-host =
-port =
-db =

apps/contract-migration/docker/Dockerfile

@@ -14,7 +14,16 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
+#RUN mkdir -vp $CONFINI_DIR
 
+#ARG cic_config_commit=24287fb253196820f23ff8a7177b122f2cd99a11
+#ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
+#RUN echo Install confini schema files && \
+#	git clone --depth 1 $cic_config_url cic-config && \
+#	cd cic-config && \
+#	git fetch --depth 1 origin $cic_config_commit && \
+#	git checkout $cic_config_commit && \	
+#	cp -v *.ini $CONFINI_DIR
 
 COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
@@ -27,16 +36,13 @@ ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages
 ARG pip_trusted_host=pypi.org
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple \
-    --pre \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r requirements.txt 
-
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple \
     --force-reinstall \
-    --pre \
     --trusted-host $pip_trusted_host \
     --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
     -r override_requirements.txt 

apps/contract-migration/docker/Dockerfile_ci

@@ -14,31 +14,28 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C20
 RUN mkdir -vp /usr/local/etc/cic
 
 ENV CONFINI_DIR /usr/local/etc/cic/
+RUN mkdir -vp $CONFINI_DIR
 
+ARG cic_config_commit=0abe0867f18077907c7023bf0ef5e466a3984dd8
+ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
+RUN echo Install confini schema files && \
+	git clone --depth 1 $cic_config_url cic-config && \
+	cd cic-config && \
+	git fetch --depth 1 origin $cic_config_commit && \
+	git checkout $cic_config_commit && \	
+	cp -v *.ini $CONFINI_DIR
 
-COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
-COPY override_requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG EXTRA_PIP_ARGS=""
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG pip_trusted_host=pypi.org
 RUN pip install --index-url https://pypi.org/simple \
-    pip install --index-url https://pypi.org/simple \
-    --pre \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
 
-RUN pip install --index-url https://pypi.org/simple \
-    --force-reinstall \
-    --pre \
-    --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    -r override_requirements.txt 
-
 COPY . .
 RUN chmod +x *.sh

apps/contract-migration/override_requirements.txt

@@ -1,4 +1,4 @@
-#eth-contract-registry==0.6.3a2
+eth-contract-registry==0.6.3a2
 #erc20-demurrage-token==0.0.2a3
 #eth-address-index==0.1.1a12
 

apps/contract-migration/requirements.txt

@@ -1,10 +1,10 @@
-cic-eth[tools]==0.12.4a4
+cic-eth[tools]==0.12.4a3
-chainlib-eth>=0.0.9a3,<0.1.0
+chainlib-eth>=0.0.9a2,<0.1.0
-eth-erc20>=0.1.2a2,<0.2.0
+eth-erc20>=0.1.2a1,<0.2.0
-erc20-demurrage-token>=0.0.5a2,<0.1.0
+erc20-demurrage-token>=0.0.5a1,<0.1.0
-eth-accounts-index>=0.1.2a2,<0.2.0
+eth-accounts-index>=0.1.2a1,<0.2.0
-eth-address-index>=0.2.3a4,<0.3.0
+eth-address-index>=0.2.3a3,<0.3.0
-cic-eth-registry>=0.6.1a2,<0.7.0
+cic-eth-registry>=0.6.1a1,<0.7.0
-erc20-transfer-authorization>=0.3.5a2,<0.4.0
+erc20-faucet>=0.3.2a1,<0.4.0
-erc20-faucet>=0.3.2a2,<0.4.0
+erc20-transfer-authorization>=0.3.5a1,<0.4.0
-sarafu-faucet>=0.0.7a2,<0.1.0
+sarafu-faucet>=0.0.7a1,<0.1.0

apps/data-seeding/cic_eth/import_balance.py

@@ -38,7 +38,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-from cic_eth.cli.chain import chain_interface
+from cic_base.eth.syncer import chain_interface
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
@@ -215,6 +215,8 @@ def main():
         logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
         sys.exit(1)
     logg.info('found token address {}'.format(token_address))
+    
+    sys.exit(0)
 
     syncer_backend = MemBackend(chain_str, 0)
 
@@ -246,7 +248,7 @@ def main():
 
     syncer_backend.set(block_offset, 0)
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
-    handler = Handler(conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle)
+    handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)
     

apps/data-seeding/cic_eth/traffic/cmd/traffic.py

@@ -140,11 +140,8 @@ class TrafficRouter:
         for k in keys:
             if len(k) > 8 and k[:8] == 'TRAFFIC_':
                 v = int(dct.get(k))
-                if v == 0:
+                self.add(k[8:].lower(), v)
-                    logg.debug('skipping traffic item {} with weight {}'.format(k, v))
+                logg.debug('found traffic item {} weight {}'.format(k, v))
-                else:
-                    logg.debug('found traffic item {} weight {}'.format(k, v))
-                    self.add(k[8:].lower(), v)
 
 
 # TODO: This will not work well with big networks. The provisioner should use lazy loading and LRU instead.

apps/data-seeding/cic_eth/traffic/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379

apps/data-seeding/cic_eth/traffic/config/redis.ini

@@ -1,3 +0,0 @@
-[redis]
-host = localhost
-port = 63379

apps/data-seeding/cic_eth/traffic/config/rpc.ini

@@ -1,2 +0,0 @@
-[rpc]
-http_provider = http://localhost:63545

apps/data-seeding/cic_eth/traffic/data/config/traffic.ini

@@ -1,4 +0,0 @@
-[traffic]
-#local.noop_traffic = 1
-local.account = 2
-local.transfer = 2

apps/data-seeding/cic_eth/traffic/traffic.py

@@ -17,10 +17,14 @@ from chainlib.eth.gas import RPCGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.block import block_latest
 from hexathon import strip_0x
-import chainlib.eth.cli
+from cic_base import (
-import cic_eth.cli
+        argparse,
-from cic_eth.cli.chain import chain_interface
+        config,
-from chainlib.eth.constant import ZERO_ADDRESS
+        log,
+        rpc,
+        signer as signer_funcs,
+        )
+from cic_base.eth.syncer import chain_interface
 
 # local imports
 #import common
@@ -38,45 +42,42 @@ from cmd.cache import (
 
 
 # common basics
-script_dir = os.path.dirname(os.path.realpath(__file__))
+script_dir = os.path.realpath(os.path.dirname(__file__))
-traffic_schema_dir = os.path.join(script_dir, 'data', 'config') 
+logg = log.create()
-logging.basicConfig(level=logging.WARNING)
+argparser = argparse.create(script_dir, argparse.full_template)
-logg = logging.getLogger()
+argparser = argparse.add(argparser, add_traffic_args, 'traffic')
+args = argparse.parse(argparser, logg)
+config = config.create(args.c, args, args.env_prefix)
 
-arg_flags = cic_eth.cli.argflag_std_read | cic_eth.cli.Flag.WALLET
+# map custom args to local config entries
-local_arg_flags = cic_eth.cli.argflag_local_taskcallback | cic_eth.cli.argflag_local_chain
+batchsize = args.batch_size
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+if batchsize < 1:
-argparser.add_argument('--batch-size', default=10, type=int, help='number of events to process simultaneously')
+    batchsize = 1
-argparser.process_local_flags(local_arg_flags)
+logg.info('batch size {}'.format(batchsize))
-args = argparser.parse_args()
+config.add(batchsize, '_BATCH_SIZE', True)
 
-extra_args = {
+config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
-    'batch_size': None,
+config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
-        }
-config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, base_config_dir=traffic_schema_dir, extra_args=extra_args)
 
-wallet = chainlib.eth.cli.Wallet()
+config.add(args.y, '_KEYSTORE_FILE', True)
-wallet.from_config(config)
 
-rpc = chainlib.eth.cli.Rpc(wallet=wallet)
+config.add(args.q, '_CELERY_QUEUE', True)
-conn = rpc.connect_by_config(config)
 
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+logg.debug(config)
-
-
-class NetworkError(Exception):
-    pass
 
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 def main():
     # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
-    signer = rpc.get_signer()
+    (signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
-    signer_address = rpc.get_sender_address()
 
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
     # set up registry
+    rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
+    #registry = registry.init_legacy(config, w3)
     CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
     registry = CICRegistry(chain_spec, conn)
 
@@ -90,7 +91,7 @@ def main():
     handler = TrafficSyncHandler(config, traffic_router, conn)
 
     # Set up syncer
-    syncer_backend = MemBackend(config.get('CHAIN_SPEC'), 0)
+    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
     o = block_latest()
     r = conn.do(o)
     block_offset = int(strip_0x(r), 16) + 1
@@ -98,28 +99,26 @@ def main():
 
     # get relevant registry entries
     token_registry = registry.lookup('TokenRegistry')
-    if token_registry == ZERO_ADDRESS:
-        raise NetworkError('TokenRegistry value missing from contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
     logg.info('using token registry {}'.format(token_registry))
     token_cache = TokenRegistryCache(chain_spec, token_registry)
 
     account_registry = registry.lookup('AccountRegistry')
-    if account_registry == ZERO_ADDRESS:
-        raise NetworkError('AccountRegistry value missing from contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
     logg.info('using account registry {}'.format(account_registry))
     account_cache = AccountRegistryCache(chain_spec, account_registry)
    
     # Set up provisioner for common task input data
+    #TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    #TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
     TrafficProvisioner.oracles['token'] = token_cache
     TrafficProvisioner.oracles['account'] = account_cache
     
     TrafficProvisioner.default_aux = {
-            'chain_spec': config.get('CHAIN_SPEC'),
+            'chain_spec': config.get('CIC_CHAIN_SPEC'),
             'registry': registry,
             'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
             'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
             'redis_db': config.get('REDIS_DB'),
-            'api_queue': config.get('CELERY_QUEUE'),
+            'api_queue': config.get('_CELERY_QUEUE'),
             }
 
     syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=handler.refresh)

apps/data-seeding/config/traffic.ini

@@ -0,0 +1,4 @@
+[traffic]
+#local.noop_traffic = 2
+#local.account = 2
+local.transfer = 2

apps/data-seeding/eth/import_balance.py

@@ -37,6 +37,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
+#from cic_base.eth.syncer import chain_interface
 from cic_eth.cli.chain import chain_interface
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry

apps/data-seeding/requirements.txt

@@ -1,13 +1,13 @@
 sarafu-faucet~=0.0.7a1
-cic-eth[tools]~=0.12.4a4
+cic-eth[tools]~=0.12.4a2
 cic-types~=0.1.0a14
 crypto-dev-signer>=0.4.15a1,<=0.4.15
 faker==4.17.1
 chainsyncer~=0.0.6a1
-chainlib-eth~=0.0.9a4
+chainlib-eth~=0.0.9a2
-eth-address-index~=0.2.3a4
+eth-address-index~=0.2.3a1
-eth-contract-registry~=0.6.3a3
+eth-contract-registry~=0.6.3a1
-eth-accounts-index~=0.1.2a3
+eth-accounts-index~=0.1.2a1
-eth-erc20~=0.1.2a2
+eth-erc20~=0.1.2a1
-erc20-faucet~=0.3.2a2
+erc20-faucet~=0.3.2a1
 psycopg2==2.8.6