Refactor import scripts

apps/cic-eth/cic_eth/api/api_task.py

@@ -6,10 +6,13 @@
 # standard imports
 import logging
 
-# third-party imports 
+# external imports 
 import celery
-from cic_registry.chain import ChainSpec
+#from cic_registry.chain import ChainSpec
 from cic_registry import CICRegistry
+from chainlib.chain import ChainSpec
+
+# local imports
 from cic_eth.eth.factory import TxFactory
 from cic_eth.db.enum import LockEnum
 

apps/cic-eth/cic_eth/callbacks/redis.py

@@ -18,7 +18,11 @@ logg = celery_app.log.get_default_logger()
 def redis(self, result, destination, status_code):
     (host, port, db, channel) = destination.split(':')
     r = redis_interface.Redis(host=host, port=port, db=db)
-    s = json.dumps(result)
+    data = {
+            'root_id': self.request.root_id,
+            'status': status_code,
+            'result': result,
+            }
     logg.debug('redis callback on host {} port {} db {} channel {}'.format(host, port, db, channel))
-    r.publish(channel, s)
+    r.publish(channel, json.dumps(data))
     r.close()

apps/cic-eth/cic_eth/db/models/base.py

@@ -114,6 +114,7 @@ class SessionBase(Model):
 
     @staticmethod
     def release_session(session=None):
+        session.flush()
         session_key = str(id(session))
         if SessionBase.localsessions.get(session_key) != None:
             logg.debug('destroying session {}'.format(session_key))

apps/cic-eth/cic_eth/db/models/role.py

@@ -24,9 +24,10 @@ class AccountRole(SessionBase):
     tag = Column(Text)
     address_hex = Column(String(42))
 
-
+    
+    # TODO: 
     @staticmethod
-    def get_address(tag):
+    def get_address(tag, session):
         """Get Ethereum address matching the given tag
 
         :param tag: Tag
@@ -34,14 +35,24 @@ class AccountRole(SessionBase):
         :returns: Ethereum address, or zero-address if tag does not exist
         :rtype: str, 0x-hex
         """
-        role = AccountRole.get_role(tag)
+        if session == None:
-        if role == None:
+            raise ValueError('nested bind session calls will not succeed as the first call to release_session in the stack will leave the db object detached further down the stack. We will need additional reference count.')
-            return zero_address
+
-        return role.address_hex
+        session = SessionBase.bind_session(session)
+
+        role = AccountRole.get_role(tag, session)
+    
+        r = zero_address
+        if role != None:
+            r = role.address_hex
+
+        SessionBase.release_session(session)
+
+        return r
 
 
     @staticmethod
-    def get_role(tag):
+    def get_role(tag, session=None):
         """Get AccountRole model object matching the given tag
 
         :param tag: Tag
@@ -49,20 +60,26 @@ class AccountRole(SessionBase):
         :returns: Role object, if found
         :rtype: cic_eth.db.models.role.AccountRole
         """
-        session = AccountRole.create_session()
+        session = SessionBase.bind_session(session)
-        role = AccountRole.__get_role(session, tag)
+        
-        session.close()
+        role = AccountRole.__get_role(tag, session)
-        #return role.address_hex
+        
+        SessionBase.release_session(session)
+
         return role
 
 
     @staticmethod
-    def __get_role(session, tag):
+    def __get_role(tag, session):
-        return session.query(AccountRole).filter(AccountRole.tag==tag).first()
+        q = session.query(AccountRole)
+        q = q.filter(AccountRole.tag==tag)
+        r = q.first()
+        session.flush()
+        return r
 
 
     @staticmethod    
-    def set(tag, address_hex):
+    def set(tag, address_hex, session=None):
         """Persist a tag to Ethereum address association. 
 
         This will silently overwrite the existing value.
@@ -74,16 +91,16 @@ class AccountRole(SessionBase):
         :returns: Role object
         :rtype: cic_eth.db.models.role.AccountRole
         """
-        #session = AccountRole.create_session()
+        session = SessionBase.bind_session(session)
-        #role = AccountRole.__get(session, tag)
+        
-        role = AccountRole.get_role(tag) #session, tag)
+        role = AccountRole.get_role(tag, session)
         if role == None:
             role = AccountRole(tag)
         role.address_hex = address_hex
-        #session.add(role)
+        
-        #session.commit()
+        SessionBase.release_session(session)
-        #session.close()
+
-        return role #address_hex
+        return role 
 
 
     @staticmethod
@@ -95,20 +112,17 @@ class AccountRole(SessionBase):
         :returns: Role tag, or None if no match
         :rtype: str or None
         """
-        localsession = session
+        session = SessionBase.bind_session(session)
-        if localsession == None:
-            localsession = SessionBase.create_session()
 
-        q = localsession.query(AccountRole)
+        q = session.query(AccountRole)
         q = q.filter(AccountRole.address_hex==address)
         role = q.first()
         tag = None
         if role != None:
             tag = role.tag
 
-        if session == None:
+        SessionBase.release_session(session)
-            localsession.close()
+
-       
         return tag
 
 

apps/cic-eth/cic_eth/eth/gas.py

@@ -52,7 +52,10 @@ class GasOracle():
         :returns: Etheerum account address
         :rtype: str, 0x-hex
         """
-        return AccountRole.get_address('GAS_GIFTER')
+        session = SessionBase.create_session()
+        a = AccountRole.get_address('GAS_GIFTER', session)
+        session.close()
+        return a
 
 
     def gas_price(self, category='safe'):

apps/cic-eth/cic_eth/eth/tx.py

@@ -399,7 +399,7 @@ def refill_gas(self, recipient_address, chain_str):
     q = session.query(Otx.tx_hash)
     q = q.join(TxCache)
     q = q.filter(Otx.status.op('&')(StatusBits.FINAL.value)==0)
-    q = q.filter(TxCache.from_value!='0x00')
+    q = q.filter(TxCache.from_value!=0)
     q = q.filter(TxCache.recipient==recipient_address)
     c = q.count()
     session.close()

apps/cic-eth/cic_eth/runnable/create.py

@@ -76,8 +76,9 @@ def main():
     t = api.create_account(register=register)
 
     ps.get_message()
-    m = ps.get_message(timeout=args.timeout)
+    o = ps.get_message(timeout=args.timeout)
-    print(json.loads(m['data']))
+    m = json.loads(o['data'])
+    print(m['result'])
 
 
 if __name__ == '__main__':

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -91,6 +91,8 @@ run = True
 
 class DispatchSyncer:
 
+    yield_delay = 0.005
+
     def __init__(self, chain_spec):
         self.chain_spec = chain_spec
         self.chain_id = chain_spec.chain_id()
@@ -138,7 +140,10 @@ class DispatchSyncer:
                 txs[k] = utxs[k]
             self.process(w3, txs)
 
-            time.sleep(interval)
+            if len(utxs) > 0:
+                time.sleep(self.yield_delay)
+            else:
+                time.sleep(interval)
 
 
 def main(): 

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -5,9 +5,10 @@ import logging
 from cic_registry.chain import ChainSpec
 
 # local imports
+from cic_eth.db.enum import StatusBits
 from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.tx import TxCache
-from cic_eth.db import Otx
+from cic_eth.db.models.otx import Otx
 from cic_eth.queue.tx import get_paused_txs
 from cic_eth.eth.task import create_check_gas_and_send_task
 from .base import SyncFilter
@@ -39,7 +40,7 @@ class GasFilter(SyncFilter):
                 return
 
             chain_spec = ChainSpec.from_chain_str(chain_str)
-            txs = get_paused_txs(StatusEnum.WAITFORGAS, r[0], chain_spec.chain_id(), session=session)
+            txs = get_paused_txs(StatusBits.GAS_ISSUES, r[0], chain_spec.chain_id(), session=session)
 
             SessionBase.release_session(session)
 

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -15,6 +15,10 @@ account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153f
 
 class RegistrationFilter(SyncFilter):
 
+    def __init__(self, queue):
+        self.queue = queue
+
+
     def filter(self, w3, tx, rcpt, chain_spec, session=None):
         logg.debug('applying registration filter')
         registered_address = None
@@ -30,6 +34,6 @@ class RegistrationFilter(SyncFilter):
                         address,
                         str(chain_spec),
                         ],
-                    queue=queue,
+                    queue=self.queue,
                     )
                 s.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/manager.py

@@ -178,7 +178,7 @@ def main():
 
     tx_filter = TxFilter(queue)
 
-    registration_filter = RegistrationFilter()
+    registration_filter = RegistrationFilter(queue)
 
     gas_filter = GasFilter(c.gas_provider(), queue)
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -32,6 +32,7 @@ from cic_eth.admin import ctrl
 from cic_eth.eth.rpc import RpcClient
 from cic_eth.eth.rpc import GasOracle
 from cic_eth.queue import tx
+from cic_eth.queue import balance
 from cic_eth.callbacks import Callback
 from cic_eth.callbacks import http
 from cic_eth.callbacks import tcp
@@ -49,6 +50,7 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', type=str, help='web3 provider')
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
+argparser.add_argument('-r', type=str, help='CIC registry address')
 argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -68,6 +70,7 @@ config.process()
 args_override = {
         'ETH_ABI_DIR': getattr(args, 'abi_dir'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
@@ -228,7 +231,7 @@ def main():
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
     oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
-    chain_registry.add_oracle('naive_erc20_oracle', oracle)
+    chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
 
 
     #chain_spec = CICRegistry.default_chain_spec

apps/cic-eth/cic_eth/sync/history.py

@@ -21,7 +21,7 @@ class HistorySyncer(MinedSyncer):
     :param mx: Maximum number of blocks to return in one call
     :type mx: int
     """
-    def __init__(self, bc_cache, mx=20):
+    def __init__(self, bc_cache, mx=500):
         super(HistorySyncer, self).__init__(bc_cache)
         self.max = mx
 

apps/cic-eth/cic_eth/sync/mined.py

@@ -23,6 +23,8 @@ class MinedSyncer(Syncer):
     :type bc_cache: Object implementing methods from cic_eth.sync.SyncerBackend 
     """
 
+    yield_delay = 0.005
+
     def __init__(self, bc_cache):
         super(MinedSyncer, self).__init__(bc_cache)
         self.block_offset = 0
@@ -100,5 +102,8 @@ class MinedSyncer(Syncer):
                 block_number = self.process(c.w3, block.hex())
                 logg.info('processed block {} {}'.format(block_number, block.hex()))
             self.bc_cache.disconnect()
-            time.sleep(interval)
+            if len(e) > 0:
+                time.sleep(self.yield_delay)
+            else:
+                time.sleep(interval)
         logg.info("Syncer no longer set to run, gracefully exiting")

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.27',
+        'alpha.30',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -2,4 +2,4 @@
 registry_address = 
 chain_spec = 
 tx_retry_delay = 
-trust_address = 
+trust_address =

apps/cic-eth/config/docker/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir = /usr/local/share/cic/bancor

apps/cic-eth/config/docker/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis://localhost:63379
+result_url = redis://localhost:63379

apps/cic-eth/config/docker/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+chain_spec = evm:bloxberg:8996
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-eth/config/docker/custody.ini

@@ -0,0 +1,2 @@
+[custody]
+account_index_address =

apps/cic-eth/config/docker/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_eth
+USER=postgres
+PASSWORD=tralala
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=1

apps/cic-eth/config/docker/dispatcer.ini

@@ -0,0 +1,2 @@
+[dispatcher]
+loop_interval = 0.9

apps/cic-eth/config/docker/eth.ini

@@ -0,0 +1,8 @@
+[eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
+provider = http://localhost:63545
+gas_provider_address = 
+#chain_id = 
+abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
+account_accounts_index_writer = 

apps/cic-eth/config/docker/redis.ini

@@ -0,0 +1,4 @@
+[redis]
+host = localhost
+port = 63379
+db = 0

apps/cic-eth/config/docker/signer.ini

@@ -0,0 +1,5 @@
+[signer]
+socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
+secret = deedbeef
+database_name = signer_test
+dev_keys_path = 

apps/cic-eth/config/docker/ssl.ini

@@ -0,0 +1,6 @@
+[SSL]
+enable_client = false
+cert_file = 
+key_file =
+password = 
+ca_file =

apps/cic-eth/config/docker/syncer.ini

@@ -0,0 +1,2 @@
+[SYNCER]
+loop_interval = 1

apps/cic-eth/config/docker/tasks.ini

@@ -0,0 +1,3 @@
+[tasks]
+transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
+trace_queue_status = 1

apps/cic-eth/requirements.txt

@@ -2,13 +2,13 @@ web3==5.12.2
 celery==4.4.7
 crypto-dev-signer~=0.4.13rc2
 confini~=0.3.6b1
-cic-registry~=0.5.3a18
+cic-registry~=0.5.3a20
 cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.10a7
+eth_accounts_index~=0.0.10a10
 erc20-approval-escrow~=0.3.0a5
 erc20-single-shot-faucet~=0.2.0a6
 rlp==2.0.1
@@ -18,5 +18,5 @@ eth-gas-proxy==0.0.1a4
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.0a8
-chainlib~=0.0.1a12
+chainlib~=0.0.1a16
 hexathon~=0.0.1a3

apps/cic-meta/CHANGELOG

@@ -1,3 +1,6 @@
+* 0.0.7-pending
+	- Add immutable content
+	- Add db lock on server
 * 0.0.6
 	- Add server build
 * 0.0.5

apps/cic-meta/package.json

@@ -1,18 +1,16 @@
 {
 	"name": "cic-client-meta",
-	"version": "0.0.6",
+	"version": "0.0.7-alpha.2",
 	"description": "Signed CRDT metadata graphs for the CIC network",
 	"main": "dist/index.js",
 	"types": "dist/index.d.ts",
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
-		"build": "node_modules/typescript/bin/tsc -d --outDir dist",
+		"build": "node_modules/typescript/bin/tsc -d --outDir dist && npm run build-server",
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
-		"clean": "rm -rf dist"
+		"clean": "rm -rf dist",
-	},
+		"prepare": "npm run build && npm run build-server"
-	"bin": {
-		"cic-meta-server": "./dist-server/scripts/server/server.js"
 	},
 	"dependencies": {
 		"@ethereumjs/tx": "^3.0.0-beta.1",

apps/cic-meta/scripts/server/server.ts

@@ -137,6 +137,7 @@ async function processRequest(req, res) {
 		console.debug('mode', mod);
 		let content = '';
 		let contentType = 'application/json';
+		console.debug('handling data', data);
 		let r:any = undefined;
 		try {
 			switch (mod) {
@@ -192,14 +193,16 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
+			console.error('empty onctent', data);
 			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;
 		}
 
+		const responseContentLength = (new TextEncoder().encode(content)).length;
 		res.writeHead(200, {
 			"Content-Type": contentType,
-			"Content-Length": content.length,
+			"Content-Length": responseContentLength,
 		});
 		res.write(content);
 		res.end();

apps/cic-meta/src/index.ts

@@ -2,3 +2,4 @@ export { PGPSigner, PGPKeyStore } from './auth';
 export { Envelope, Syncable } from './sync';
 export { User } from './assets/user';
 export { Phone } from './assets/phone';
+export { Config } from './config';

apps/contract-migration/dev/balance.py

@@ -1,58 +0,0 @@
-#!python3
-
-"""Gas transfer script
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746 
-
-"""
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import os
-import json
-import argparse
-import logging
-
-# third-party imports
-import web3
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-logging.getLogger('web3').setLevel(logging.WARNING)
-logging.getLogger('urllib3').setLevel(logging.WARNING)
-
-default_abi_dir = os.environ.get('ETH_ABI_DIR', '/usr/share/local/cic/solidity/abi')
-default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', default=default_eth_provider, type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-t', '--token-address', dest='t', type=str, help='Token address. If not set, will return gas balance')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, default=default_abi_dir, help='Directory containing bytecode and abi (default {})'.format(default_abi_dir))
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('account', type=str, help='Account address')
-args = argparser.parse_args()
-
-
-if args.v:
-    logg.setLevel(logging.DEBUG)
-
-def main():
-    w3 = web3.Web3(web3.Web3.HTTPProvider(args.p))
-
-    balance = None
-    if args.t != None:
-        f = open(os.path.join(args.abi_dir, 'ERC20.json'))
-        abi = json.load(f)
-        f.close()
-        c = w3.eth.contract(abi=abi, address=args.t)
-        balance = c.functions.balanceOf(args.account).call()
-    else:
-        balance =w3.eth.getBalance(args.account)
-
-    print(balance)
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/dev/create.py

@@ -1,47 +0,0 @@
-#!/usr/bin/python
-import sys
-import os
-import logging
-
-import celery
-from cic_eth.api import Api
-import confini
-import argparse
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger('create_account_script')
-logging.getLogger('confini').setLevel(logging.WARNING)
-logging.getLogger('gnupg').setLevel(logging.WARNING)
-
-config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logg.setLevel(logging.DEBUG)
-if args.v:
-    logg.setLevel(logging.INFO)
-
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-api = Api(config.get('CIC_CHAIN_SPEC'))
-
-registration_account = None
-#t = api.create_account(registration_account=registration_account)
-if len(sys.argv) > 1:
-    registration_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER', None)
-
-logg.debug('accounts index writer NOT USED {}'.format(registration_account))
-
-register = not args.no_register
-logg.debug('register {}'.format(register))
-t = api.create_account(register=register)
-
-print(t.get())

apps/contract-migration/dev/decode.py

@@ -1,48 +0,0 @@
-#!python3
-
-"""Decode raw transaction
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746 
-
-"""
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import os
-import json
-import argparse
-import logging
-
-# third-party imports
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_abi_dir = os.environ.get('ETH_ABI_DIR', '/usr/share/local/cic/solidity/abi')
-default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-i', '--chain-id', dest='i', type=int, help='Numeric network id')
-argparser.add_argument('tx', type=str, help='hex-encoded signed raw transaction')
-args = argparser.parse_args()
-
-
-if args.v:
-    logg.setLevel(logging.DEBUG)
-
-def main():
-    tx_raw = args.tx
-    if tx_raw[:2] == '0x':
-        tx_raw = tx_raw[2:]
-    tx_raw_bytes = bytes.fromhex(tx_raw)
-    tx = unpack_signed_raw_tx(tx_raw_bytes, args.i)
-    for k in tx.keys():
-        print('{}: {}'.format(k, tx[k]))
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/dev/env.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-cic_data_dir=${CIC_DATA_DIR:-/tmp/cic}
-t=${1:-$(mktemp)}
-prefix=''
-if [ ! -z $2 ]; then
-	prefix="${2}_"
-fi
-
-echo "#!/bin/bash" > $t
-echo "set +a" >> $t
-cat $cic_data_dir/.env | sed -e "s/^\([A-Z]\)/export ${prefix}\1/g" >> $t
-
-#if [ -f $cic_data_dir/.env ]; then
-#cat $cic_data_dir/.env | sed -e "s/^\([A-Z]\)/export ${prefix}\1/g"  >> $t
-#fi
-echo "export CONFINI_DIR=$(dirname $(realpath .))/config_template" >> $t
-source $t
-echo "export CELERY_BROKER_URL=redis://localhost:${HTTP_PORT_REDIS}" >> $t
-echo "export CELERY_RESULT_URL=redis://localhost:${HTTP_PORT_REDIS}" >> $t
-echo "export ETH_PROVIDER=http://localhost:${HTTP_PORT_ETH}" >> $t
-echo "export META_PROVIDER=http://localhost:${HTTP_PORT_CIC_META}" >> $t
-echo "set -a" >> $t
-echo $t

apps/contract-migration/dev/gas.py

@@ -1,105 +0,0 @@
-#!python3
-
-"""Gas transfer script
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746 
-
-"""
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import os
-import json
-import argparse
-import logging
-
-# third-party imports
-import web3
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore import DictKeystore
-from crypto_dev_signer.eth.helper import EthTxExecutor
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-logging.getLogger('web3').setLevel(logging.WARNING)
-logging.getLogger('urllib3').setLevel(logging.WARNING)
-
-default_abi_dir = '/usr/share/local/cic/solidity/abi'
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
-argparser.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
-argparser.add_argument('-a', '--signer-address', dest='a', type=str, help='Signing address')
-argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-argparser.add_argument('recipient', type=str, help='Ethereum address of recipient')
-argparser.add_argument('amount', type=int, help='Amount of tokens to mint and gift')
-args = argparser.parse_args()
-
-
-if args.vv:
-    logg.setLevel(logging.DEBUG)
-elif args.v:
-    logg.setLevel(logging.INFO)
-
-block_last = args.w
-block_all = args.ww
-
-w3 = web3.Web3(web3.Web3.HTTPProvider(args.p))
-
-signer_address = None
-keystore = DictKeystore()
-if args.y != None:
-    logg.debug('loading keystore file {}'.format(args.y))
-    signer_address = keystore.import_keystore_file(args.y)
-    logg.debug('now have key for signer address {}'.format(signer_address))
-signer = EIP155Signer(keystore)
-
-chain_pair = args.i.split(':')
-chain_id = int(chain_pair[1])
-
-helper = EthTxExecutor(
-        w3,
-        signer_address,
-        signer,
-        chain_id,
-        block=args.ww,
-    )
-
-
-def build_gas_transaction(recipient, value):
-    def builder(tx):
-        tx['to'] = recipient
-        tx['value'] = value
-        tx['data'] = '0x'
-        return tx
-    return builder
-
-
-def main():
-    recipient = args.recipient
-    value = args.amount
-
-    logg.debug('sender {} balance before: {}'.format(signer_address, w3.eth.getBalance(signer_address)))
-    logg.debug('recipient {} balance before: {}'.format(recipient, w3.eth.getBalance(recipient)))
-    (tx_hash, rcpt) = helper.sign_and_send(
-            [
-                build_gas_transaction(recipient, value),
-                ],
-            )
-    logg.debug('sender {} balance after: {}'.format(signer_address, w3.eth.getBalance(signer_address)))
-    logg.debug('recipient {} balance after: {}'.format(recipient, w3.eth.getBalance(recipient)))
-
-    if block_last:
-        helper.wait_for()
-
-    print(tx_hash)
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/dev/js/register_users.js

@@ -1,101 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const cic = require('cic-client-meta');
-const http = require('http');
-const confini = require('confini');
-
-console.debug('sorry this script doesnt read cli flags, set all in env vars');
-
-let config_data_dir = process.env.CONFINI_DIR;
-if (config_data_dir === undefined) {
-	config_data_dir = '/usr/local/etc/cic';
-}
-const config = new confini.Config(config_data_dir, process.env.CONFINI_ENV_PREFIX);
-config.process();
-Object.keys(config.store).forEach((k) => {
-	console.debug(k, config.get(k));
-});
-
-// flatten file list from directories recursively
-// cheekily though gratefully stolen from https://coderrocketfuel.com/article/recursively-list-all-the-files-in-a-directory-using-node-js 
-const getAllFiles = function(dirPath, arrayOfFiles) {
-  files = fs.readdirSync(dirPath)
-
-  arrayOfFiles = arrayOfFiles || []
-
-  files.forEach(function(file) {
-    if (fs.statSync(dirPath + "/" + file).isDirectory()) {
-      arrayOfFiles = getAllFiles(dirPath + "/" + file, arrayOfFiles)
-    } else {
-      arrayOfFiles.push(path.join(dirPath, "/", file))
-    }
-  })
-
-  return arrayOfFiles
-}
-
-async function sendit(uid, envelope) {
-	const d = envelope.toJSON();
-
-	const opts = {
-		method: 'PUT',
-		headers: {
-			'Content-Type': 'application/json',
-			'Content-Length': d.length,
-			'X-CIC-AUTOMERGE': 'client',
-
-		},
-	};
-	let url = config.get('META_PROVIDER'); //['archiveUrl'];
-	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
-	const req = http.request(url + uid, opts, (res) => {
-		res.on('data', process.stdout.write);
-		res.on('end', () => {
-			console.log('result', res.statusCode, res.headers);
-		});
-	});
-
-	req.write(d);
-	req.end();
-}
-
-function doit(keystore) {
-	dataDir = 'data';
-	if (process.argv.length > 2) {
-		dataDir = process.argv[2];
-	}
-	console.log('argv', process.argv);
-	console.log('datadir', path.join(dataDir, 'person'));
-	getAllFiles(path.join(dataDir, 'person')).forEach((filename) => {
-		console.debug('person file', filename);
-		const signer = new cic.PGPSigner(keystore);
-		const parts = filename.split('.');
-		const uid = path.basename(parts[0]);
-		
-		const d = fs.readFileSync(filename, 'utf-8');
-		const o = JSON.parse(d);
-
-		const s = new cic.Syncable(uid, o);
-		console.log(s);
-		s.setSigner(signer);
-		s.onwrap = (env) => {
-			console.log('env', env);
-			//console.log('sign', s.m.signature.digest);
-			sendit(uid, env);
-		};
-		s.sign();
-	});
-}
-
-pk = fs.readFileSync(path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATEKEY_FILE')));
-pubk = fs.readFileSync(path.join(config.get('PGP_EXPORTS_DIR'), config.get('DEV_PGP_PUBLICKEYS_ACTIVE_FILE')));
-
-new cic.PGPKeyStore(
-	process.env['PGP_PASSPHRASE'],
-	pk,
-	pubk,
-	undefined,
-	undefined,
-	doit,
-);
-

apps/contract-migration/dev/keys.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-mkdir -vp .tmp
-echo -n '' > .tmp/.env_accounts
-account_labels=(
-	DEV_ETH_ACCOUNT_BANCOR_DEPLOYER
-	DEV_ETH_ACCOUNT_GAS_PROVIDER
-	DEV_ETH_ACCOUNT_RESERVE_OWNER
-	DEV_ETH_ACCOUNT_RESERVE_MINTER
-	DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER
-	DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
-	DEV_ETH_ACCOUNT_SARAFU_OWNER
-	DEV_ETH_ACCOUNT_SARAFU_GIFTER
-	DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
-	DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
-)
-bip39gen -n ${#account_labels[@]} "$DEV_MNEMONIC"
-i=0
-for a in `bip39gen -n ${#account_labels[@]} "$DEV_MNEMONIC" | jq -r .address[]`; do
-	exportline=${account_labels[$i]}=$a
-	export $exportline
-	echo $exportline >> .tmp/.env_accounts
-	echo exportline $exportline
-	i=$(($i+1))
-done
-

apps/contract-migration/dev/python/scripts/setup.py

@@ -1,4 +0,0 @@
-from setuptools import setup
-
-setup(
-        )

apps/contract-migration/dev/python/scripts/tx_driver.py

@@ -1,150 +0,0 @@
-# standard imports
-import os
-import logging
-import argparse
-import re
-import json
-import signal
-import random
-import time
-
-# third-party imports
-import confini
-import web3
-from cic_registry.chain import ChainSpec
-from cic_registry.chain import ChainRegistry
-from cic_registry import CICRegistry
-from eth_token_index import TokenUniqueSymbolIndex as TokenIndex
-from eth_accounts_index import AccountRegistry
-
-from cic_eth.api import Api
-
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
-logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
-
-default_data_dir = '/usr/local/share/cic/solidity/abi'
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default='./config', help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, default=default_data_dir, help='Directory containing bytecode and abi (default: {})'.format(default_data_dir))
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-argparser.add_argument('--wait-max', dest='wait_max', default=2.0, type=float, help='maximum time in decimal seconds to wait between transactions')
-argparser.add_argument('--account-index-address', dest='account_index', type=str, help='Contract address of accounts index')
-argparser.add_argument('--token-index-address', dest='token_index', type=str, help='Contract address of token index')
-argparser.add_argument('--approval-escrow-address', dest='approval_escrow', type=str, help='Contract address for transfer approvals')
-argparser.add_argument('--declarator-address', dest='declarator', type=str, help='Address of declarations contract to perform lookup against')
-argparser.add_argument('-a', '--accounts-index-writer', dest='a', type=str, help='Address of account with access to add to accounts index')
-
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'DEV_ETH_ACCOUNTS_INDEX_ADDRESS': getattr(args, 'account_index'),
-        'DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER': getattr(args, 'a'),
-        'DEV_ETH_ERC20_APPROVAL_ESCROW_ADDRESS': getattr(args, 'approval_escrow'),
-        'DEV_ETH_TOKEN_INDEX_ADDRESS': getattr(args, 'token_index'),
-        }
-config.dict_override(args_override, 'cli flag')
-config.validate()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-re_websocket = r'^wss?:'
-re_http = r'^https?:'
-blockchain_provider = None
-if re.match(re_websocket, config.get('ETH_PROVIDER')):
-    blockchain_provider = web3.Web3.WebsocketProvider(config.get('ETH_PROVIDER'))
-elif re.match(re_http, config.get('ETH_PROVIDER')):
-    blockchain_provider = web3.Web3.HTTPProvider(config.get('ETH_PROVIDER'))
-w3 = web3.Web3(blockchain_provider)
-
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-CICRegistry.init(w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-
-chain_registry = ChainRegistry(chain_spec)
-CICRegistry.add_chain_registry(chain_registry, True)
-
-run = True
-
-def inthandler(name, frame):
-    logg.warning('got {}, stopping'.format(name))
-    global run
-    run = False
-
-signal.signal(signal.SIGTERM, inthandler)
-signal.signal(signal.SIGINT, inthandler)
-
-api = Api(str(chain_spec))
-
-f = open(os.path.join(config.get('ETH_ABI_DIR'), 'ERC20.json'))
-erc20_abi = json.load(f)
-f.close()
-
-def get_tokens():
-    tokens = []
-    token_index = TokenIndex(w3, config.get('CIC_TOKEN_INDEX_ADDRESS'))
-    token_count = token_index.count()
-    for i in range(token_count):
-        tokens.append(token_index.get_index(i))
-    logg.debug('tokens {}'.format(tokens))
-    return tokens
-
-def get_addresses():
-    address_index = AccountRegistry(w3, config.get('CIC_ACCOUNTS_INDEX_ADDRESS'))
-    address_count = address_index.count()
-    addresses = address_index.last(address_count-1)
-    logg.debug('addresses {} {}'.format(address_count, addresses))
-    return addresses
-
-random.seed()
-
-while run:
-    n = random.randint(0, 255)
-
-    # some of the time do other things than transfers
-    if n & 0xf8 == 0xf8:
-        t = api.create_account()
-        logg.info('create account {}'.format(t))
-             
-    else:
-        tokens = get_tokens()
-        addresses = get_addresses()
-        address_pair = random.choices(addresses, k=2)
-        sender = address_pair[0]
-        recipient = address_pair[1]
-        token = random.choice(tokens)
-
-        c = w3.eth.contract(abi=erc20_abi, address=token)
-        sender_balance = c.functions.balanceOf(sender).call()
-        token_symbol = c.functions.symbol().call()
-        amount = int(random.random() * (sender_balance / 2))
-
-        n = random.randint(0, 255)
-
-        if n & 0xc0 == 0xc0: 
-            t = api.transfer_request(sender, recipient, config.get('CIC_APPROVAL_ESCROW_ADDRESS'), amount, token_symbol)
-            logg.info('transfer REQUEST {} {} from {} to {} => {}'.format(amount, token_symbol, sender, recipient, t))
-        else:
-            t = api.transfer(sender, recipient, amount, token_symbol)
-            logg.info('transfer {} {} from {} to {} => {}'.format(amount, token_symbol, sender, recipient, t))
-
-    time.sleep(random.random() * args.wait_max)

apps/contract-migration/dev/python/scripts/tx_seed.py

@@ -1,51 +0,0 @@
-#!/usr/bin/python
-
-import csv
-import logging
-import argparse
-import os
-import re
-
-import web3
-import confini
-
-from cic_registry import CICRegistry
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.INFO)
-logg = logging.getLogger()
-
-confini_default_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=confini_default_dir, help='config data dir')
-argparser.add_argument('-a', '--token-gifter-address', dest='a', type=str, help='Token gifter address')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-s', '--token-symbol', dest='s', type=str, help='Token symbol')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-cic_eth_api = Api(config.get('CIC_CHAIN_SPEC'))
-
-token_gifter_address = args.a
-
-if __name__ == '__main__':
-    f = open('./data/amounts', 'r')
-    cr = csv.reader(f)
-    for r in cr:
-        logg.info('sending {} {} from {} to {}'.format(r[1], args.s, token_gifter_address, r[0]))
-        cic_eth_api.transfer(token_gifter_address, r[0], int(r[1]), args.s)
-    f.close()

apps/contract-migration/dev/python/scripts/users.py

@@ -1,212 +0,0 @@
-#!/usr/bin/python
-
-import json
-import time
-import datetime
-import random
-import logging
-import os
-import base64
-import hashlib
-import sys
-
-import vobject
-
-import celery
-import web3
-from faker import Faker
-import cic_registry
-import confini
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.DEBUG)
-logg = logging.getLogger()
-
-fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
-
-#f = open('cic.conf', 'r')
-#config = json.load(f)
-#f.close()
-#
-
-config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-logg.info('loaded config\n{}'.format(config))
-
-
-w3s = None
-w3s = web3.Web3(web3.Web3.IPCProvider(config.get('SIGNER_SOCKET_PATH')))
-#w3s = web3.Web3(web3.Web3.IPCProvider(config['signer']['provider']))
-#w3 = web3.Web3(web3.Web3.WebsocketProvider(config['eth']['provider']))
-
-dt_now = datetime.datetime.utcnow()
-dt_then = dt_now - datetime.timedelta(weeks=150)
-ts_now = int(dt_now.timestamp())
-ts_then = int(dt_then.timestamp())
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-api = Api(config.get('CIC_CHAIN_SPEC'))
-
-gift_max = 10000
-gift_factor = (10**9)
-
-categories = [
-        "food/water",
-        "fuel/energy",
-        "education",
-        "health",
-        "shop",
-        "environment",
-        "transport",
-        "farming/labor",
-        "savingsgroup",
-        ]
-
-phone_idx = []
-
-
-def genPhoneIndex(phone):
-    h = hashlib.new('sha256')
-    h.update(phone.encode('utf-8'))
-    h.update(b'cic.msisdn')
-    return h.digest().hex()
-
-
-def genId(addr, typ):
-    h = hashlib.new('sha256')
-    h.update(bytes.fromhex(addr[2:]))
-    h.update(typ.encode('utf-8'))
-    return h.digest().hex()
-
-
-def genDate():
-
-    logg.info(ts_then)
-    ts = random.randint(ts_then, ts_now)
-    return datetime.datetime.fromtimestamp(ts).timestamp()
-
-
-def genPhone():
-    return fake.msisdn()
-
-
-def genPersonal(phone):
-    fn = fake.first_name()
-    ln = fake.last_name()
-    e = fake.email()
-
-    v = vobject.vCard()
-    first_name = fake.first_name()
-    last_name = fake.last_name()
-    v.add('n')
-    v.n.value = vobject.vcard.Name(family=last_name, given=first_name)
-    v.add('fn')
-    v.fn.value = '{} {}'.format(first_name, last_name)
-    v.add('tel')
-    v.tel.typ_param = 'CELL'
-    v.tel.value = phone
-    v.add('email')
-    v.email.value = fake.email()
-
-    vcard_serialized = v.serialize()
-    vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8'))
-
-    return vcard_base64.decode('utf-8')
-
-
-def genCats():
-    i = random.randint(0, 3)
-    return random.choices(categories, k=i)
-
-
-def genAmount():
-    return random.randint(0, gift_max) * gift_factor
-
-
-def gen():
-    old_blockchain_address = '0x' + os.urandom(20).hex()
-    accounts_index_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER')
-    if not accounts_index_account:
-        accounts_index_account = None
-    logg.debug('accounts indexwriter {}'.format(accounts_index_account))
-    t = api.create_account()
-    new_blockchain_address = t.get()
-    gender = random.choice(['female', 'male', 'other'])
-    phone = genPhone()
-    v = genPersonal(phone)
-    o = {
-        'date_registered': genDate(),
-        'vcard': v,
-        'gender': gender,
-        'key': {
-            'ethereum': [
-                old_blockchain_address,
-                new_blockchain_address,
-                ],
-            },
-        'location': {
-            'latitude': str(fake.latitude()),
-            'longitude': str(fake.longitude()),
-            'external': { # add osm lookup
-                }
-            },
-        'selling': genCats(),
-            }
-    uid = genId(new_blockchain_address, 'cic.person')
-
-    #logg.info('gifting {} to {}'.format(amount, new_blockchain_address))
-
-    return (uid, phone, o)
-
-
-def prepareLocalFilePath(datadir, address):
-    parts = [
-                address[:2],
-                address[2:4],
-            ]
-    dirs = '{}/{}/{}'.format(
-            datadir,
-            parts[0],
-            parts[1],
-            )
-    os.makedirs(dirs, exist_ok=True)
-    return dirs
-
-
-if __name__ == '__main__':
-
-    os.makedirs('data/person', exist_ok=True)
-    os.makedirs('data/phone', exist_ok=True)
-
-    fa = open('./data/amounts', 'w')
-    fb = open('./data/addresses', 'w')
-
-    #for i in range(10):
-    for i in range(int(sys.argv[1])):
-    
-        (uid, phone, o) = gen()
-        eth = o['key']['ethereum'][1]
-
-        print(o)
-
-        d = prepareLocalFilePath('./data/person', uid)
-        f = open('{}/{}'.format(d, uid), 'w')
-        json.dump(o, f)
-        f.close()
-
-        pidx = genPhoneIndex(phone)
-        d = prepareLocalFilePath('./data/phone', uid)
-        f = open('{}/{}'.format(d, pidx), 'w')
-        f.write(eth)
-        f.close()
-
-        amount = genAmount()
-        fa.write('{},{}\n'.format(eth,amount))
-        fb.write('{}\n'.format(eth))
-        logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
-
-    fb.close()
-    fa.close()

apps/contract-migration/dev/python/setup.cfg

@@ -1,40 +0,0 @@
-[metadata]
-name = cic-dev-fake
-version = 0.0.1
-description = Fake data generator tools
-author = Louis Holbrook
-author_email = dev@holbrook.no
-url = https://gitlab.com/nolash/simple-multisig
-keywords =
-	ethereum
-classifiers =
-	Programming Language :: Python :: 3
-	Operating System :: OS Independent
-	Development Status :: 3 - Alpha
-	Environment :: No Input/Output (Daemon)
-	Intended Audience :: Developers
-	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-	Topic :: Internet
-	#Topic :: Blockchain :: EVM
-license = GPL3
-licence_files =
-	LICENSE
-
-[options]
-python_requires = >= 3.6
-install_requires =
-	web3==5.12.2
-	vobject==0.9.6.1
-	faker==4.17.1
-tests_require =
-	eth-tester==0.5.0b2
-	py-evm==0.3.0a20
-scripts =
-	scripts/users.py
-	scripts/tx_generator.py
-	scripts/tx_seed.py
-
-[options.extras_require]
-testing =
-	eth-tester==0.5.0b2
-	py-evm==0.3.0a20

apps/contract-migration/dev/python/setup.py

@@ -1,4 +0,0 @@
-from setuptools import setup
-
-setup(
-        )

apps/contract-migration/dev/requirements.txt

@@ -1,10 +0,0 @@
-cryptocurrency-cli-tools~=0.0.4
-giftable-erc20-token~=0.0.7b1
-eth-accounts-index~=0.0.10a2
-erc20-single-shot-faucet~=0.2.0a1
-erc20-approval-escrow~=0.3.0a1
-cic-eth==0.10.0a5
-vobject==0.9.6.1
-faker==4.17.1
-eth-address-index~=0.1.0a1
-crypto-dev-signer~=0.4.13b9

apps/contract-migration/dev/setup.sh

@@ -1,138 +0,0 @@
-#!/bin/bash
-
-# Debug flag
-#debug='-v'
-keystore_file=../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c
-
-debug=''
-abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
-
-# Determine token amount
-token_amount=${DEV_ETH_RESERVE_AMOUNT:0:-1}
-
-# Determine gas amount
-#gas_amount=20000000000000000000
-gas_amount=2000000000000000000
-
-export DATABASE_NAME=$DATABASE_NAME_CIC_ETH
-export DATABASE_PORT=$HTTP_PORT_POSTGRES
-export DATABASE_HOST=localhost
-
-set -e
-set -a
-
-old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
-DEV_ETH_ACCOUNT_GAS_GIFTER=`python ./create.py --no-register`
-echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER
-export DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER
-cic-eth-tag GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
-
-DEV_ETH_ACCOUNT_SARAFU_GIFTER=`python ./create.py --no-register`
-echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER
-export DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER
-cic-eth-tag SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER
-
-
-DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER=`python ./create.py --no-register`
-echo DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER=$DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
-export DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER=$DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
-cic-eth-tag TRANSFER_APPROVAL_OWNER $DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER
-
-
-DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER=`python ./create.py --no-register`
-echo DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER=$DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
-export DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER=$DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
-cic-eth-tag FAUCET_OWNER $DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER
-
-
-DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=`python ./create.py --no-register`
-echo DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=$DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
-export DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=$DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
-cic-eth-tag ACCOUNTS_INDEX_WRITER $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER
-
-
-# Transfer gas to custodial gas provider adddress
-#echo $old_gas_provider
->&2 echo gift gas to gas gifter
->&2 echo "python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount"
->&2 python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
-
->&2 echo gift gas to sarafu token owner
->&2 echo "python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_SARAFU_OWNER $gas_amount"
->&2 python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_SARAFU_OWNER $gas_amount
-
->&2 echo gift gas to account index owner
->&2 echo "python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER $gas_amount"
->&2 python gas.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER $gas_amount
-
-
-# Send reserve to token creator
->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ETH_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_OWNER -w $debug $token_amount
-
-
-# Create token
-#DEV_ETH_SARAFU_TOKEN_ADDRESS=`cic-bancor-token -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -z $DEV_ETH_RESERVE_ADDRESS -o $DEV_ETH_ACCOUNT_SARAFU_OWNER -n $DEV_ETH_SARAFU_TOKEN_NAME -s $DEV_ETH_SARAFU_TOKEN_SYMBOL -d $DEV_ETH_SARAFU_TOKEN_DECIMALS -i $CIC_CHAIN_SPEC $debug $token_amount`
-DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
-echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_SARAFU_TOKEN_ADDRESS 
-export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_SARAFU_TOKEN_ADDRESS 
-
-
-# Transfer tokens to gifter address
->&2 python transfer.py -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS --abi-dir $abi_dir -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
-
-# Deploy transfer approval contract
-CIC_APPROVAL_ESCROW_ADDRESS=`erc20-approval-escrow-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --approver $DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER -w $debug`
-echo CIC_APPROVAL_ESCROW_ADDRESS=$CIC_APPROVAL_ESCROW_ADDRESS 
-export CIC_APPROVAL_ESCROW_ADDRESS=$CIC_APPROVAL_ESCROW_ADDRESS 
-
-# Register transfer approval contract
->&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TransferApproval -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $CIC_APPROVAL_ESCROW_ADDRESS
-
-# Deploy one-time token faucet for newly created token
-DEV_ETH_SARAFU_FAUCET_ADDRESS=`erc20-single-shot-faucet-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS --editor $DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER --set-amount 1048576 -w $debug`
-echo DEV_ETH_SARAFU_FAUCET_ADDRESS=$DEV_ETH_SARAFU_FAUCET_ADDRESS 
-export DEV_ETH_SARAFU_FAUCET_ADDRESS=$DEV_ETH_SARAFU_FAUCET_ADDRESS 
-
-# Transfer tokens to faucet contract
->&2 python transfer.py  -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS --abi-dir $abi_dir -w $debug $DEV_ETH_SARAFU_FAUCET_ADDRESS ${token_amount:0:-1}
-
-# Registry faucet entry
->&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k Faucet -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_SARAFU_FAUCET_ADDRESS
-
-# Deploy token endorser registry
-#DEV_ETH_TOKEN_ENDORSER_ADDRESS=`eth-token-endorser-deploy -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_SARAFU_OWNER $debug`
-#echo DEV_ETH_TOKEN_ENDORSER_ADDRESS=$DEV_ETH_TOKEN_ENDORSER_ADDRESS
-#export DEV_ETH_TOKEN_ENDORSER_ADDRESS=$DEV_ETH_TOKEN_ENDORSER_ADDRESS
-#ENDORSEMENT_MSG=`echo -n 'very cool token' | sha256sum | awk '{print $1;}'`
-#>&2 eth-token-endorser-add -p $ETH_PROVIDER -a $DEV_ETH_TOKEN_ENDORSER_ADDRESS -t $DEV_ETH_SARAFU_TOKEN_ADDRESS -o $DEV_ETH_ACCOUNT_SARAFU_OWNER $debug $ENDORSEMENT_MSG
-CIC_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug`
-echo CIC_TOKEN_INDEX_ADDRESS=$CIC_TOKEN_INDEX_ADDRESS
-export CIC_TOKEN_INDEX_ADDRESS=$CIC_TOKEN_INDEX_ADDRESS
->&2 eth-token-index-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_TOKEN_INDEX_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS
-
-# Register token registry
->&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TokenRegistry -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER $CIC_TOKEN_INDEX_ADDRESS
-
-# Deploy address declarator registry
-declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
-CIC_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $declarator_description`
-echo CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
-export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
-token_description_one=`sha256sum sarafu_declaration.json | awk '{ print $1; }'`
-token_description_two=0x54686973206973207468652053617261667520746f6b656e0000000000000000
->&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_one
->&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_two
-
-
-# Register address declarator
->&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k AddressDeclarator -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER $CIC_DECLARATOR_ADDRESS 
-
-# We're done with the registry at this point, seal it off
->&2 cic-registry-seal -y $keystore_file -i $CIC_CHAIN_SPEC -r $CIC_REGISTRY_ADDRESS -p $ETH_PROVIDER
-
-
-# Add accounts index writer with key from keystore
->&2 eth-accounts-index-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_ACCOUNTS_INDEX_ADDRESS --writer $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER -w $debug
-
-set +a
-set +e

apps/contract-migration/dev/tests/create.py

@@ -1,31 +0,0 @@
-#!python3
-
-# Author:	Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746
-# SPDX-License-Identifier:	GPL-3.0-or-later
-# File-version: 1
-# Description: Smoke test for cic-eth create account api
-
-# standard imports
-import os
-import logging
-
-# third-party imports
-import celery
-import confini
-
-# platform imports
-from cic_eth import Api
-
-script_dir = os.path.dirname(__file__)
-
-logging.basicConfig(level=logging.DEBUG)
-logg = logging.getLogger()
-
-config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic/')
-
-config = confini.Config(config_dir)
-config.process()
-
-a = Api()
-t = a.create_account()
-logg.debug('create account task uuid {}'.format(t))

apps/contract-migration/dev/transfer.py

@@ -1,105 +0,0 @@
-#!python3
-
-"""Token transfer script
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746 
-
-"""
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import os
-import json
-import argparse
-import logging
-
-# third-party imports
-import web3
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore import DictKeystore
-from crypto_dev_signer.eth.helper import EthTxExecutor
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-logging.getLogger('web3').setLevel(logging.WARNING)
-logging.getLogger('urllib3').setLevel(logging.WARNING)
-
-default_abi_dir = '/usr/share/local/cic/solidity/abi'
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
-argparser.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
-argparser.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
-argparser.add_argument('--token-address', required='True', dest='t', type=str, help='Token address')
-argparser.add_argument('-a', '--sender-address', dest='s', type=str, help='Sender account address')
-argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, default=default_abi_dir, help='Directory containing bytecode and abi (default {})'.format(default_abi_dir))
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-argparser.add_argument('recipient', type=str, help='Recipient account address')
-argparser.add_argument('amount', type=int, help='Amount of tokens to mint and gift')
-args = argparser.parse_args()
-
-
-if args.vv:
-    logg.setLevel(logging.DEBUG)
-elif args.v:
-    logg.setLevel(logging.INFO)
-
-block_last = args.w
-block_all = args.ww
-
-w3 = web3.Web3(web3.Web3.HTTPProvider(args.p))
-
-signer_address = None
-keystore = DictKeystore()
-if args.y != None:
-    logg.debug('loading keystore file {}'.format(args.y))
-    signer_address = keystore.import_keystore_file(args.y)
-    logg.debug('now have key for signer address {}'.format(signer_address))
-signer = EIP155Signer(keystore)
-
-chain_pair = args.i.split(':')
-chain_id = int(chain_pair[1])
-
-helper = EthTxExecutor(
-        w3,
-        signer_address,
-        signer,
-        chain_id,
-        block=args.ww,
-    )
-
-
-def main():
-    # TODO: Add pure ERC20 abi to collection
-    f = open(os.path.join(args.abi_dir, 'ERC20.json'), 'r')
-    abi = json.load(f)
-    f.close()
-
-    c = w3.eth.contract(abi=abi, address=args.t)
-
-    recipient = args.recipient
-    value = args.amount
-
-    logg.debug('sender {} balance before: {}'.format(signer_address, c.functions.balanceOf(signer_address).call()))
-    logg.debug('recipient {} balance before: {}'.format(recipient, c.functions.balanceOf(recipient).call()))
-    (tx_hash, rcpt) = helper.sign_and_send(
-            [
-                c.functions.transfer(recipient, value).buildTransaction,
-                ],
-            )
-    logg.debug('sender {} balance after:  {}'.format(signer_address, c.functions.balanceOf(signer_address).call()))
-    logg.debug('recipient {} balance after:  {}'.format(recipient, c.functions.balanceOf(recipient).call()))
-
-    if block_last:
-        helper.wait_for()
-
-    print(tx_hash)
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/docker/Dockerfile

@@ -53,7 +53,7 @@ RUN apt-get update && \
 
 RUN echo installing nodejs tooling
 
-COPY contract-migration/dev/nvm.sh /root/
+COPY contract-migration/nvm.sh /root/
 
 # Install nvm with node and npm
 # https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
@@ -106,7 +106,7 @@ RUN cd cic-bancor/python && \
 	pip install --extra-index-url $pip_extra_index_url .
 
 RUN echo installing common python tooling
-ARG cic_python_commit=beecee783ceac2ea0fa711f888ce4c82f1a81490
+ARG cic_python_commit=9bd1d5319aad8791ebf353707eabbb7b3e7ab5d0
 ARG cic_python_url=https://gitlab.com/grassrootseconomics/cic-python.git/
 RUN echo Install sum of python dependencies across all components && \
 	git clone --depth 1 $cic_python_url cic-python && \
@@ -120,39 +120,48 @@ ARG cryptocurrency_cli_tools_version=0.0.4
 RUN pip install --extra-index-url $pip_extra_index_url cryptocurrency-cli-tools==$cryptocurrency_cli_tools_version
 
 RUN echo Install smart contract interface implementations, least frequently changed first
-ARG giftable_erc20_token_version=0.0.7b10
+ARG giftable_erc20_token_version=0.0.7b12
 RUN pip install --extra-index-url $pip_extra_index_url giftable-erc20-token==$giftable_erc20_token_version 
 
-ARG eth_accounts_index_version=0.0.10a6
+ARG eth_accounts_index_version=0.0.10a9
 RUN pip install --extra-index-url $pip_extra_index_url eth-accounts-index==$eth_accounts_index_version
 
-ARG erc20_approval_escrow_version=0.3.0a4
+#ARG erc20_approval_escrow_version=0.3.0a4
-RUN pip install --extra-index-url $pip_extra_index_url erc20-approval-escrow==$erc20_approval_escrow_version 
+#RUN pip install --extra-index-url $pip_extra_index_url erc20-approval-escrow==$erc20_approval_escrow_version 
+ARG erc20_transfer_authorization_version=0.3.0a9
+RUN pip install --extra-index-url $pip_extra_index_url erc20-transfer-authorization==$erc20_transfer_authorization_version
 
 #ARG erc20_single_shot_faucet_version=0.2.0a5
 #RUN pip install --extra-index-url $pip_extra_index_url erc20-single-shot-faucet==$erc20_single_shot_faucet_version 
 
-ARG sarafu_faucet_version==0.0.1a10
+ARG sarafu_faucet_version=0.0.1a11
 RUN pip install --extra-index-url $pip_extra_index_url sarafu-faucet==$sarafu_faucet_version
 
-ARG eth_address_index_version==0.1.0a8
+ARG eth_address_index_version=0.1.0a10
 RUN pip install --extra-index-url $pip_extra_index_url eth-address-index==$eth_address_index_version
 
 RUN echo Install cic specific python packages
-ARG cic_registry_version=0.5.3a18
+ARG cic_registry_version=0.5.3a20+build.30d0026c
-RUN pip install  --extra-index-url  $pip_extra_index_url  cic-registry==$cic_registry_version
+RUN pip install  --extra-index-url $pip_extra_index_url  cic-registry==$cic_registry_version
 
 RUN echo Install misc helpers
 
-ARG crypto_dev_signer_version==0.4.13rc2
+ARG crypto_dev_signer_version=0.4.13rc2
 RUN pip install  --extra-index-url $pip_extra_index_url crypto-dev-signer==$crypto_dev_signer_version
 
-ARG eth_gas_proxy_version==0.0.1a4
+ARG eth_gas_proxy_version=0.0.1a4
 RUN pip install  --extra-index-url $pip_extra_index_url eth-gas-proxy==$eth_gas_proxy_version
 
-ARG cic_contracts_version==0.0.2a2
+ARG cic_contracts_version=0.0.2a2
 RUN pip install  --extra-index-url $pip_extra_index_url cic-contracts==$cic_contracts_version
 
+ARG chainlib_version=0.0.1a16
+RUN pip install  --extra-index-url $pip_extra_index_url chainlib==$chainlib_version
+
+ARG chainsyncer_version=0.0.1a8
+RUN pip install  --extra-index-url $pip_extra_index_url chainsyncer==$chainsyncer_version
+
+
 WORKDIR /root
 
 COPY contract-migration/testdata/pgp testdata/pgp

apps/contract-migration/scripts/README

@@ -0,0 +1,72 @@
+# DATA GENERATION TOOLS
+
+This folder contains tools to generate and import test data.
+
+## DATA CREATION
+
+Does not need the cluster to run.
+
+Vanilla:
+
+`python create_import_users.py [--dir <datadir>] <number_of_users>`
+
+If you want to use the `import_balance.py` script to add to the user's balance from an external address, add:
+
+`python create_import_users.py --gift-threshold <max_units_to_send> [--dir <datadir>] <number_of_users>`
+
+
+## IMPORT
+
+Make sure the following is running in the cluster:
+	* eth
+	* postgres
+	* redis
+	* cic-eth-tasker
+	* cic-eth-dispatcher
+	* cic-eth-manager-head
+
+
+You will want to run these in sequence:
+
+
+## 1. Metadata
+
+`node import_meta.js <datadir> <number_of_users>`
+
+Monitors a folder for output from the `import_users.py` script, adding the metadata found to the `cic-meta` service.
+
+
+## 2. Balances
+
+(Only if you used the `--gift-threshold` option above)
+
+`python -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> --head -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
+
+This will monitor new mined blocks and send balances to the newly created accounts.
+
+
+### 3. Users
+
+Without any modifications to the cluster and config files:
+
+`python -c config --redis-host-callback redis <datadir>`
+
+** A note on the The callback**:  The script uses a redis callback to retrieve the newly generated custodial address. This is the redis server _from the perspective of the cic-eth component_.
+
+
+## VERIFY
+
+`python verify.py -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> <datadir>` 
+
+Checks
+	* Private key is in cic-eth keystore
+	* Address is in accounts index
+	* Address has balance matching the gift threshold 
+	* Metadata can be retrieved and has exact match
+
+Should exit with code 0 if all input data is found in the respective services.
+
+
+## KNOWN ISSUES
+
+If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount.

apps/contract-migration/scripts/cmd/traffic.py

@@ -0,0 +1,417 @@
+# standard imports
+import logging
+import json
+import uuid
+import importlib
+import random
+import copy
+from argparse import RawTextHelpFormatter
+
+# external imports
+import redis
+from cic_eth.api.api_task import Api
+
+logg = logging.getLogger(__name__)
+
+
+def add_args(argparser):
+    """Parse script specific command line arguments
+
+    :param argparser: Top-level argument parser
+    :type argparser: argparse.ArgumentParser
+    """
+    argparser.formatter_class = formatter_class=RawTextHelpFormatter
+    argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+    argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+    argparser.add_argument('--batch-size', dest='batch_size', default=10, type=int, help='number of events to process simultaneously')
+    argparser.description = """Generates traffic on the cic network using dynamically loaded modules as event sources
+
+"""
+    return argparser
+
+
+class TrafficItem:
+    """Represents a single item of traffic meta that will be processed by a traffic generation method
+
+    The traffic generation module passed in the argument must implement a method "do" with interface conforming to local.noop_traffic.do.
+
+    :param item: Traffic generation module.
+    :type item: function
+    """
+    def __init__(self, item):
+        self.method = item.do
+        self.uuid = uuid.uuid4()
+        self.ext = None
+        self.result = None
+        self.sender = None
+        self.recipient = None
+        self.source_token = None
+        self.destination_token = None
+        self.source_value = 0
+
+
+    def __str__(self):
+        return 'traffic item method {} uuid {}'.format(self.method, self.uuid)
+
+
+class TrafficRouter:
+    """Holds and selects from the collection of traffic generator modules that will be used for the execution.
+
+    :params batch_size: Amount of simultaneous traffic items that can simultanously be in flight.
+    :type batch_size: number
+    :raises ValueError: If batch size is zero of negative
+    """
+    def __init__(self, batch_size=1):
+        if batch_size < 1:
+            raise ValueError('batch size cannot be 0')
+        self.items = []
+        self.weights = []
+        self.total_weights = 0
+        self.batch_size = batch_size
+        self.reserved = {}
+        self.reserved_count = 0
+        self.traffic = {}
+
+
+    def add(self, item, weight):
+        """Add a traffic generator module to the list of modules to choose between for traffic item exectuion.
+
+        The probability that a module will be chosen for any single item is the ratio between the weight parameter and the accumulated weights for all items.
+
+        See local.noop for which criteria the generator module must fulfill.
+
+        :param item: Qualified class path to traffic generator module. Will be dynamically loaded.
+        :type item: str
+        :param weight: Selection probability weight
+        :type weight: number
+        :raises ModuleNotFound: Invalid item argument
+        """
+        self.weights.append(self.total_weights)
+        self.total_weights += weight
+        m = importlib.import_module(item)
+        self.items.append(m)
+        
+
+    def reserve(self):
+        """Selects the module to be used to execute the next traffic item, using the provided weights.
+
+        If the current number of calls to "reserve" without corresponding calls to "release" equals the set batch size limit, None will be returned. The calling code should allow a short grace period before trying the call again.
+        :raises ValueError: No items have been added
+        :returns: A traffic item with the selected module method as the method property.
+        :rtype: TrafficItem|None
+        """
+        if len(self.items) == 0:
+            raise ValueError('Add at least one item first')
+
+        if len(self.reserved) == self.batch_size:
+            return None
+
+        n = random.randint(0, self.total_weights)
+        item = self.items[0]
+        for i in range(len(self.weights)):
+            if n <= self.weights[i]:
+                item = self.items[i]
+                break
+
+        ti = TrafficItem(item)
+        self.reserved[ti.uuid] = ti
+        return ti
+
+
+    def release(self, traffic_item):
+        """Releases the traffic item from the list of simultaneous traffic items in flight.
+
+        :param traffic_item: Traffic item
+        :type traffic_item: TrafficItem
+        """
+        del self.reserved[traffic_item.uuid]
+
+
+    def apply_import_dict(self, keys, dct):
+        """Convenience method to add traffic generator modules from a dictionary.
+
+        :param keys: Keys in dictionary to add
+        :type keys: list of str
+        :param dct: Dictionary to choose module strings from
+        :type dct: dict
+        :raises ModuleNotFoundError: If one of the module strings refer to an invalid module.
+        """
+        # parse traffic items
+        for k in keys:
+            if len(k) > 8 and k[:8] == 'TRAFFIC_':
+                v = int(dct.get(k))
+                self.add(k[8:].lower(), v)
+                logg.debug('found traffic item {} weight {}'.format(k, v))
+
+
+# TODO: This will not work well with big networks. The provisioner should use lazy loading and LRU instead.
+class TrafficProvisioner:
+    """Loads metadata necessary for traffic item execution.
+
+    Instantiation will by default trigger retrieval of accounts and tokens on the network.
+
+    It will also populate the aux property of the instance with the values from the static aux parameter template. 
+    """
+
+    oracles = {
+        'account': None,
+        'token': None,
+            }
+    """Data oracles to be used for traffic item generation"""
+    default_aux = {
+            }
+    """Aux parameter template to be passed to the traffic generator module"""
+
+
+    def __init__(self):
+        self.tokens = self.oracles['token'].get_tokens()
+        self.accounts = self.oracles['account'].get_accounts()
+        self.aux = copy.copy(self.default_aux)
+        self.__balances = {}
+        for a in self.accounts:
+            self.__balances[a] = {}
+
+
+    # Caches a single address' balance of a single token
+    def __cache_balance(self, holder_address, token, value):
+        if self.__balances.get(holder_address) == None:
+            self.__balances[holder_address] = {}
+        self.__balances[holder_address][token] = value
+        logg.debug('setting cached balance of {} token {} to {}'.format(holder_address, token, value))
+
+
+    def add_aux(self, k, v):
+        """Add a key-value pair to the aux parameter list.
+
+        Does not protect existing entries from being overwritten.
+
+        :param k: Key
+        :type k: str
+        :param v: Value
+        :type v: any
+        """
+        logg.debug('added {} = {} to traffictasker'.format(k, v))
+        self.aux[k] = v
+
+
+    # TODO: Balance list type should perhaps be a class (provided by cic-eth package) due to its complexity.
+    def balances(self, refresh_accounts=None):
+        """Retrieves all token balances for the given account list.
+
+        If refresh_accounts is not None, the balance values for the given accounts will be retrieved from upstream. If the argument is an empty list, the balances will be updated for all tokens of all ccounts. If there are many accounts and/or tokens, this may be a VERY EXPENSIVE OPERATION. The "balance" method can be used instead to update individual account/token pair balances.
+
+        :param accounts: List of accounts to refresh balances for.
+        :type accounts: list of str, 0x-hex
+        :returns: Dict of dict of dicts; v[accounts][token] = {balance_types}
+        :rtype: dict
+        """
+        if refresh_accounts != None:
+            accounts = refresh_accounts
+            if len(accounts) == 0:
+                accounts = self.accounts
+            for account in accounts:
+                for token in self.tokens:
+                    value = self.balance(account, token)
+                    self.__cache_balance(account, token.symbol(), value)
+                    logg.debug('balance sender {} token {} = {}'.format(account, token, value))
+        else:
+            logg.debug('returning cached balances')
+
+        return self.__balances
+
+
+    # TODO: use proper redis callback 
+    def balance(self, account, token):
+        """Update balance for a single token of a single account from upstream.
+        
+        The balance will be the spendable balance at the time of the call. This value may be less than the balance reported by the consensus network, if a previous outgoing transaction is still pending in the network or the custodial system queue.
+
+        :param account: Account to update
+        :type account: str, 0x-hex
+        :param token: Token to update balance for
+        :type token: cic_registry.token.Token
+        :returns: Updated balance
+        :rtype: complex balance dict
+        """
+        api = Api(
+            str(self.aux['chain_spec']),
+            queue=self.aux['api_queue'],
+            #callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
+            #callback_task='cic_eth.callbacks.redis.redis',
+            #callback_queue=queue,
+            )
+        t = api.balance(account, token.symbol())
+        r = t.get()
+        for c in t.collect():
+            r = c[1]
+        assert t.successful()
+        #return r[0]['balance_network'] - r[0]['balance_outgoing']
+        return r[0]
+
+
+    def update_balance(self, account, token, value):
+        """Manually set a token balance for an account.
+
+        :param account: Account to update
+        :type account: str, 0x-hex
+        :param token: Token to update balance for
+        :type token: cic_registry.token.Token
+        :param value: Balance value to set
+        :type value: number
+        :returns: Balance value (unchanged)
+        :rtype: complex balance dict
+        """
+        self.__cache_balance(account, token.symbol(), value)
+        return value
+
+
+# TODO: Abstract redis with a generic pubsub adapter
+class TrafficSyncHandler:
+    """Encapsulates callback methods required by the chain syncer.
+    
+    This implementation uses a redis subscription as backend to retrieve results from asynchronously executed tasks.
+
+    :param config: Configuration of current top-level execution
+    :type config: object with dict get interface
+    :param traffic_router: Traffic router instance to use for the syncer session.
+    :type traffic_router: TrafficRouter
+    :raises Exception: Any Exception redis may raise on connection attempt.
+    """
+    def __init__(self, config, traffic_router):
+        self.traffic_router = traffic_router
+        self.redis_channel = str(uuid.uuid4())
+        self.pubsub = self.__connect_redis(self.redis_channel, config)
+        self.traffic_items = {}
+        self.config = config
+        self.init = False
+
+
+    # connects to redis
+    def __connect_redis(self, redis_channel, config):
+        r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB'))
+        redis_pubsub = r.pubsub()
+        redis_pubsub.subscribe(redis_channel)
+        logg.debug('redis connected on channel {}'.format(redis_channel))
+        return redis_pubsub
+
+
+    # TODO: This method is too long, split up
+    # TODO: This method will not yet cache balances for newly created accounts
+    def refresh(self, block_number, tx_index):
+        """Traffic method and item execution driver to be called on every loop execution of the chain syncer. 
+
+        Implements the signature required by callbacks called from chainsyncer.driver.loop.
+
+        :param block_number: Syncer block height at time of call.
+        :type block_number: number
+        :param tx_index: Syncer block transaction index at time of call.
+        :type tx_index: number
+        """
+        traffic_provisioner = TrafficProvisioner()
+        traffic_provisioner.add_aux('redis_channel', self.redis_channel)
+
+        refresh_accounts = None
+        # Note! This call may be very expensive if there are a lot of accounts and/or tokens on the network
+        if not self.init:
+            refresh_accounts = traffic_provisioner.accounts
+        balances = traffic_provisioner.balances(refresh_accounts=refresh_accounts)
+        self.init = True
+
+        if len(traffic_provisioner.tokens) == 0:
+            logg.error('patiently waiting for at least one registered token...')
+            return
+
+        logg.debug('executing handler refresh with accounts {}'.format(traffic_provisioner.accounts))
+        logg.debug('executing handler refresh with tokens {}'.format(traffic_provisioner.tokens))
+
+        sender_indices = [*range(0, len(traffic_provisioner.accounts))]
+        # TODO: only get balances for the selection that we will be generating for
+
+        while True:
+            traffic_item = self.traffic_router.reserve()
+            if traffic_item == None:
+                logg.debug('no traffic_items left to reserve {}'.format(traffic_item))
+                break
+
+            # TODO: temporary selection
+            token_pair = (
+                    traffic_provisioner.tokens[0],
+                    traffic_provisioner.tokens[0],
+                    )
+            sender_index_index = random.randint(0, len(sender_indices)-1)
+            sender_index = sender_indices[sender_index_index]
+            sender = traffic_provisioner.accounts[sender_index]
+            #balance_full = balances[sender][token_pair[0].symbol()]
+            if len(sender_indices) == 1:
+                sender_indices[m] = sender_sender_indices[len(senders)-1]
+            sender_indices = sender_indices[:len(sender_indices)-1]
+
+            balance_full = traffic_provisioner.balance(sender, token_pair[0])
+
+            recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
+            recipient = traffic_provisioner.accounts[recipient_index]
+          
+            logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
+            (e, t, balance_result,) = traffic_item.method(
+                    token_pair,
+                    sender,
+                    recipient,
+                    balance_full,
+                    traffic_provisioner.aux,
+                    block_number,
+                    tx_index,
+                    )
+            traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
+            sender_indices.append(recipient_index)
+
+            if e != None:
+                logg.info('failed {}: {}'.format(str(traffic_item), e))
+                self.traffic_router.release(traffic_item)
+                continue
+
+            if t == None:
+                logg.info('traffic method {} completed immediately')
+                self.traffic_router.release(traffic_item)
+            traffic_item.ext = t
+            self.traffic_items[traffic_item.ext] = traffic_item
+
+
+        while True:
+            m = self.pubsub.get_message(timeout=0.1)
+            if m == None:
+                break
+            logg.debug('redis message {}'.format(m))
+            if m['type'] == 'message':
+                message_data = json.loads(m['data'])
+                uu = message_data['root_id']
+                match_item = self.traffic_items[uu]
+                self.traffic_router.release(match_item)
+                logg.debug('>>>>>>>>>>>>>>>>>>> match item {} {} {}'.format(match_item, match_item.result, dir(match_item)))
+                if message_data['status'] != 0:
+                    logg.error('task item {} failed with error code {}'.format(match_item, message_data['status']))
+                else:
+                    match_item.result = message_data['result']
+                    logg.debug('got callback result: {}'.format(match_item))
+
+
+    def name(self):
+        """Returns the common name for the syncer callback implementation. Required by the chain syncer.
+        """
+        return 'traffic_item_handler'
+
+
+    def filter(self, conn, block, tx, db_session):
+        """Callback for every transaction found in a block. Required by the chain syncer.
+
+        Currently performs no operation.
+
+        :param conn: A HTTPConnection object to the chain rpc provider.
+        :type conn: chainlib.eth.rpc.HTTPConnection
+        :param block: The block object of current transaction
+        :type block: chainlib.eth.block.Block
+        :param tx: The block transaction object
+        :type tx: chainlib.eth.tx.Tx
+        :param db_session: Syncer backend database session
+        :type db_session: SQLAlchemy.Session
+        """
+        logg.debug('handler get {}'.format(tx))

apps/contract-migration/scripts/common/__init__.py

@@ -0,0 +1,8 @@
+from . import (
+    log,
+    argparse,
+    config,
+    signer,
+    rpc,
+    registry,
+    )

apps/contract-migration/scripts/common/argparse.py

@@ -0,0 +1,73 @@
+# standard imports
+import logging
+import argparse
+import os
+import sys
+
+default_config_dir = os.environ.get('CONFINI_DIR')
+full_template = { 
+        # (long arg and key name, short var, type, default, help,)
+        'provider': ('p', str, None, 'RPC provider url',),
+        'registry_address': ('r', str, None, 'CIC registry address',),
+        'keystore_file': ('y', str, None, 'Keystore file',),
+        'config_dir': ('c', str, default_config_dir, 'Configuration directory',),
+        'queue': ('q', str, 'cic-eth', 'Celery task queue',),
+        'chain_spec': ('i', str, None, 'Chain spec string',),
+        'abi_dir': (None, str, None, 'Smart contract ABI search path',),
+        'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
+        }
+default_include_args = [
+    'config_dir',
+    'provider',
+    'env_prefix',
+    ]
+
+sub = None
+
+def create(caller_dir, include_args=default_include_args):
+
+    argparser = argparse.ArgumentParser()
+
+    for k in include_args:
+        a = full_template[k]
+        long_flag = '--' + k.replace('_', '-')
+        short_flag = None
+        dest = None
+        if a[0] != None:
+            short_flag = '-' + a[0]
+            dest = a[0]
+        else:
+            dest = k
+        default = a[2]
+        if default == None and k == 'config_dir':
+            default = os.path.join(caller_dir, 'config')
+        
+        if short_flag == None:
+            argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
+        else:
+            argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
+
+    argparser.add_argument('-v', action='store_true', help='Be verbose')
+    argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+
+    return argparser
+
+
+def add(argparser, processor, name, description=None):
+    processor(argparser)
+
+    return argparser
+
+
+def parse(argparser, logger=None):
+
+    args = argparser.parse_args(sys.argv[1:])
+
+    # handle logging input
+    if logger != None:
+        if args.vv:
+            logger.setLevel(logging.DEBUG)
+        elif args.v:
+            logger.setLevel(logging.INFO)
+
+    return args

apps/contract-migration/scripts/common/config.py

@@ -0,0 +1,39 @@
+# external imports
+import logging
+import confini
+
+logg = logging.getLogger(__name__)
+
+default_arg_overrides = {
+    'abi_dir': 'ETH_ABI_DIR',
+    'p': 'ETH_PROVIDER',
+    'i': 'CIC_CHAIN_SPEC',
+    'r': 'CIC_REGISTRY_ADDRESS',
+    }
+
+
+def override(config, override_dict, label):
+    config.dict_override(override_dict, label)
+    config.validate()
+    return config
+
+
+def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
+    # handle config input
+    config = confini.Config(config_dir, env_prefix)
+    config.process()
+    if arg_overrides != None and args != None:
+        override_dict = {}
+        for k in arg_overrides:
+            v = getattr(args, k)
+            if v != None:
+                override_dict[arg_overrides[k]] = v
+        config = override(config, override_dict, 'args')
+    else:
+        config.validate()
+
+    return config
+
+
+def log(config):
+    logg.debug('config loaded:\n{}'.format(config))

apps/contract-migration/scripts/common/log.py

@@ -0,0 +1,18 @@
+# standard imports
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+
+default_mutelist = [
+        'urllib3',
+        'websockets.protocol',
+        'web3.RequestManager',
+        'web3.providers.WebsocketProvider',
+        'web3.providers.HTTPProvider',
+        ]
+
+def create(name=None, mutelist=default_mutelist):
+    logg = logging.getLogger(name)
+    for m in mutelist:
+        logging.getLogger(m).setLevel(logging.CRITICAL)
+    return logg

apps/contract-migration/scripts/common/registry.py

@@ -0,0 +1,86 @@
+# standard imports
+import logging
+import copy
+
+# external imports
+from cic_registry import CICRegistry
+from eth_token_index import TokenUniqueSymbolIndex
+from eth_accounts_index import AccountRegistry
+from chainlib.chain import ChainSpec
+from cic_registry.chain import ChainRegistry
+from cic_registry.helper.declarator import DeclaratorOracleAdapter
+
+logg = logging.getLogger(__name__)
+
+
+class TokenOracle:
+
+    def __init__(self, conn, chain_spec, registry):
+        self.tokens = []
+        self.chain_spec = chain_spec
+        self.registry = registry
+
+        token_registry_contract = CICRegistry.get_contract(chain_spec, 'TokenRegistry', 'Registry')
+        self.getter = TokenUniqueSymbolIndex(conn, token_registry_contract.address())
+
+
+    def get_tokens(self):
+        token_count = self.getter.count()
+        if token_count == len(self.tokens):
+            return self.tokens
+
+        for i in range(len(self.tokens), token_count):
+            token_address = self.getter.get_index(i)
+            t = self.registry.get_address(self.chain_spec, token_address)
+            token_symbol = t.symbol()
+            self.tokens.append(t)
+
+            logg.debug('adding token idx {} symbol {} address {}'.format(i, token_symbol, token_address))
+
+        return copy.copy(self.tokens)
+
+
+class AccountsOracle:
+
+    def __init__(self, conn, chain_spec, registry):
+        self.accounts = []
+        self.chain_spec = chain_spec
+        self.registry = registry
+
+        accounts_registry_contract = CICRegistry.get_contract(chain_spec, 'AccountRegistry', 'Registry')
+        self.getter = AccountRegistry(conn, accounts_registry_contract.address())
+
+
+    def get_accounts(self):
+        accounts_count = self.getter.count()
+        if accounts_count == len(self.accounts):
+            return self.accounts
+
+        for i in range(len(self.accounts), accounts_count):
+            account = self.getter.get_index(i)
+            self.accounts.append(account)
+            logg.debug('adding account {}'.format(account))
+
+        return copy.copy(self.accounts)
+
+
+def init_legacy(config, w3):
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    CICRegistry.init(w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
+    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
+
+    chain_registry = ChainRegistry(chain_spec)
+    CICRegistry.add_chain_registry(chain_registry, True)
+
+    declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
+    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+    if trusted_addresses_src == None:
+        raise ValueError('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+    trusted_addresses = trusted_addresses_src.split(',')
+    for address in trusted_addresses:
+        logg.info('using trusted address {}'.format(address))
+
+    oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
+    chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
+
+    return CICRegistry

apps/contract-migration/scripts/common/rpc.py

@@ -0,0 +1,18 @@
+# standard imports
+import re
+
+# external imports
+import web3
+
+def create(url):
+    # web3 input
+    # TODO: Replace with chainlib
+    re_websocket = r'^wss?:'
+    re_http = r'^https?:'
+    blockchain_provider = None
+    if re.match(re_websocket, url):
+        blockchain_provider = web3.Web3.WebsocketProvider(url)
+    elif re.match(re_http, url):
+        blockchain_provider = web3.Web3.HTTPProvider(url)
+    w3 = web3.Web3(blockchain_provider)
+    return w3

apps/contract-migration/scripts/common/signer.py

@@ -0,0 +1,23 @@
+# standard imports
+import logging
+
+# external imports 
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+
+logg = logging.getLogger(__name__)
+
+keystore = DictKeystore()
+
+def from_keystore(keyfile):
+    global keystore
+
+    # signer
+    if keyfile == None:
+        raise ValueError('please specify signer keystore file')
+
+    logg.debug('loading keystore file {}'.format(keyfile))
+    address = keystore.import_keystore_file(keyfile)
+
+    signer = EIP155Signer(keystore)
+    return (address, signer,)

apps/contract-migration/scripts/config/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis://localhost:63379
+result_url = redis://localhost:63379

apps/contract-migration/scripts/config/cic.ini

@@ -0,0 +1,9 @@
+[cic]
+registry_address = 0x32E860c2A0645d1B7B005273696905F5D6DC5D05
+token_index_address =
+accounts_index_address = 
+declarator_address = 
+approval_escrow_address =
+chain_spec = evm:bloxberg:8996
+tx_retry_delay = 
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/contract-migration/scripts/config/database.ini

@@ -0,0 +1,5 @@
+[database]
+name = sempo
+host = localhost
+port = 5432
+user = postgres

apps/contract-migration/scripts/config/eth.ini

@@ -0,0 +1,8 @@
+[eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
+provider = http://localhost:63545
+gas_provider_address = 
+#chain_id = 
+abi_dir = /usr/local/share/cic/solidity/abi
+account_accounts_index_writer = 

apps/contract-migration/scripts/config/meta.ini

@@ -0,0 +1,2 @@
+[meta]
+url = http://localhost:63380

apps/contract-migration/scripts/config/pgp.ini

@@ -0,0 +1,5 @@
+[pgp]
+exports_dir = ../testdata/pgp
+private_key_file = privatekeys_meta.asc
+public_key_file = publickeys_meta.asc
+passphrase = merman

apps/contract-migration/scripts/config/redis.ini

@@ -0,0 +1,4 @@
+[redis]
+host = localhost
+port = 63379
+db = 0

apps/contract-migration/scripts/config/traffic.ini

@@ -0,0 +1,4 @@
+[traffic]
+#local.noop_traffic = 2
+local.account = 2
+local.transfer = 2

apps/contract-migration/scripts/create_import_users.py

@@ -0,0 +1,216 @@
+#!/usr/bin/python
+
+# standard imports
+import json
+import time
+import datetime
+import random
+import logging
+import os
+import base64
+import hashlib
+import sys
+import argparse
+import random
+
+# external imports
+import vobject
+import celery
+import web3
+from faker import Faker
+import cic_registry
+import confini
+from cic_eth.api import Api
+from cic_types.models.person import (
+        Person,
+        generate_vcard_from_contact_data,
+        get_contact_data_from_vcard,
+        )
+from chainlib.eth.address import to_checksum
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
+
+script_dir = os.path.realpath(os.path.dirname(__file__))
+#config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
+argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('--dir', default='out', type=str, help='path to users export dir tree')
+argparser.add_argument('user_count', type=int, help='amount of users to generate')
+args = argparser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+logg.info('loaded config\n{}'.format(config))
+
+
+dt_now = datetime.datetime.utcnow()
+dt_then = dt_now - datetime.timedelta(weeks=150)
+ts_now = int(dt_now.timestamp())
+ts_then = int(dt_then.timestamp())
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+api = Api(config.get('CIC_CHAIN_SPEC'))
+
+gift_max = args.gift_threshold or 0
+gift_factor = (10**6)
+
+categories = [
+        "food/water",
+        "fuel/energy",
+        "education",
+        "health",
+        "shop",
+        "environment",
+        "transport",
+        "farming/labor",
+        "savingsgroup",
+        ]
+
+phone_idx = []
+
+user_dir = args.dir
+user_count = args.user_count
+
+def genPhoneIndex(phone):
+    h = hashlib.new('sha256')
+    h.update(phone.encode('utf-8'))
+    h.update(b'cic.msisdn')
+    return h.digest().hex()
+
+
+def genId(addr, typ):
+    h = hashlib.new('sha256')
+    h.update(bytes.fromhex(addr[2:]))
+    h.update(typ.encode('utf-8'))
+    return h.digest().hex()
+
+
+def genDate():
+
+    logg.info(ts_then)
+    ts = random.randint(ts_then, ts_now)
+    return datetime.datetime.fromtimestamp(ts).timestamp()
+
+
+def genPhone():
+    return fake.msisdn()
+
+
+def genPersonal(phone):
+    fn = fake.first_name()
+    ln = fake.last_name()
+    e = fake.email()
+
+    return generate_vcard_from_contact_data(ln, fn, phone, e)
+
+
+def genCats():
+    i = random.randint(0, 3)
+    return random.choices(categories, k=i)
+
+
+def genAmount():
+    return random.randint(0, gift_max) * gift_factor
+
+def genDob():
+    dob_src = fake.date_of_birth(minimum_age=15)
+    dob = {}
+
+    if random.random() < 0.5:
+        dob['year'] = dob_src.year
+
+        if random.random() > 0.5:
+            dob['month'] = dob_src.month
+            dob['day'] = dob_src.day
+    
+    return dob
+
+
+def gen():
+    old_blockchain_address = '0x' + os.urandom(20).hex()
+    old_blockchain_checksum_address = to_checksum(old_blockchain_address)
+    gender = random.choice(['female', 'male', 'other'])
+    phone = genPhone()
+    city = fake.city_name()
+    v = genPersonal(phone)
+
+    contact_data = get_contact_data_from_vcard(v)
+    p = Person()
+    p.load_vcard(contact_data)
+
+    p.date_registered = genDate()
+    p.date_of_birth = genDob()
+    p.gender = gender
+    p.identities = {
+            'evm': {
+                'oldchain:1': [
+                        old_blockchain_checksum_address,
+                    ],
+                },
+            }
+    p.location['area_name'] = city
+    if random.randint(0, 1):
+        p.identities['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
+        p.identities['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
+    
+    return (old_blockchain_checksum_address, phone, p)
+
+
+def prepareLocalFilePath(datadir, address):
+    parts = [
+                address[:2],
+                address[2:4],
+            ]
+    dirs = '{}/{}/{}'.format(
+            datadir,
+            parts[0],
+            parts[1],
+            )
+    os.makedirs(dirs, exist_ok=True)
+    return dirs
+
+
+if __name__ == '__main__':
+
+    base_dir = os.path.join(user_dir, 'old')
+    os.makedirs(base_dir, exist_ok=True)
+
+    fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
+
+    for i in range(user_count):
+    
+        (eth, phone, o) = gen()
+        uid = eth[2:].upper()
+
+        print(o)
+
+        d = prepareLocalFilePath(base_dir, uid)
+        f = open('{}/{}'.format(d, uid + '.json'), 'w')
+        json.dump(o.serialize(), f)
+        f.close()
+
+        pidx = genPhoneIndex(phone)
+        d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), uid)
+        f = open('{}/{}'.format(d, pidx), 'w')
+        f.write(eth)
+        f.close()
+
+        amount = genAmount()
+        fa.write('{},{}\n'.format(eth,amount))
+        logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+
+    fa.close()

apps/contract-migration/scripts/false/anonymous-cli-metrics.json

@@ -0,0 +1 @@
+{"metricId":"ea11447f-da1c-49e6-b0a2-8a988a99e3ce","metrics":{"from":"2021-02-12T18:59:21.666Z","to":"2021-02-12T18:59:21.666Z","successfulInstalls":0,"failedInstalls":1}}

apps/contract-migration/scripts/import_balance.py

@@ -0,0 +1,295 @@
+# standard imports
+import os
+import sys
+import logging
+import time
+import argparse
+import sys
+import re
+import hashlib
+import csv
+import json
+
+# third-party impotts
+import eth_abi
+import confini
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainsyncer.backend import MemBackend
+from chainsyncer.driver import HeadSyncer
+from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.eth.address import to_checksum
+from chainlib.eth.erc20 import ERC20TxFactory
+from chainlib.eth.gas import DefaultGasOracle
+from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.tx import TxFactory
+from chainlib.eth.rpc import jsonrpc_template
+from chainlib.eth.error import EthException
+from chainlib.chain import ChainSpec
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+from cic_types.models.person import Person
+
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_dir = '/usr/local/etc/cic-syncer'
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
+argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='oldchain:1', help='chain spec')
+argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
+argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--offset', type=int, default=0, help='block offset to start syncer from')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('user_dir', type=str, help='user export directory')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+signer_address = None
+keystore = DictKeystore()
+if args.y != None:
+    logg.debug('loading keystore file {}'.format(args.y))
+    signer_address = keystore.import_keystore_file(args.y)
+    logg.debug('now have key for signer address {}'.format(signer_address))
+signer = EIP155Signer(keystore)
+
+queue = args.q
+chain_str = config.get('CIC_CHAIN_SPEC')
+block_offset = 0
+if args.head:
+    block_offset = -1
+else:
+    block_offset = args.offset
+
+chain_spec = ChainSpec.from_chain_str(chain_str)
+old_chain_spec_str = args.old_chain_spec
+
+user_dir = args.user_dir # user_out_dir from import_users.py
+
+token_symbol = args.token_symbol
+
+
+class Handler:
+
+    account_index_add_signature = keccak256_string_to_hex('add(address)')[:8]
+
+    def __init__(self, conn, chain_spec, user_dir, balances, token_address, signer, gas_oracle, nonce_oracle):
+        self.token_address = token_address
+        self.user_dir = user_dir
+        self.balances = balances
+        self.chain_spec = chain_spec
+        self.tx_factory = ERC20TxFactory(signer, gas_oracle, nonce_oracle, chain_spec.network_id())
+
+
+    def name(self):
+        return 'balance_handler'
+
+
+    def filter(self, conn, block, tx, db_session):
+        if tx.payload == None or len(tx.payload) == 0:
+            logg.debug('no payload, skipping {}'.format(tx))
+            return
+
+        if tx.payload[:8] == self.account_index_add_signature:
+            recipient = eth_abi.decode_single('address', bytes.fromhex(tx.payload[-64:]))
+            #original_address = to_checksum(self.addresses[to_checksum(recipient)])
+            user_file = 'new/{}/{}/{}.json'.format(
+                    recipient[2:4].upper(),
+                    recipient[4:6].upper(),
+                    recipient[2:].upper(),
+                    )
+            filepath = os.path.join(self.user_dir, user_file)
+            o = None
+            try:
+                f = open(filepath, 'r')
+                o = json.load(f)
+                f.close()
+            except FileNotFoundError:
+                logg.error('no import record of address {}'.format(recipient))
+                return
+            u = Person.deserialize(o)
+            original_address = u.identities['evm'][old_chain_spec_str][0]
+            balance = self.balances[original_address]
+
+            # TODO: store token object in handler ,get decimals from there
+            multiplier = 10**6
+            balance_full = balance * multiplier
+            logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance_full))
+
+            (tx_hash_hex, o) = self.tx_factory.erc20_transfer(self.token_address, signer_address, recipient, balance_full)
+            logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
+            r = conn.do(o)
+#        except TypeError as e:
+#            logg.warning('typerror {}'.format(e))
+#            pass
+#        except IndexError as e:
+#            logg.warning('indexerror {}'.format(e))
+#            pass
+#        except EthException as e:
+#            logg.error('send error {}'.format(e).ljust(200))
+        #except KeyError as e:
+        #    logg.error('key record not found in imports: {}'.format(e).ljust(200))
+
+
+class BlockGetter:
+
+    def __init__(self, conn, gas_oracle, nonce_oracle, chain_id):
+        self.conn = conn
+        self.tx_factory = ERC20TxFactory(signer=signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, chain_id=chain_id)
+
+
+    def get(self, n):
+        o = block_by_number(n)
+        r = self.conn.do(o)
+        b = None
+        try:
+            b = Block(r)
+        except TypeError as e:
+            if r == None:
+                logg.debug('block not found {}'.format(n))
+            else:
+                logg.error('block retrieve error {}'.format(e))
+        return b
+
+
+def progress_callback(block_number, tx_index, s):
+    sys.stdout.write(s.ljust(200) + "\n")
+
+
+
+def main():
+    global chain_str, block_offset, user_dir
+    
+    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = DefaultGasOracle(conn)
+    nonce_oracle = DefaultNonceOracle(signer_address, conn)
+
+    # Get Token registry address
+    txf = TxFactory(signer=signer, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.network_id())
+    tx = txf.template(signer_address, config.get('CIC_REGISTRY_ADDRESS'))
+
+    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
+    txf.set_code(tx, data)
+    
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    token_index_address = to_checksum(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    logg.info('found token index address {}'.format(token_index_address))
+
+
+    # Get Sarafu token address
+    tx = txf.template(signer_address, token_index_address)
+    data = add_0x(registry_addressof_method)
+    h = hashlib.new('sha256')
+    h.update(token_symbol.encode('utf-8'))
+    z = h.digest()
+    data += eth_abi.encode_single('bytes32', z).hex()
+    txf.set_code(tx, data)
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    try:
+        sarafu_token_address = to_checksum(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    except ValueError as e:
+        logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
+        sys.exit(1)
+    logg.info('found token address {}'.format(sarafu_token_address))
+
+    syncer_backend = MemBackend(chain_str, 0)
+
+    if block_offset == -1:
+        o = block_latest()
+        r = conn.do(o)
+        block_offset = int(strip_0x(r), 16) + 1
+#
+#    addresses = {}
+#    f = open('{}/addresses.csv'.format(user_dir, 'r'))
+#    while True:
+#        l = f.readline()
+#        if l == None:
+#            break
+#        r = l.split(',')
+#        try:
+#            k = r[0]
+#            v = r[1].rstrip()
+#            addresses[k] = v
+#            sys.stdout.write('loading address mapping {} -> {}'.format(k, v).ljust(200) + "\r")
+#        except IndexError as e:
+#            break
+#    f.close()
+
+    # TODO get decimals from token
+    balances = {}
+    f = open('{}/balances.csv'.format(user_dir, 'r'))
+    remove_zeros = 10**6
+    i = 0
+    while True:
+        l = f.readline()
+        if l == None:
+            break
+        r = l.split(',')
+        try:
+            address = to_checksum(r[0])
+            sys.stdout.write('loading balance {} {} {}'.format(i, address, r[1]).ljust(200) + "\r")
+        except ValueError:
+            break
+        balance = int(int(r[1].rstrip()) / remove_zeros)
+        balances[address] = balance
+        i += 1
+
+    f.close()
+
+    syncer_backend.set(block_offset, 0)
+    syncer = HeadSyncer(syncer_backend, progress_callback=progress_callback)
+    handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
+    syncer.add_filter(handler)
+    syncer.loop(1, conn)
+    
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/import_balance.sh

@@ -0,0 +1 @@
+python import_balance.py -c config -i evm:bloxberg:8996 -y /home/lash/tmp/d/keystore/UTC--2021-02-07T09-58-35.341813355Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c -v $@

apps/contract-migration/scripts/import_meta.js

@@ -0,0 +1,120 @@
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+
+const cic = require('cic-client-meta');
+
+//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
+
+const config = new cic.Config('./config'); 
+config.process();
+console.log(config);
+
+
+function sendit(uid, envelope) {
+	const d = envelope.toJSON();
+
+	const contentLength = (new TextEncoder().encode(d)).length;
+	const opts = {
+		method: 'PUT',
+		headers: {
+			'Content-Type': 'application/json',
+			'Content-Length': contentLength,
+			'X-CIC-AUTOMERGE': 'client',
+
+		},
+	};
+	let url = config.get('META_URL');
+	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
+	console.log('posting to url: ' + url + uid);
+	const req = http.request(url + uid, opts, (res) => {
+		res.on('data', process.stdout.write);
+		res.on('end', () => {
+			console.log('result', res.statusCode, res.headers);
+		});
+	});
+	if (!req.write(d)) {
+		console.error('foo', d);
+		process.exit(1);
+	}
+	req.end();
+}
+
+function doOne(keystore, filePath) {
+	const signer = new cic.PGPSigner(keystore);
+	const parts = path.basename(filePath).split('.');
+	const ethereum_address = path.basename(parts[0]);
+
+	cic.User.toKey('0x' + ethereum_address).then((uid) => {
+		const d = fs.readFileSync(filePath, 'utf-8');
+		const o = JSON.parse(d);
+		//console.log(o);
+		fs.unlinkSync(filePath);
+
+		const s = new cic.Syncable(uid, o);
+		s.setSigner(signer);
+		s.onwrap = (env) => {
+			sendit(uid, env);
+		};
+		s.sign();
+	});
+}
+
+const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+pk = fs.readFileSync(privateKeyPath);
+pubk = fs.readFileSync(publicKeyPath);
+
+new cic.PGPKeyStore(
+	config.get('PGP_PASSPHRASE'),
+	pk,
+	pubk,
+	undefined,
+	undefined,
+	importMeta,
+);
+
+const batchSize = 50;
+const batchDelay = 1000;
+const total = parseInt(process.argv[3]);
+const workDir = path.join(process.argv[2], 'meta');
+let count = 0;
+let batchCount = 0;
+
+
+function importMeta(keystore) {
+	let err;
+	let files;
+
+	try {
+		err, files = fs.readdirSync(workDir);
+	} catch {
+		console.error('source directory not yet ready', workDir);
+		setTimeout(importMeta, batchDelay, keystore);
+		return;
+	}
+	let limit = batchSize;
+	if (files.length < limit) {
+		limit = files.length;
+	}
+	for (let i = 0; i < limit; i++) {
+		const file = files[i];
+		if (file.substr(-5) != '.json') {
+			console.debug('skipping file', file);	
+		}
+		const filePath = path.join(workDir, file);
+		doOne(keystore, filePath);
+		count++;
+		batchCount++;
+		if (batchCount == batchSize) {
+			console.debug('reached batch size, breathing');
+			batchCount=0;
+			setTimeout(importMeta, batchDelay, keystore);
+			return;
+		}
+	}
+	if (count == total) {
+		return;
+	}
+	setTimeout(importMeta, 100, keystore);
+}

apps/contract-migration/scripts/import_users.py

@@ -0,0 +1,177 @@
+# standard imports
+import os
+import sys
+import json
+import logging
+import argparse
+import uuid
+import datetime
+import time
+from glob import glob
+
+# third-party imports
+import redis
+import confini
+import celery
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.eth.address import to_checksum
+from cic_types.models.person import Person
+from cic_eth.api.api_task import Api
+from cic_registry.chain import ChainSpec
+from cic_types.processor import generate_metadata_pointer
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
+argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
+argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = argparser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        }
+config.dict_override(args_override, 'cli')
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+r = redis.Redis(redis_host, redis_port, redis_db)
+
+ps = r.pubsub()
+
+user_new_dir = os.path.join(args.user_dir, 'new')
+os.makedirs(user_new_dir)
+
+meta_dir = os.path.join(args.user_dir, 'meta')
+os.makedirs(meta_dir)
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_str = str(chain_spec)
+
+batch_size = args.batch_size
+batch_delay = args.batch_delay
+
+
+def register_eth(i, u):
+    redis_channel = str(uuid.uuid4())
+    ps.subscribe(redis_channel)
+    ps.get_message()
+    api = Api(
+        config.get('CIC_CHAIN_SPEC'),
+        queue=args.q,
+        callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+        callback_task='cic_eth.callbacks.redis.redis',
+        callback_queue=args.q,
+        )
+    t = api.create_account(register=True)
+
+    ps.get_message()
+    m = ps.get_message(timeout=args.timeout)
+    try:
+        r = json.loads(m['data'])
+        address = r['result']
+    except TypeError as e:
+        if m == None:
+            logg.critical('empty response from redis callback (did the service crash?)')
+        else:
+            logg.critical('unexpected response from redis callback: {}'.format(m))
+        sys.exit(1)
+    logg.debug('[{}] register eth {} {}'.format(i, u, address))
+
+    return address
+   
+
+def register_ussd(u):
+    pass
+
+
+if __name__ == '__main__':
+
+    #fi = open(os.path.join(user_out_dir, 'addresses.csv'), 'a')
+
+    i = 0
+    j = 0
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            if y[len(y)-5:] != '.json':
+                continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
+                f.close()
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person.deserialize(o)
+
+            new_address = register_eth(i, u)
+            if u.identities.get('evm') == None:
+                u.identities['evm'] = {}
+            u.identities['evm'][chain_str] = [new_address]
+
+            register_ussd(u)
+
+            new_address_clean = strip_0x(new_address)
+            filepath = os.path.join(
+                    user_new_dir,
+                    new_address_clean[:2].upper(),
+                    new_address_clean[2:4].upper(),
+                    new_address_clean.upper() + '.json',
+                    )
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
+            o = u.serialize()
+            f = open(filepath, 'w')
+            f.write(json.dumps(o))
+            f.close()
+
+            #old_address = to_checksum(add_0x(y[:len(y)-5]))
+            #fi.write('{},{}\n'.format(new_address, old_address))
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
+            meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
+            os.symlink(os.path.realpath(filepath), meta_filepath)
+
+            i += 1
+            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+        
+            j += 1
+            if j == batch_size:
+                time.sleep(batch_delay)
+                j = 0
+
+    #fi.close()

apps/contract-migration/scripts/import_users.sh

@@ -0,0 +1 @@
+python import_users.py -c config --redis-host-callback redis -vv $@

apps/contract-migration/scripts/local/account.py

@@ -0,0 +1,37 @@
+# standard imports
+import logging
+
+# external imports
+from cic_eth.api.api_task import Api
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+queue = 'cic-eth'
+name = 'account'
+
+
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+    """Triggers creation and registration of new account through the custodial cic-eth component.
+
+    It expects the following aux parameters to exist:
+    - redis_host_callback: Redis host name exposed to cic-eth, for callback
+    - redis_port_callback: Redis port exposed to cic-eth, for callback
+    - redis_db: Redis db, for callback
+    - redis_channel: Redis channel, for callback
+    - chain_spec: Chain specification for the chain to execute the transfer on
+
+    See local.noop.do for details on parameters and return values.
+    """
+    logg.debug('running {} {} {}'.format(__name__, token_pair, sender, recipient))
+    api = Api(
+        str(aux['chain_spec']),
+        queue=queue,
+        callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
+        callback_task='cic_eth.callbacks.redis.redis',
+        callback_queue=queue,
+        )
+
+    t = api.create_account(register=True)
+
+    return (None, t, sender_balance, )

apps/contract-migration/scripts/local/noop_traffic.py

@@ -0,0 +1,37 @@
+# standard imports
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+    """Defines the function signature for a traffic generator. The method itself only logs the input parameters.
+
+    If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
+
+    If the task_id position in the return tuple is None, the calling code should interpret the traffic item to have been synchronously completed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
+
+    The balance element of the result is the balance dict passed as argument, with fields updated according to the expected delta as a result of the operation. However, in the event that the generator module dispatches an asynchronous event then there is no guarantee that this balance will actually be the correct result. The caller should take care to periodically re-sync balance from the upstream.
+
+    :param token_pair: Source and destination tokens for the traffic item.
+    :type token_pair: 2-element tuple with cic_registry.token.Token
+    :param sender: Sender address
+    :type sender: str, 0x-hex
+    :param recipient: Recipient address
+    :type recipient: str, 0x-hex
+    :param sender_balance: Sender balance in full decimal resolution
+    :type sender_balance: complex balance dict
+    :param aux: Custom parameters defined by traffic generation client code
+    :type aux: dict
+    :param block_number: Syncer block number position at time of method call
+    :type block_number: number
+    :param tx_index: Syncer block transaction index position at time of method call
+    :type tx_index: number
+    :raises KeyError: Missing required aux element
+    :returns: Exception|None, task_id|None and adjusted_sender_balance respectively
+    :rtype: tuple
+    """
+    logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
+
+    return (None, None, sender_balance, )

apps/contract-migration/scripts/local/transfer.py

@@ -0,0 +1,51 @@
+# standard imports
+import logging
+import random
+
+# external imports
+from cic_eth.api.api_task import Api
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+queue = 'cic-eth'
+name = 'erc20_transfer'
+
+
+def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
+    """Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
+
+    It expects the following aux parameters to exist:
+    - redis_host_callback: Redis host name exposed to cic-eth, for callback
+    - redis_port_callback: Redis port exposed to cic-eth, for callback
+    - redis_db: Redis db, for callback
+    - redis_channel: Redis channel, for callback
+    - chain_spec: Chain specification for the chain to execute the transfer on
+
+    See local.noop.do for details on parameters and return values.
+    """
+    logg.debug('running {} {} {} {}'.format(__name__, token_pair, sender, recipient))
+
+    decimals = token_pair[0].decimals()
+
+    sender_balance_value = sender_balance['balance_network'] - sender_balance['balance_outgoing']
+
+    balance_units = int(sender_balance_value / decimals)
+
+    if balance_units <= 0:
+        return (AttributeError('sender {} has zero balance'), None, 0,)
+
+    spend_units = random.randint(1, balance_units)
+    spend_value = spend_units * decimals
+
+    api = Api(
+        str(aux['chain_spec']),
+        queue=queue,
+        callback_param='{}:{}:{}:{}'.format(aux['redis_host_callback'], aux['redis_port_callback'], aux['redis_db'], aux['redis_channel']),
+        callback_task='cic_eth.callbacks.redis.redis',
+        callback_queue=queue,
+        )
+    t = api.transfer(sender, recipient, spend_value, token_pair[0].symbol())
+
+    sender_balance['balance_outgoing'] += spend_value
+    return (None, t, sender_balance,)

apps/contract-migration/scripts/requirements.txt

@@ -0,0 +1,12 @@
+psycopg2==2.8.6
+chainlib~=0.0.1a15
+chainsyncer~=0.0.1a10
+cic-eth==0.10.0a30+build.fdb16130
+cic-registry~=0.5.3a19
+confini~=0.3.6rc3
+celery==4.4.7
+redis==3.5.3
+hexathon~=0.0.1a3
+faker==4.17.1
+cic-types==0.1.0a7+build.1c254367
+eth-accounts-index~=0.0.10a10

apps/contract-migration/scripts/traffic.py

@@ -0,0 +1,101 @@
+# standard imports
+import os
+import logging
+import re
+import sys
+import json
+
+# external imports
+import redis
+import celery
+from chainsyncer.backend import MemBackend
+from chainsyncer.driver import HeadSyncer
+from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.gas import DefaultGasOracle
+from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.block import block_latest
+from hexathon import strip_0x
+
+# local imports
+import common
+from cmd.traffic import (
+        TrafficItem,
+        TrafficRouter,
+        TrafficProvisioner,
+        TrafficSyncHandler,
+        )
+from cmd.traffic import add_args as add_traffic_args
+
+
+# common basics
+script_dir = os.path.realpath(os.path.dirname(__file__))
+logg = common.log.create()
+argparser = common.argparse.create(script_dir, common.argparse.full_template)
+argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
+args = common.argparse.parse(argparser, logg)
+config = common.config.create(args.c, args, args.env_prefix)
+
+# map custom args to local config entries
+batchsize = args.batch_size
+if batchsize < 1:
+    batchsize = 1
+logg.info('batch size {}'.format(batchsize))
+config.add(batchsize, '_BATCH_SIZE', True)
+
+config.add(args.redis_host_callback, '_REDIS_HOST_CALLBACK', True)
+config.add(args.redis_port_callback, '_REDIS_PORT_CALLBACK', True)
+
+config.add(args.y, '_KEYSTORE_FILE', True)
+
+config.add(args.q, '_CELERY_QUEUE', True)
+
+common.config.log(config)
+
+
+def main():
+    # create signer (not currently in use, but needs to be accessible for custom traffic item generators)
+    (signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
+
+    # connect to celery
+    celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+    # set up registry
+    w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
+    registry = common.registry.init_legacy(config, w3)
+
+    # Connect to blockchain with chainlib
+    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = DefaultGasOracle(conn)
+    nonce_oracle = DefaultNonceOracle(signer_address, conn)
+
+    # Set up magic traffic handler
+    traffic_router = TrafficRouter()
+    traffic_router.apply_import_dict(config.all(), config)
+    handler = TrafficSyncHandler(config, traffic_router)
+
+    # Set up syncer
+    syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
+    o = block_latest()
+    r = conn.do(o)
+    block_offset = int(strip_0x(r), 16) + 1
+    syncer_backend.set(block_offset, 0)
+
+    # Set up provisioner for common task input data
+    TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
+    TrafficProvisioner.default_aux = {
+            'chain_spec': config.get('CIC_CHAIN_SPEC'),
+            'registry': registry,
+            'redis_host_callback': config.get('_REDIS_HOST_CALLBACK'),
+            'redis_port_callback': config.get('_REDIS_PORT_CALLBACK'),
+            'redis_db': config.get('REDIS_DB'),
+            'api_queue': config.get('_CELERY_QUEUE'),
+            }
+
+    syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
+    syncer.add_filter(handler)
+    syncer.loop(1, conn)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/verify.py

@@ -0,0 +1,306 @@
+# standard imports
+import os
+import sys
+import logging
+import time
+import argparse
+import sys
+import re
+import hashlib
+import csv
+import json
+import urllib
+
+# external impotts
+import celery
+import eth_abi
+import confini
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from cic_registry.chain import ChainSpec
+from chainsyncer.backend import MemBackend
+from chainsyncer.driver import HeadSyncer
+from chainlib.eth.connection import HTTPConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.eth.address import to_checksum
+from chainlib.eth.erc20 import ERC20TxFactory
+from chainlib.eth.gas import DefaultGasOracle
+from chainlib.eth.nonce import DefaultNonceOracle
+from chainlib.eth.tx import TxFactory
+from chainlib.eth.rpc import jsonrpc_template
+from chainlib.eth.error import EthException
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+from cic_eth.api.api_admin import AdminApi
+from cic_types.models.person import (
+        Person,
+        generate_metadata_pointer,
+        )
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+config_dir = '/usr/local/etc/cic-syncer'
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='oldchain:1', help='chain spec')
+argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
+argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
+argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('user_dir', type=str, help='user export directory')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_str = str(chain_spec)
+old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
+old_chain_str = str(old_chain_spec)
+user_dir = args.user_dir # user_out_dir from import_users.py
+meta_url = args.meta_provider
+
+
+class VerifierError(Exception):
+
+    def __init__(self, e, c):
+        super(VerifierError, self).__init__(e)
+        self.c = c
+
+
+    def __str__(self):
+        super_error = super(VerifierError, self).__str__()
+        return '[{}] {}'.format(self.c, super_error)
+
+
+class Verifier:
+
+    def __init__(self, conn, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, data_dir):
+        self.conn = conn
+        self.gas_oracle = gas_oracle
+        self.chain_spec = chain_spec
+        self.index_address = index_address
+        self.token_address = token_address
+        self.erc20_tx_factory = ERC20TxFactory(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
+        self.tx_factory = TxFactory(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
+        self.api = cic_eth_api
+        self.data_dir = data_dir
+
+
+    def verify_accounts_index(self, address):
+        tx = self.tx_factory.template(ZERO_ADDRESS, self.index_address)
+        data = keccak256_string_to_hex('have(address)')[:8]
+        data += eth_abi.encode_single('address', address).hex()
+        tx = self.tx_factory.set_code(tx, data)
+        tx = self.tx_factory.normalize(tx)
+        o = jsonrpc_template()
+        o['method'] = 'eth_call'
+        o['params'].append(tx)
+        r = self.conn.do(o)
+        logg.debug('index check for {}: {}'.format(address, r))
+        n = eth_abi.decode_single('uint256', bytes.fromhex(strip_0x(r)))
+        if n != 1:
+            raise VerifierError(n, 'accounts index')
+
+
+    def verify_balance(self, address, balance):
+        o = self.erc20_tx_factory.erc20_balance(self.token_address, address)
+        r = self.conn.do(o)
+        actual_balance = int(strip_0x(r), 16)
+        logg.debug('balance for {}: {}'.format(address, balance))
+        if balance != actual_balance:
+            raise VerifierError((actual_balance, balance), 'balance')
+
+
+    def verify_local_key(self, address):
+        r = self.api.have_account(address, str(self.chain_spec))
+        logg.debug('verify local key result {}'.format(r))
+        if r != address:
+            raise VerifierError((address, r), 'local key')
+
+
+    def verify_metadata(self, address):
+        k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person')
+        url = os.path.join(meta_url, k)
+        logg.debug('verify metadata url {}'.format(url))
+        try:
+            res = urllib.request.urlopen(url)
+        except urllib.error.HTTPError as e:
+            raise VerifierError(
+                    '({}) {}'.format(url, e),
+                    'metadata (person)',
+                    )
+        b = res.read()
+        o_retrieved = json.loads(b.decode('utf-8'))
+
+        upper_address = strip_0x(address).upper()
+        f = open(os.path.join(
+            self.data_dir,
+            'new',
+            upper_address[:2],
+            upper_address[2:4],
+            upper_address + '.json',
+            ), 'r'
+            )
+        o_original = json.load(f)
+        f.close()
+
+        if o_original != o_retrieved:
+            raise VerifierError(o_retrieved, 'metadata (person)')
+
+
+    def verify(self, address, balance):
+        logg.debug('verify {} {}'.format(address, balance))
+    
+        try:
+            self.verify_local_key(address)
+            self.verify_accounts_index(address)
+            self.verify_balance(address, balance)
+            self.verify_metadata(address)
+        except VerifierError as e:
+            logg.critical('verification failed: {}'.format(e))
+            sys.exit(1)
+
+
+class MockClient:
+
+    w3 = None
+
+def main():
+    global chain_str, block_offset, user_dir
+    
+    conn = HTTPConnection(config.get('ETH_PROVIDER'))
+    gas_oracle = DefaultGasOracle(conn)
+
+    # Get Token registry address
+    txf = TxFactory(signer=None, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.chain_id())
+    tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS'))
+
+    registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
+    txf.set_code(tx, data)
+    
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    print('r {}'.format(r))
+    token_index_address = to_checksum(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    logg.info('found token index address {}'.format(token_index_address))
+
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'AccountRegistry').hex()
+    txf.set_code(tx, data)
+    
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    account_index_address = to_checksum(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    logg.info('found account index address {}'.format(account_index_address))
+
+
+    # Get Sarafu token address
+    tx = txf.template(ZERO_ADDRESS, token_index_address)
+    data = add_0x(registry_addressof_method)
+    h = hashlib.new('sha256')
+    h.update(b'SRF')
+    z = h.digest()
+    data += eth_abi.encode_single('bytes32', z).hex()
+    txf.set_code(tx, data)
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    print('r {}'.format(r))
+    sarafu_token_address = to_checksum(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    logg.info('found token address {}'.format(sarafu_token_address))
+
+    balances = {}
+    f = open('{}/balances.csv'.format(user_dir, 'r'))
+    remove_zeros = 10**12
+    i = 0
+    while True:
+        l = f.readline()
+        if l == None:
+            break
+        r = l.split(',')
+        try:
+            address = to_checksum(r[0])
+            sys.stdout.write('loading balance {} {}'.format(i, address).ljust(200) + "\r")
+        except ValueError:
+            break
+        balance = int(int(r[1].rstrip()) / remove_zeros)
+        balances[address] = balance
+        i += 1
+
+    f.close()
+
+    api = AdminApi(MockClient())
+
+    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, user_dir)
+
+    user_new_dir = os.path.join(user_dir, 'new')
+    for x in os.walk(user_new_dir):
+        for y in x[2]:
+            if y[len(y)-5:] != '.json':
+                continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
+                f.close()
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+
+            u = Person.deserialize(o)
+            logg.debug('data {}'.format(u.identities['evm']))
+
+            new_address = u.identities['evm'][chain_str][0]
+            old_address = u.identities['evm'][old_chain_str][0]
+            balance = balances[old_address]
+            logg.debug('checking {} -> {} = {}'.format(old_address, new_address, balance))
+
+            verifier.verify(new_address, balance)
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/seed_cic_eth.sh

@@ -20,7 +20,8 @@ debug='-vv'
 abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
 gas_amount=100000000000000000000000
 token_amount=${gas_amount}
-faucet_amount=1000000000
+#faucet_amount=1000000000
+faucet_amount=0
 env_out_file=${CIC_DATA_DIR}/.env_seed
 init_level_file=${CIC_DATA_DIR}/.init
 truncate $env_out_file -s 0
@@ -29,7 +30,8 @@ truncate $env_out_file -s 0
 set -e
 set -a
 
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a25 chainlib==0.0.1a11
+# We need to not install these here...
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a30+build.fdb16130 chainlib==0.0.1a16
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
@@ -87,13 +89,13 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 
 >&2 echo "deploy transfer authorization contract"
-CIC_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-approval-escrow-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --approver $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER -w $debug`
+#CIC_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-approval-escrow-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --approver $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER -w $debug`
-echo CIC_APPROVAL_ESCROW_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS  >> $env_out_file
+#echo CIC_APPROVAL_ESCROW_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS  >> $env_out_file
-export CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS 
+#export CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS 
 
 # Register transfer approval contract
->&2 echo "add transfer approval request contract to registry"
+#>&2 echo "add transfer approval request contract to registry"
->&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TransferApproval -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $CIC_TRANSFER_AUTHORIZATION_ADDRESS
+#>&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k TransferApproval -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $CIC_TRANSFER_AUTHORIZATION_ADDRESS
 
 
 # Deploy one-time token faucet for newly created token

apps/contract-migration/testdata/pgp/privatekeys_meta.asc

@@ -0,0 +1,241 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lQWGBF+hSOgBDACpkPQEjADjnQtjmAsdPYpx5N+OMJBYj1DAoIYsDtV6vbcBJQt9
+4Om3xl7RBhv9m2oLgzPsiRwjCEFRWyNSu0BUp5CFjcXfm0S4K2egx4erFnTnSSC9
+S6tmVNrVNEXvScE6sKAnmJ7JNX1ExJuEiWPbUDRWJ1hoI9+AR+8EONeJRLo/j0Np
++S4IFDn0PsxdT+SB0GY0z2cEgjvjoPr4lW9IAb8Ft9TDYp+mOzejn1Fg7CuIrlBR
+SAv+sj7bVQw15dh1SpbwtS5xxubCa8ExEGI4ByXmeXdR0KZJ+EA5ksO0iSsQ/6ip
+SOdSg+i0niOClFNm1P/OhbUsYAxCUfiX654FMn2zoxVBEjJ3e7l0pH7ktodaxEct
+PofQLBA9LSDUIejqJsU0npw/DHDD2uvxG+/A6lgV9L8ETlvgp8RzeOCf2bHuiKYY
+z87txvkFwsXgU1+TZxbk+mtCBbngsVPLNarY/KGkVJL+yhcHRD0Pl4wXUd6auQuY
+6vQ9AuKiCT1We2sAEQEAAf4HAwK2fexgxtQ8CfgdeIlzdeY9K+HZL18brETddoya
+3BeC1MSH7gxXqtCVQ5qdBk27J4wlGl0H83kYSCeVQs6hmrSrv8JCErguIdpZIJ/D
+kcjGlGrOELfnXeif0VfUZN3LWxJZizCIS8I9F8VKD9c57nZEcbWcKTLizV0j1BeT
+sdrumt/3UDhpCJTj1q3biRsiUbpmX+jPlRWN3OeSZJaRRyy4FnzTs3bndBYmkOsk
+ZNKRk7jRNEU/LItbABStuP2zDrZsampVntKcNRXBVE2170t4T/Q4Gc0ckz4ohprY
+lGykE2DdwapCdcKWccVXhM+svDwoLf+g4kjKuCE7R11v6rZlRxYrfquZXwtUx0DB
+17x+JqyBaacyWm3Vq65DcNyiQw2NqCPdJU+iZoOGaermKIz3BqwxY+WE0HyjxQkH
+P5KUpKQTmsTIlwHWFOVDYMRUUvD7P7XiElOBECDb3bJL9+z2SHZWTE6OaZKnmBFf
+ZTdXgtGe/Ctx97PgWZOwM500Q45QC+D59NCYtXRtqi9WCLGsZpQbSZmojIUOJRuD
+s5un+8lA3T0BhlJS4DC9CgN8Lxs4kT/XV/LYiXU4Z8MWEahurEbpDwH6YNzGktUR
+zuE9HOe0fesdrOV0Sk2aol5CCRj3vTcsROTFUcT6UYPq28vy0U3zUJVvyNa0swk7
+PUiB+xhCi24Z9dy+0F1q1L20tJ/YCjC/tyLI36Rkl85PnoviwOOOll0+/claf8BV
+e9x43voYe0o8Q7ttU0aFxVH/lGaTRyVMcXJFw0EPLuwIrcGrcauatcbO7lI2nVww
+kBZFepWh7JBRl2x5SXnvTqLnWp2D5w1viUPcBN5xAj9IKOWrRr2kIRLiOVIGh9ta
+Hiio2+vg/ZmhsmMzA36xYkH6NvyjNAeLUgTVfEAtsCrRXdW8FYTTGOKDmw55Ma+P
+Ej1QWWzbwqPU+h+AOyklVZ1xGncxTkyad5niXYEzBJbbA01QoAtZeY7kSg0ae6uD
+YPRQGf+0G6YlCKPOZjBH8AvbedhyjIKZhBT8M2sHIKSESPP0Vs8yS16rYzy8o6+e
+7uYsIST+PMWXxDpJHmN2Ks5uo789+TiHfffHzbsTuevNIwk9FbMA6gpDdtMCaFZX
+abZxz6sxLv9MoWjIKR2vDZKHjK5DVlJv4V1De3gTsCmfQhhToPzNGGFEI00aBki6
+IJIyisOuZtQiXhHy1vN499evLDwkc8u1S6ex6Q7blp75IQmJJ4/WG+XA55D+Mfnd
+QSbV+zP9WQu66RR+RDsx+c7L7Bg58bqXE3bPcoLzaHOmDwpw74BGmNu84dfmyKbI
+FocSAWP+Oe3sBxcdE7aVS+FB+B30It25LbQeTWVyIE1hbiA8bWVybWFuQGdyZXlz
+a3VsbC5jb20+iQHUBBMBCAA+FiEE8/r2aOgu9RJNUYe67yb0aCND9pIFAl+hSOgC
+GwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ7yb0aCND9pLwiwwA
+hFJbAyUK05TJKfDz81757N472STtB8sfr0auwmRr8Zs1utHRVM0b/jkjTuo4uJNr
+7YVVKTKgE7+rJ+pwhm3wlTQ44LVLjByWAi/7NWg3E9b2elm+qkfgm/RfFt3vkuOx
+GSyZyIFFh+/twv6iABPvr6w7MZwrFaS0UP3g1VGa5TFqg6KNxod9H/gPLxv45lut
+Xf3VvBZTJpr1pxn7aLHlFzEyIgNZbP/N1QF44GSrN/k0DfL631sZjauUXaZXbi5x
+GsKKCYwJ1g3q587pi6mTdTV3n0hKgVuipO8hGy5++YeOv+hXsCxDwyZ+Shv+qavd
+/SapxYgCdEueuwONIFfsIsWCd3SCcjKXicTTEFMu8nvBmf7xuo2hv6vEOxoijlXV
++4LkGrskdB8ZMg8PywEx6DLmDokgnAhTLrTc1ShbkOtQ3yNjjyFK7BDpqobsJal6
+d8SpbhccUJLepaSmsk0CgJsTjhAl6EwX0EYgTo3kP5fScqrbD8VwQaT8CcE4rCV4
+nQWGBF+hSOgBDADHtpTT1k4x+6FN5OeURpKAaIsoPHghkJ2lb6yWmESCa+DaR6GX
+AKlbd0L9UMcXLqnaCn4SpZvbf8hP4fJRgWdRl5uVN/rmyVbZLUVjM8NcVdFRIrTs
+Nyu4mLBmydc3iA/90sCTEOj9e7DSvxLmmLFjpwM5xXLd6z0l6+9G+woNmARXVS3V
+/RryFntyKC3ATCqVlJoQBG45Tj2gMIunpadTJXWmdioooeGW3sLeUv5MM98mSB4S
+jKRlJqGPNjx5lO6MmJbZeXZ/L/aO6EsXUQD2h82Wphll4rpGYWPiHTCYqZYiqNYr
+6E3xUpzcvWVp3uCYVJWP6Ds117p7BoyKVz00yxC9ledF3eppktZWqFVowCMihQE3
+676L3DDTZsnJf1/8xKUh5U2Mj3lBvjlvCECKi00qo8b1mn/OklQjJ5T4WzTrH6X+
+/zpez8ZkmtcOayHdUKD/64roZ9dXbXG/hp5A+UWj8oSVYKg2QNAwAnZ+aiZ2KVRE
+/Y61DCgFg6Ccx/cAEQEAAf4HAwLvYCWT4e84+PjE5pF2+FQAEMmVwTUm5pv9XhBd
+Lnw68o0N/OGhi8LLMuhiI22u60W+//6Pknws1FfHI6zVeHZ1V4DcE8JtJcbSqGk4
+X1IFSXB60kduyCDLxq7PgqlLac2vr8jOsZAGTM8okJ3jrCrXd0oEPMIPQzo4RKZJ
+PeBwUyzTU1+jA5pZjpj+DgpBoC5uZTeGLB2ftbN/w3wBUsZZR3q7WiM7p34+xvST
+Obe1u5PerN5BH6zizvCWr2yRGF0RdUYz6q0kQdUorDjqrowYlNi5Em3RIyK1IoFR
+MpcZPf9zMODMPZ2VlBruDQu40thr/Ho/5w15QmJ/7SmstGreKerI2jUziHPa4XMo
+pUS+jGpIC3pZRa2Y+4UpgtYciuc5CusxzAOYbSh+py1kLuL/tkI54QsLYG2gDcd5
+dGz/jxun4irlZ/Iy1GtGM5+SrREktwRD2lIou295XqWOHwJPahPG7xb172VeUfoK
+AObWonSJ9uWcsG/FKNo1at9ENA1x+zUV6s+F8B78snQJ96iFIHtz+5NAXQR0pEnD
+i7DIHSSGaeZdj2NcbmM6t5/dyN40KHwymYxrItHGL19uRUJiJfgGeI9+dNCRfMOU
+4YK+/kiGqH4Yr4WNBmF8zeP51gWDCspCzMKp+Z3wtGXx7j+147iWqW/6ARZ5krJa
+oWF+gmesFYFWz54Lr/IuA4usaRSbt+ZnXpJTQip74NOrKF7JpXeVMWY7BN5wcnyO
+SXrJrg3xKupq/oZlHnpGiL/UGrr9NZmT/ajg1xjVArkWD0YkwnTRP+CBXLNyrhtd
+eLzClaDiv8wXMIm1uWImX7zVv+H7ngfU2aQOMQiU1BbV+pU69bAVdD73glniID1R
+HYJHFhOxyF9nFTfBkPM/3rNuJDURLyMhkIyZ3OhIOiDv+5W2Q1swhlfLI5Tf7eCv
+wxMGBM508I7TuemCuUk0oqsDnm1Z+oCEWqEI06qvMpGPPO9HU90kELdGDVlnVo6J
+wP9UOgXa9LsywaFO+otV/spEpntQXXmHgzLgESyCxe0iHSSv9GxBLk1lTTCgi2qW
+B8KI60TJiK3+jTiBR422XMQs5mkvDqOBLuX4dpOuosewPwAEfrl9ZF6z1f2TVVwk
+piuHzNcz0NaLWkIrfDb2wIEPEzdCU+pVSfrh3g4S8dMiAK0IMWTYvye5xZ1bd9tN
+vwI7ottJiJDk97ScnBU6b//Pb8QbQjjtXbssrfkBaJH/e0cE2WGkUzIQd6sJ8qnq
+7mofMB7zU9iD0C3B5BCSnh36vKtGecosrpUmRNfGm79DattdQqAzZSY8rBHvJ+22
+KWF1VcqZVxYk5B33jc0p7tXjix2xyMc9IYkBvAQYAQgAJhYhBPP69mjoLvUSTVGH
+uu8m9GgjQ/aSBQJfoUjoAhsMBQkDwmcAAAoJEO8m9GgjQ/aSIPcL/3jqL2A2SmC+
+s0BO4vMPEfCpa2gZ/vo1azzjUieZu5WhIxb5ik0V6T75EW5F0OeZj9qXI06gW+IM
+8+C6ImUgaR3l47UjBiBPq+uKO9QuT/nOtbSs2dXoTNCLMQN7MlrdUBix+lnqZZGS
+Dgh6n/uVyAYw8Sh4c3/3thHUiR7xzVKGxAKDT8LoVjhHshTzYuQq8MqlfvwVI4eE
+SLaryQ+Y+j5+VLDzSLgPAnnIqF/ui2JQjefJxm/VLoYNaPAGdqoz/u/R0Tmz94bZ
+UfLjgQaDoUpnxYywK2JGlf3mPZ3PNWjxJzuQTF5Ge5bz/TylnRYIyBT7KD7oaKHO
+62fhDbYPJ4f94iZN4B6nnTAeP34zFDlkUbX4AHudXU7bvxT5OUk9x9c2tj7xwxQH
+aEhq2+JsYW0EVw27RLhbymnBfLjVVUktNF0nQGvU2TEocw4pr2ZkDHQkSnlbNa4k
+ujlL7VzbpnEgyOmi5er9GaIuVSVADovBu+pz/Ov1y/3jUe8hZ/KleZUFhgRfoUka
+AQwA2r2HiLvpnclyZMoeck1LFoVyEU/CjPcYWF1B76ekO9mrlYvbKsnsyL0WcuEq
+wCmHdLk70i743Fn21WQK4uvvlvrEpev9aj9DihyLctv4qrPm6wAU/Xibf75tg1iR
+L+muMQfv6hQhjdhwkYFx/7XQ6UWkEibqFS7xJwrhz9lHL4KTA4sO5PeW713+mpz7
+tM5RmGV6NOQAyEEfAv6OawlWk0f5o8xngIoyo2BS5qIeEBO+iz45+GG8GQC6XufO
+Ix7VVl++ZpsxZKtDq/AXfAskxfLRwZMqH9Db5pPMzrL1bPV16AwoWqhAGd2HIMkO
+DLEC5XTGIKCqO5+n288rHhAJTqFmE7TpAo+Eb0Tkk4jfm6LyRonmQGpu/Zxa53n5
+D6d+AgYWAMeHkEthWJkES4mKpZu4nV21+n9mynnPg8wzthL705Q6IBjtlxX8EP6e
+eRFE1BUCNp2RZttTSdI+8iwzYsGOJdJeeXeLOGhvU9/PLkRj9jgZLgCLAo1QGo2o
+xetZABEBAAH+BwMCYxRGMNwlr/T4SMsvXNo05Y9gvmJ/vNY89nIF3J/WsBcBChWT
+MAls+3BDxHbEjjXb4sWQeGE5IxNUv1TMjZ1CLDAzga5Rm/KICYl3Yo6hWKRWk6qx
+fdacQ8Z5aHXtQQ8qJxX2dIPbZtTkmhlCIj3B1H7xThFF/b+oh1+hV8F6kWuKZ2jJ
+3cm+hy/sBpnENU0EOMvDAcQZ5QikmyyYPe03MMMEhl4Q51NbwFZi1Fnb1qYieGdh
+lRX+92/+V5okFj0zTKLTtglwBcYobAs8Vlwa0bC9Bw5U21U1b4uU0wVrOHsdnGZp
+LLZFXxON1t8ZSdNixus2kuUZDuCX2xGKestufSL+6rgf2pQAcoHI61uwwQT1LZGf
+wmAieWHy6v+KWBODmTO6P6a3w1mCfI9gVATfWSuhbuIbqgUMLBWsimUB0pdWTwX9
+oVKMS+OxL+ZHPoaixFwkFz6GqCJVRJ+rKafmjgOfmCWwCl3VoqGp/fkZKKgrBprP
+HB1aIUkiiOvgPOW3ZbjG5SwBFSdjKt+KiWVEAVKnl9XAtzB+SS4fk2aKvezNB3Yf
+LW6wmq4U+OkXEfGLpk1KJ81wb/D/ULAI3FRauxB5drlTwJ2mrWqeuJsR4A2sy49t
+LKapWlbDlOvFtXtynultB/mc8mhphiaJdMoSKuOspiSSXNk/On9UdSVn0tDDlZEh
+QU6iYwtvATo3Q1/RWZI74V4IZqt8R1d0Y+HIn8SNfUp3Zcs5vcqb+YvUGzqveLnl
+Dn6ndCrLq7spDAFk9WVObApFYtnuEt9pKmrluQczckXwb7yH6CFCgoF+DjMdYi+L
+En869iCp+jW2SVjo0q6SOODrIB2aiIEW8PoRIC/vFSTVgv528s7A6DjXeh0c/hkb
+Ud3b5KNCJosz3RArv7ljiYq58Kj4scFr45orj80XulsLbr+tFaN3VNKgEsBDp0ZD
+wgISoJr6fzAttqTKsPdzHGh3lNY5RNuP4r3VTgu3dN2ZxIDXxhiIWhbWiXmBz2p0
+Y+TRwtgoUluDnMJhFDx8m1w07AqrLT7ivISgHrHwcDZgDGZ8l6rviDk3b8AsKtqY
+r//yTXMpTC0kgEb89oHqRd2NiCS4R+2bjWZG+2CtQ7TpCYscbdNdYucEhQGiAUMk
+7MJISwC0VSw3xesuHcF8Nx+5vY+GlTrZDIkrS0qKkmOvwSWP0xtSWa1jvIvsd4UK
+yoHgDCdvME9UBeIrfqa9JfKAPFE1iGN3uXmq04hwnWwu/vybFA6IjeA2tfbFWWaO
+oh2YyXDqhuL8HbUMESiyPOybFXm3aw6HRgIr3OM/R4O6Hv02zNeWJXnkATTKgTje
+1xkJuQNXY5N6bpBPkw01Kr20IkJlYXN0IE1hbiA8YmVhc3RtYW5AZ3JleXNrdWxs
+LmNvbT6JAdQEEwEIAD4WIQT2ReBH7lvE4oJMlNtC3JHPqKugKwUCX6FJGgIbAwUJ
+A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBC3JHPqKugK25hC/9VF1fe
+kj0IKnrOJRUcK/Cv4RBowl60V91w27ApsoP2awEJiFhY7qRijtkA3NKrT3tke7aT
+nC3yAJ8SFOmvIAC94ijb7Iv97xkG+1IIz8pvru9y+dzd2NnvCkts8gFF0CI/xtEM
+E90rU3Pay9B5IyrpP++UdmSmnp3Neuwi94BZDfMlqkeiYOzWWSeYbmSSVfKTXeBd
+UuTyfRI4m/bPbh6gegOB/XdgSIrNY74D0nR3np0I+s0IGZepK24kgBKfUPwRDk7f
+98PXCh29iL3xH+TBxu30WHq7xKmPoXxCRyFLtnKF0MN5Ib276fHnJZM+hXf5i/1E
+Pi4NLnk86e7fNI69hwiUd1msEt3VmZWe7anJe/1p3sSXwbQGhhGWM5K41/rQ1CZ9
+qD95d6wkHRSc0n4z78qxgYV73yJHinN8xIFnPWbopPPIJbELSoM3IEpHobsj95pH
+4hzZAPSmDfOfLzV1G2ec1QPfWnTqUriUt7edDs4//7Cczj6sRh2B6ax2diCdBYYE
+X6FJGgEMAMqxn5io6fWKnMz8h5THqp4gEzDuoImapfMKbAKcxEtJmcLkvn+4ufEP
+/hcll66InqJHsqMOrdb+zbduCruYWpizhqCIGSsuRu7+ZEEkQFmF5juCOV/5qKQJ
+gZZmxSKbRtboapMRR/jmg1pvhnUG7wJOGWi7qv+iRdsWKskDO7tUQE34+ID7IwfD
+Ze2fbFKxf66nPlUunF8aMglsvGmtCEzm/xwjunHnmoqZBQIzTdEXIaEwhVosbgY7
+A1iwOJ/gT2dcF2KJa7tygrtcbgdVzYCibynwtlvDGXukweuYLQFsObyBG3UHRhJg
+61p7n344sy1U9uwCP3/pVCr9bNY9mLZpCgHFkqxErmB8cWouQkbwnqxQFm21KtGF
+zjUawuKBXVtDEeA8C5Ha0sx7lw5JrX8GD3EL60qKWjqujJsR1kyijXx1No7Xr9NW
+WuPoIDYH06ZoYE+j065VTRqZIGr3NjUZnqT7s9M41roQMnKAzRBXousRXRW9dXfS
+5YIG4nWTlwARAQAB/gcDAsDkrCv+8rcr+OKtXIf6oDyx2tbPr+tpZJII4Lqchego
+FTB0/GoqHF+iu+uYDCuzkwXBSIAPTCudjhZ+0cwvO4WgjdqGC3zqCc4bCP68cItN
+fcLsof5L7rJ8BXX/0YXhua3gFtWGw/EtGpO4tqFCrzkpgEvovP/N1CLFaHnRzWSN
+AE0ebsdfTCRYjWuZiAKlWjKCMNmHrE7AB5TraGqclP5GlY28lm7T9KXnNXixFaaR
+pLDaLFyGZDEilEjkCKx1cyg3oBNeqUP/Ra6DYEF3PWTGpX8PxBF4lA2qnq+XuUK8
+30Nz2upz38Hb1jG1sdNlYEWLv05bFc0vMLWmzwAd6Ij0I4C6WdsakT213frlFw4w
++hoilBcrW5+UOBc1dbU3UFh72khzLdKz1aUVC2N5HN4gS7WTSw3of0sOy+LR4JaH
+O8kSlZAIMXCooBDKr/R6x97A5sq8zMQ0vI0LSN2FpfwgdApwWLJYFBAy7ZJU9efO
+0f79yEqk7d9xFEsIIpn2R+zVcUdAvj9/EDnbu/QaEj9jl+2PH6arqp1AGurF0Dp/
+cB4T7ZCuaunIet5MqEN6Ac4WdjpEcC7tKjB4cQ53Y2f9zCSouXa4JUypsgm4AQZX
+O6hejChIiFC31T2x0a3M6eD6+XNw64ShdyX6i153xOe07d78Zq5qnhw+Vz28FQjZ
+Lmvbm1sj26WaZmLH6LzyjAJjjV4YH7ijwLjUMdeKeuato0fsCff/cVO7MKDj0aPe
+zQCWSbqcnsxl1Agsop82k6Y3W9gco0tVhqYgIwmCjsWvCXAWILk2yIuxIxINRNqX
+Pt+TYZR0BuDAUK4x15fUT5tXuu7DmmnmZlWlaba44dtJPB3SFtEM7jJ7xNF0zie3
+G+6hxtZSmrjfYHBK2ZD+2veP1j0P/tYD/8n/rZx7u4pHuJiiZksj6ZwGF61HP03Y
+zOu0LhhtrTQ1yYaE52mUdkLlQRNwD5b+qDkqN9/PeSzIoDesRVuVE2rbr7sIJ3GS
+jxZHin4kHsxzqFQmecm1ctPgx+BpksMPok+MJGzSZ3OwE5tuK0VLvEIcMritgfM7
+BixcNPyDv+RkwEguSMlsHq7Lrr+LXtR2XmeFBMgDiulZ5FUxoGiGBfyyG3bsAug9
+D4ceg0yUh5HjTTjqoQ1Qo95Wi/yF64WPcZDllJ2BaBznDxlESNR/jmVhwQsrqOdn
+NF54SCpU0e3N+XBsdHd4cRsS2lxemn5boK67/FVKdUmVgxo9VnAlreoeB+cVFHap
+gfPSXD5V/MuFMiKSFuF63s61EP1T1Okl1cvrE2oAsJTXMgCIVSZdyLUwYKmjsMVD
+rtfQXcoOQgFbA90qj7uOOtZId04NiQG8BBgBCAAmFiEE9kXgR+5bxOKCTJTbQtyR
+z6iroCsFAl+hSRoCGwwFCQPCZwAACgkQQtyRz6iroCt8igwAgopqy+UgxJ7oTL2z
+vOgL1ez7bv+E/U1/7Rdy5MHwr4WF6oZRpIBlgv3GXXeIFH9bFdDhgyPKgh+Tz24J
+BL+7YjUtWGe/G/pmmNK1YazB/OxrwiGFpTCyk1zhxEkhMu7Hu3LgD571K+4TUUpa
+PCqEeoBBg6O3T29DH1AxpWpEPGXlOrRDHYgVziEpLdUNahAjF53auNWvya+Vc2qZ
+wM4NFt608LLf7J5yIA2vbsvf6+gVopPE3whXESKXo08B2hC1f3Pr9/Tgt6oIvy9/
+dAcTMalxRyyc42E2wX5kyzDlfhY9kqaNNfaGMZJO5g//gB7BdtrAfo/LhWtary/Y
+fAOtbbnMYkf+HODAPZItaIjMZngBM0c0m78YoCetAQE8uBFK6aXmht3BZGPOwgyZ
+pK5QT6ClYst2N9ca3tPUEfnddotKySmCEk/JWtu5/0lFl75WzHulc7iUNGJmnUff
+VZyH12CjBWsTtqombHDkdEKFocavqpVcCCbKbtW5GZhuZC65lQWGBF+hSUIBDADS
+tlWquV7SdREZtxXBVVzdCkV1xkeHYfo2Z244W0LTwmvpbO+o6P5GCAW2c336qWEl
+sMO9ujeV2nuUZy3k3AtJLx19iWC+ywYVzJ8f878XAxq0ya1VBBnfsBc7iRI3umf2
+JSi+fHXf9l+rJ8Zr5AkLrUo3tQoxX8xWQIfUVY481nlkOvuMtxEI6h1t+z7PWjAJ
+sdKKdevRPApPIBGXX0iGE/98ATsLYtvh9ln26j1SrSdtKpPktuYve3zkphlZAdf5
+ReViicik6gpEdyEfIxNab6nyV8LTbSeCHe+6/cz+AEqA+cr3K3MwriaapPzNhRV8
+izzGnIWChIZptGBKH5nLivfIAB/hbOgU6tM+YgUKrpJCXXA1My2q68o2kARJxh6s
+0tuuT6pFEAG9RmzS3ywrPz4PAgkwrJA1uUa9fy9ngkOnQN3CEeVQTUU55b+6zVhW
+1Qq8PII6AGqj1lSY9jLpjxEr3q227OlTaxfgg19x5o9rcyccAZlQqzL2p3Z7HZ0A
+EQEAAf4HAwKyxiOcJwMkLPhbpalG07ErjqLt73SKDP3Qv5zzkUnBcqE0TbyFtFlp
+HFf/Lv60X1m1OBgP4htz+JfikL5XVbWiGEBWvWPJP6VBBLJm+vjENjfKXzrRpcR6
+zhpfmJXm3BSXSpRg746AVW5Tjt2Z+dG7leTL+bddgu321OLYrpghyOUblKnRJZ0g
+0+vLByFbLWlgtFs3VxPQJw6FmdN9+m748xeVbqzxXwEzScpBZhcGrjHUgnYL4/XY
+PxzmZpUZ3qFW/P0uPZ8PdzI9MjEXaDhdxxOj/TP3cc2+XnrpBeWAGajMtulMvt+O
+PA/jisn0ZViy7q1fNz+B3j/V++l3UxRAHnI5yaRY91pPlOmnaG4ScCP2o7NAUIiA
+/Q3O13hIvVB+iIt9Y3p5WQSBbppHURVlhOOxDkSpXuxe5OhXqFYuDjGyx6hId4xL
+b/IP4Gs29ZSG4+6nOZa7GWl4M21Zcw0AX1Gs0+6PPuqlIecW+6e28xxwFQjj7IKt
+OvHq6zI810ReWdw9qVp3g9mzqI8x9KcFGdDvZmd4sA0R9GYR6UhvTKTIhdV4wrdO
+w2oBe3CpmEnrggtsTrUFykAfjuYRS6aYUjRVv6rdeiWFKyQzBqqboLO9si2RkuNK
+H8P2G6BdsLMax/kZKoXuuQ39xq/Li8NJjAoWEMz8iiZ2Io7MGPZobXssoA18Q3dn
+tNRPM06cojXoDkXxc5jkQMwJUpuAaa59Zcsgp0sFv7/8nez9ejCaEBTqm1pkEQtd
+b6178ld2T6q1jMb/tHWl8CjhH1sZVX2DdEk4SraIFdtGD5vUXo9SkI/QiY0RYwtY
+t3tzNnlWMPAWmC+GaZ9QjmPYwEGCXvaGZ4rB2iRPQ+wAvHV6b49txRckLSGm56jb
+8WMY7hSC0q3Bj4vFSx78Ytn/H2xwEh/XUiXe1rZhFRXxf7ocLoVS8xx+FL94kzaC
+pLKTKoX1udNmYtebkO9llpkW/Z4KeQWJ73uSsjTZhMXVr3fJRanH2twjY8XodG+J
+KXuERxMkM2sqnhecsAS8yCLndFLGSDSWyNIA7o6VAtCOpS4TKlYRNmv2XOaxrGgT
+7y4hZQBJT8CwP8QuZl9R4WBtNQLPOn9LJCYtKtOznpb2v27BEyeAk4DlKgHJAfcr
+kT2xBj/UoJsD72iJMh7SOhmWr7T+gbwAnDlhM1TmtMfWUCAG9Y3fz/metPlMHCKv
+ttrgTfyLyvQHgiDTh3iqNEZ7rGK46on72/YYS+DXA3uSNckCaNQXupoIrxqpDjfA
+WrrmoRqL4IWMxHzF1RVKAsjXWaYtpbvlMOSNtyMff29WM+MkZqG3IdbkokKJdJf4
+/+iQU+738VD43SdPurQcSGUgTWFuIDxoZW1hbkBncmV5c2t1bGwuY29tPokB1AQT
+AQgAPhYhBIYPcR68MZb6cOhv9wDz8yhlQWZrBQJfoUlCAhsDBQkDwmcABQsJCAcC
+BhUKCQgLAgQWAgMBAh4BAheAAAoJEADz8yhlQWZrD0YMAJp6WkrSzghIgrGmEquh
+UPu4n8dnaGraGxu1Om9Z6HrUvphBvm/yZMlZxYbsQRvd8DUCuQD7fScBS12WX3AY
+e001REfAbj0kDAdDQ0Z8sFCeCDSBJ9ulX07FzTHH0qROcSv6NONjGYVeTFicL2W0
+rATygnFzzjjSGboMq1qA8u6/5JNM7MAxJcIS0Dr8Fhdwv8TwTJrVg6ZzJDHN8OVA
+UkPaciQI5lDDP5+kOVqbZZ92Ua8byxKtNACCdSsWZr2OvYyjUz4JKMp5X6yHbDQB
+3vlwRkRS7Voo3pUGsdLwiBWiryklSa++DIbBemrALFLc5YnLgfCV0frPOEqsdDwW
+ECRxwN4r+2DjY6TYCEEDfhM2Hm7MoMx/jM4uhI4KwPdOKmHsBPVBeXqBRXz32NMM
+Zg6to0HRjDapR8AkbfdC5vjiuwnDA6llmxnVtx2oPX3g8RVOIw65f8KfWzWSfzEq
+hoKTccsHMMza8J1ax6T6HXkqa/Tt/B/3d7nUzp53V3luG50FhgRfoUlCAQwA4rFx
+mKwr4RAoVEqhDDWl8ecd/KQXEg0iCpkkmED6mEpPE9qAi8ORNId66E+rveS1Ssbm
+bqVlrN9iHphtvYqvlwwb2IkgPaFpmVSqWrQ3yzEPrL5CLAWiiEq7M4ux7pueYKcO
+mv3wQSta9eMgy9jaGUXrxFl4qotCevcEsLzkKC045OdVxkL++NFsiQUSfMYOtgGK
+XuBh0ycI/pOb66lY186zPT0tR+QA18uzeCizEjhCZmPIlPHjN8NOEM7ZLU4UQrLd
+Srm1quhO6DvGEoO5FulvGtp5hVHdJL5oB7svzNurXB3WVjdXCnRijoaCR07A/X9J
+VZY2+kRxdl6ZkuLZxb5UE6usW7pTA5DKiuFG/w6CSGZA1Dv3+yoZnjN8KhnGmIWm
+EJgvddWWoaJ3wFvSAGkYa3qBLX3noV3ZCm0c/r2LBcyFGyuyddEhg9wrqWU9vM7W
+/4BkTqSJdeMRlS9FD803V9GqxAJBJ1KOSFt2s6b+ekYCI/d+Buso8GPp8eUHABEB
+AAH+BwMCnL1QLv+DJ3P4dP2//f1cC7xTsDp9/ogeuz8gxIm6aWtNBhgWgRVgXnma
+HsmQeEm7c70Vvt+Kjo9DbKUQbo32pc1Gwd8wvnNZUKtj+9E71hDd05f/SiA2ZTck
+8AIRgRUV30Nj2qEgg0nFCWDNfMf0Lx7XH5APMJEZ2GXioiUdUInFlfXBvK6zv4wO
+0jIyB/lRO5sCLcC8jNsNfe5oQVcoizziMxaAK91Fv93DeVa2hwqTK3VqBPXa/uyz
+6iRMYe//nYIJCNllEsY8whKKfsskIOk1Dwofyuh2IYP6dv3SXhTj+l+qp1uqsg2r
+JiThiyNXs0+zeVRxURBSZJrxMLHAs4tdcyckt0fCvM6bUCcDRo9+6w52GSMtb1w3
+08oJ+4YOLilJIR041x+Jzs1oTMhAWI5XH02x0mEFKADg/iSexOFSKIfT5RvFYFEj
+Fpil+RalypUWzoxjaHFrSV9gxXpdys/qlHb4dr/nMTc/42x2d1xH6HTmlLtTp3rl
+vM8/6tmeIhdTkfPtWIyrSfmi61ZCTJ21tKgDNj8r79lxkB6vqX+c86X/ug1tv3Ma
+BN96f4QJOGHIhefImnStAw7OyQn3F9qnkj3x7u2f9f1XyDJO4T+WaQcYf67DEDy0
+KLpxwjEuT63BYIiWcQHli27lGOj8gAalTnDaWOWRckw7KAemL6cMGwZAHT91aHVH
+IKd+dwl3gbArYJxWQ9Fc7lF0Nv4BfEghCOssrQuli9jKkhok61pQrx6L/ekkfeRt
+mBjDtZOCO5NOTeeAZlc23TpZ/yjSBY/GPY0jXfnZ40Vm/Kl5VHW3Poc/rJDI67/5
+Zz+mL/sTOh2SRKUzsDGQqoQeq0ud1o8LQNf9m/3R+qxII3UsaRxKPDM4O2z7uLqG
+v6DG9WVO/6nvoEMrItyh01BfU8l4zLkvXpkcrgbRT4D62w5BgoYpAfHprdqwxCDr
+gRIiRKgNy2+kfxv6MVaTlmO8Fa9CR5wxeynx+YvtlIjVEF9SXQaXyb1g/zmnimn2
+kAjp/zdPQDLtZRW/cR6EEP6h8zW2jg3p+Owh9tVaZ1WoDfuelRMCuFFoiJ0RHXRQ
+ocSzXfw7cB0YCpWR8Rrr0QlQYh/GEbQahTjjk+x0FXmEiCGkvOQeBFY2KUG/597g
+maYHwRqfP2LjprG1mFgk0wUz6Juf86RZYD1XszIQPAL1CXf8kSuh49t7MRSgCiSo
+qfMfZsMZgftjld5pD0lEqbpohHw/qpZdEklqUpkNUxJbBCWr9lPKirKadeLiXLKP
+JI6Q0UEKkdw6lRLrg7UoDtr0vx/Izb3QB1jpKX7m1E/YZhTeVgYnLjrHCBjhJ8cE
+kFmM7YC0iuh1TduJAbwEGAEIACYWIQSGD3EevDGW+nDob/cA8/MoZUFmawUCX6FJ
+QgIbDAUJA8JnAAAKCRAA8/MoZUFma/gCC/9xkH8EF1Ka3TUa1kiBdcII4dyoX7gs
+/dA/os0+fLb/iZZcG+bJZcKLma7DRiyDGXYc7nG3uPvho7/cOCUUg5P/EG5z0CDX
+zLbmBrk2WlRnREmK/5NTcisCyezRMXHOxpya4pmExVMqSPGA0QbKGwdHqfbHQv2O
+yI3PYBKvlN+eu6e5SEbT76AQijj5RSPcgbko24/sSqJylD1lnRocQK1p4XelosBr
+aty4wzYSvQY9dRD4nafxPHI3YjKiAG0I7nJDQ0d1jDaW5FP0BkMvn51SmfGsuSg1
+s46h9JlGRZvS0enjBb1Ic9oBmHAWGQhlD1hvILlqIZOCdj8oWVjwmpZ7BK3/82wO
+dVkUxy09IdIot+AIH+F/LA3KKgfDmjldyhXjI/HDrpmXwSUkJOBHebNLz5t1Edau
+F+4DY5BHMsgtyyiYJBzRGT5pgrXMt4yCqZP+0jZwKt1Ech/Q6djIKjt+9wOGe9UB
+1VrzRbOS5ymseDJcjejtMxuCOuSTN9R5KuQ=
+=VqO+
+-----END PGP PRIVATE KEY BLOCK-----

apps/contract-migration/testdata/pgp/publickeys_meta.asc

@@ -0,0 +1,151 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGNBF+hSOgBDACpkPQEjADjnQtjmAsdPYpx5N+OMJBYj1DAoIYsDtV6vbcBJQt9
+4Om3xl7RBhv9m2oLgzPsiRwjCEFRWyNSu0BUp5CFjcXfm0S4K2egx4erFnTnSSC9
+S6tmVNrVNEXvScE6sKAnmJ7JNX1ExJuEiWPbUDRWJ1hoI9+AR+8EONeJRLo/j0Np
++S4IFDn0PsxdT+SB0GY0z2cEgjvjoPr4lW9IAb8Ft9TDYp+mOzejn1Fg7CuIrlBR
+SAv+sj7bVQw15dh1SpbwtS5xxubCa8ExEGI4ByXmeXdR0KZJ+EA5ksO0iSsQ/6ip
+SOdSg+i0niOClFNm1P/OhbUsYAxCUfiX654FMn2zoxVBEjJ3e7l0pH7ktodaxEct
+PofQLBA9LSDUIejqJsU0npw/DHDD2uvxG+/A6lgV9L8ETlvgp8RzeOCf2bHuiKYY
+z87txvkFwsXgU1+TZxbk+mtCBbngsVPLNarY/KGkVJL+yhcHRD0Pl4wXUd6auQuY
+6vQ9AuKiCT1We2sAEQEAAbQeTWVyIE1hbiA8bWVybWFuQGdyZXlza3VsbC5jb20+
+iQHUBBMBCAA+FiEE8/r2aOgu9RJNUYe67yb0aCND9pIFAl+hSOgCGwMFCQPCZwAF
+CwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ7yb0aCND9pLwiwwAhFJbAyUK05TJ
+KfDz81757N472STtB8sfr0auwmRr8Zs1utHRVM0b/jkjTuo4uJNr7YVVKTKgE7+r
+J+pwhm3wlTQ44LVLjByWAi/7NWg3E9b2elm+qkfgm/RfFt3vkuOxGSyZyIFFh+/t
+wv6iABPvr6w7MZwrFaS0UP3g1VGa5TFqg6KNxod9H/gPLxv45lutXf3VvBZTJpr1
+pxn7aLHlFzEyIgNZbP/N1QF44GSrN/k0DfL631sZjauUXaZXbi5xGsKKCYwJ1g3q
+587pi6mTdTV3n0hKgVuipO8hGy5++YeOv+hXsCxDwyZ+Shv+qavd/SapxYgCdEue
+uwONIFfsIsWCd3SCcjKXicTTEFMu8nvBmf7xuo2hv6vEOxoijlXV+4LkGrskdB8Z
+Mg8PywEx6DLmDokgnAhTLrTc1ShbkOtQ3yNjjyFK7BDpqobsJal6d8SpbhccUJLe
+paSmsk0CgJsTjhAl6EwX0EYgTo3kP5fScqrbD8VwQaT8CcE4rCV4uQGNBF+hSOgB
+DADHtpTT1k4x+6FN5OeURpKAaIsoPHghkJ2lb6yWmESCa+DaR6GXAKlbd0L9UMcX
+LqnaCn4SpZvbf8hP4fJRgWdRl5uVN/rmyVbZLUVjM8NcVdFRIrTsNyu4mLBmydc3
+iA/90sCTEOj9e7DSvxLmmLFjpwM5xXLd6z0l6+9G+woNmARXVS3V/RryFntyKC3A
+TCqVlJoQBG45Tj2gMIunpadTJXWmdioooeGW3sLeUv5MM98mSB4SjKRlJqGPNjx5
+lO6MmJbZeXZ/L/aO6EsXUQD2h82Wphll4rpGYWPiHTCYqZYiqNYr6E3xUpzcvWVp
+3uCYVJWP6Ds117p7BoyKVz00yxC9ledF3eppktZWqFVowCMihQE3676L3DDTZsnJ
+f1/8xKUh5U2Mj3lBvjlvCECKi00qo8b1mn/OklQjJ5T4WzTrH6X+/zpez8ZkmtcO
+ayHdUKD/64roZ9dXbXG/hp5A+UWj8oSVYKg2QNAwAnZ+aiZ2KVRE/Y61DCgFg6Cc
+x/cAEQEAAYkBvAQYAQgAJhYhBPP69mjoLvUSTVGHuu8m9GgjQ/aSBQJfoUjoAhsM
+BQkDwmcAAAoJEO8m9GgjQ/aSIPcL/3jqL2A2SmC+s0BO4vMPEfCpa2gZ/vo1azzj
+UieZu5WhIxb5ik0V6T75EW5F0OeZj9qXI06gW+IM8+C6ImUgaR3l47UjBiBPq+uK
+O9QuT/nOtbSs2dXoTNCLMQN7MlrdUBix+lnqZZGSDgh6n/uVyAYw8Sh4c3/3thHU
+iR7xzVKGxAKDT8LoVjhHshTzYuQq8MqlfvwVI4eESLaryQ+Y+j5+VLDzSLgPAnnI
+qF/ui2JQjefJxm/VLoYNaPAGdqoz/u/R0Tmz94bZUfLjgQaDoUpnxYywK2JGlf3m
+PZ3PNWjxJzuQTF5Ge5bz/TylnRYIyBT7KD7oaKHO62fhDbYPJ4f94iZN4B6nnTAe
+P34zFDlkUbX4AHudXU7bvxT5OUk9x9c2tj7xwxQHaEhq2+JsYW0EVw27RLhbymnB
+fLjVVUktNF0nQGvU2TEocw4pr2ZkDHQkSnlbNa4kujlL7VzbpnEgyOmi5er9GaIu
+VSVADovBu+pz/Ov1y/3jUe8hZ/KleZkBjQRfoUkaAQwA2r2HiLvpnclyZMoeck1L
+FoVyEU/CjPcYWF1B76ekO9mrlYvbKsnsyL0WcuEqwCmHdLk70i743Fn21WQK4uvv
+lvrEpev9aj9DihyLctv4qrPm6wAU/Xibf75tg1iRL+muMQfv6hQhjdhwkYFx/7XQ
+6UWkEibqFS7xJwrhz9lHL4KTA4sO5PeW713+mpz7tM5RmGV6NOQAyEEfAv6OawlW
+k0f5o8xngIoyo2BS5qIeEBO+iz45+GG8GQC6XufOIx7VVl++ZpsxZKtDq/AXfAsk
+xfLRwZMqH9Db5pPMzrL1bPV16AwoWqhAGd2HIMkODLEC5XTGIKCqO5+n288rHhAJ
+TqFmE7TpAo+Eb0Tkk4jfm6LyRonmQGpu/Zxa53n5D6d+AgYWAMeHkEthWJkES4mK
+pZu4nV21+n9mynnPg8wzthL705Q6IBjtlxX8EP6eeRFE1BUCNp2RZttTSdI+8iwz
+YsGOJdJeeXeLOGhvU9/PLkRj9jgZLgCLAo1QGo2oxetZABEBAAG0IkJlYXN0IE1h
+biA8YmVhc3RtYW5AZ3JleXNrdWxsLmNvbT6JAdQEEwEIAD4WIQT2ReBH7lvE4oJM
+lNtC3JHPqKugKwUCX6FJGgIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIX
+gAAKCRBC3JHPqKugK25hC/9VF1fekj0IKnrOJRUcK/Cv4RBowl60V91w27ApsoP2
+awEJiFhY7qRijtkA3NKrT3tke7aTnC3yAJ8SFOmvIAC94ijb7Iv97xkG+1IIz8pv
+ru9y+dzd2NnvCkts8gFF0CI/xtEME90rU3Pay9B5IyrpP++UdmSmnp3Neuwi94BZ
+DfMlqkeiYOzWWSeYbmSSVfKTXeBdUuTyfRI4m/bPbh6gegOB/XdgSIrNY74D0nR3
+np0I+s0IGZepK24kgBKfUPwRDk7f98PXCh29iL3xH+TBxu30WHq7xKmPoXxCRyFL
+tnKF0MN5Ib276fHnJZM+hXf5i/1EPi4NLnk86e7fNI69hwiUd1msEt3VmZWe7anJ
+e/1p3sSXwbQGhhGWM5K41/rQ1CZ9qD95d6wkHRSc0n4z78qxgYV73yJHinN8xIFn
+PWbopPPIJbELSoM3IEpHobsj95pH4hzZAPSmDfOfLzV1G2ec1QPfWnTqUriUt7ed
+Ds4//7Cczj6sRh2B6ax2diC5AY0EX6FJGgEMAMqxn5io6fWKnMz8h5THqp4gEzDu
+oImapfMKbAKcxEtJmcLkvn+4ufEP/hcll66InqJHsqMOrdb+zbduCruYWpizhqCI
+GSsuRu7+ZEEkQFmF5juCOV/5qKQJgZZmxSKbRtboapMRR/jmg1pvhnUG7wJOGWi7
+qv+iRdsWKskDO7tUQE34+ID7IwfDZe2fbFKxf66nPlUunF8aMglsvGmtCEzm/xwj
+unHnmoqZBQIzTdEXIaEwhVosbgY7A1iwOJ/gT2dcF2KJa7tygrtcbgdVzYCibynw
+tlvDGXukweuYLQFsObyBG3UHRhJg61p7n344sy1U9uwCP3/pVCr9bNY9mLZpCgHF
+kqxErmB8cWouQkbwnqxQFm21KtGFzjUawuKBXVtDEeA8C5Ha0sx7lw5JrX8GD3EL
+60qKWjqujJsR1kyijXx1No7Xr9NWWuPoIDYH06ZoYE+j065VTRqZIGr3NjUZnqT7
+s9M41roQMnKAzRBXousRXRW9dXfS5YIG4nWTlwARAQABiQG8BBgBCAAmFiEE9kXg
+R+5bxOKCTJTbQtyRz6iroCsFAl+hSRoCGwwFCQPCZwAACgkQQtyRz6iroCt8igwA
+gopqy+UgxJ7oTL2zvOgL1ez7bv+E/U1/7Rdy5MHwr4WF6oZRpIBlgv3GXXeIFH9b
+FdDhgyPKgh+Tz24JBL+7YjUtWGe/G/pmmNK1YazB/OxrwiGFpTCyk1zhxEkhMu7H
+u3LgD571K+4TUUpaPCqEeoBBg6O3T29DH1AxpWpEPGXlOrRDHYgVziEpLdUNahAj
+F53auNWvya+Vc2qZwM4NFt608LLf7J5yIA2vbsvf6+gVopPE3whXESKXo08B2hC1
+f3Pr9/Tgt6oIvy9/dAcTMalxRyyc42E2wX5kyzDlfhY9kqaNNfaGMZJO5g//gB7B
+dtrAfo/LhWtary/YfAOtbbnMYkf+HODAPZItaIjMZngBM0c0m78YoCetAQE8uBFK
+6aXmht3BZGPOwgyZpK5QT6ClYst2N9ca3tPUEfnddotKySmCEk/JWtu5/0lFl75W
+zHulc7iUNGJmnUffVZyH12CjBWsTtqombHDkdEKFocavqpVcCCbKbtW5GZhuZC65
+mQGNBF+hSUIBDADStlWquV7SdREZtxXBVVzdCkV1xkeHYfo2Z244W0LTwmvpbO+o
+6P5GCAW2c336qWElsMO9ujeV2nuUZy3k3AtJLx19iWC+ywYVzJ8f878XAxq0ya1V
+BBnfsBc7iRI3umf2JSi+fHXf9l+rJ8Zr5AkLrUo3tQoxX8xWQIfUVY481nlkOvuM
+txEI6h1t+z7PWjAJsdKKdevRPApPIBGXX0iGE/98ATsLYtvh9ln26j1SrSdtKpPk
+tuYve3zkphlZAdf5ReViicik6gpEdyEfIxNab6nyV8LTbSeCHe+6/cz+AEqA+cr3
+K3MwriaapPzNhRV8izzGnIWChIZptGBKH5nLivfIAB/hbOgU6tM+YgUKrpJCXXA1
+My2q68o2kARJxh6s0tuuT6pFEAG9RmzS3ywrPz4PAgkwrJA1uUa9fy9ngkOnQN3C
+EeVQTUU55b+6zVhW1Qq8PII6AGqj1lSY9jLpjxEr3q227OlTaxfgg19x5o9rcycc
+AZlQqzL2p3Z7HZ0AEQEAAbQcSGUgTWFuIDxoZW1hbkBncmV5c2t1bGwuY29tPokB
+1AQTAQgAPhYhBIYPcR68MZb6cOhv9wDz8yhlQWZrBQJfoUlCAhsDBQkDwmcABQsJ
+CAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEADz8yhlQWZrD0YMAJp6WkrSzghIgrGm
+EquhUPu4n8dnaGraGxu1Om9Z6HrUvphBvm/yZMlZxYbsQRvd8DUCuQD7fScBS12W
+X3AYe001REfAbj0kDAdDQ0Z8sFCeCDSBJ9ulX07FzTHH0qROcSv6NONjGYVeTFic
+L2W0rATygnFzzjjSGboMq1qA8u6/5JNM7MAxJcIS0Dr8Fhdwv8TwTJrVg6ZzJDHN
+8OVAUkPaciQI5lDDP5+kOVqbZZ92Ua8byxKtNACCdSsWZr2OvYyjUz4JKMp5X6yH
+bDQB3vlwRkRS7Voo3pUGsdLwiBWiryklSa++DIbBemrALFLc5YnLgfCV0frPOEqs
+dDwWECRxwN4r+2DjY6TYCEEDfhM2Hm7MoMx/jM4uhI4KwPdOKmHsBPVBeXqBRXz3
+2NMMZg6to0HRjDapR8AkbfdC5vjiuwnDA6llmxnVtx2oPX3g8RVOIw65f8KfWzWS
+fzEqhoKTccsHMMza8J1ax6T6HXkqa/Tt/B/3d7nUzp53V3luG7kBjQRfoUlCAQwA
+4rFxmKwr4RAoVEqhDDWl8ecd/KQXEg0iCpkkmED6mEpPE9qAi8ORNId66E+rveS1
+SsbmbqVlrN9iHphtvYqvlwwb2IkgPaFpmVSqWrQ3yzEPrL5CLAWiiEq7M4ux7pue
+YKcOmv3wQSta9eMgy9jaGUXrxFl4qotCevcEsLzkKC045OdVxkL++NFsiQUSfMYO
+tgGKXuBh0ycI/pOb66lY186zPT0tR+QA18uzeCizEjhCZmPIlPHjN8NOEM7ZLU4U
+QrLdSrm1quhO6DvGEoO5FulvGtp5hVHdJL5oB7svzNurXB3WVjdXCnRijoaCR07A
+/X9JVZY2+kRxdl6ZkuLZxb5UE6usW7pTA5DKiuFG/w6CSGZA1Dv3+yoZnjN8KhnG
+mIWmEJgvddWWoaJ3wFvSAGkYa3qBLX3noV3ZCm0c/r2LBcyFGyuyddEhg9wrqWU9
+vM7W/4BkTqSJdeMRlS9FD803V9GqxAJBJ1KOSFt2s6b+ekYCI/d+Buso8GPp8eUH
+ABEBAAGJAbwEGAEIACYWIQSGD3EevDGW+nDob/cA8/MoZUFmawUCX6FJQgIbDAUJ
+A8JnAAAKCRAA8/MoZUFma/gCC/9xkH8EF1Ka3TUa1kiBdcII4dyoX7gs/dA/os0+
+fLb/iZZcG+bJZcKLma7DRiyDGXYc7nG3uPvho7/cOCUUg5P/EG5z0CDXzLbmBrk2
+WlRnREmK/5NTcisCyezRMXHOxpya4pmExVMqSPGA0QbKGwdHqfbHQv2OyI3PYBKv
+lN+eu6e5SEbT76AQijj5RSPcgbko24/sSqJylD1lnRocQK1p4XelosBraty4wzYS
+vQY9dRD4nafxPHI3YjKiAG0I7nJDQ0d1jDaW5FP0BkMvn51SmfGsuSg1s46h9JlG
+RZvS0enjBb1Ic9oBmHAWGQhlD1hvILlqIZOCdj8oWVjwmpZ7BK3/82wOdVkUxy09
+IdIot+AIH+F/LA3KKgfDmjldyhXjI/HDrpmXwSUkJOBHebNLz5t1EdauF+4DY5BH
+MsgtyyiYJBzRGT5pgrXMt4yCqZP+0jZwKt1Ech/Q6djIKjt+9wOGe9UB1VrzRbOS
+5ymseDJcjejtMxuCOuSTN9R5KuSZAY0EX6UhqgEMAO/22am2Urhbg5ClpEYzz2/W
+L8ez3tkXKQZa7PsvKUv69jwBwNQmEpMhIPFXhKKwcmmLgYcvnd64xrXM5STxWedy
+NaTPlCUDZGW+N4laCbrnHN98Ztu9TvjfjjiQvhHjD/9Ilc5fw5nZsawwTtGOwCkK
+opBVKsgHaGrKRl7QP2RTwITwo7CkDBf77kp8wGCECrrSel0cVezSf6UmDs7V3q12
+zf7gXBSjWlbA3NnSok6kTNej14IMKfdhiuUG6WFibxEfsOrm8Rv9RbbgpYUTN/ll
+yWDTqbVDjYefq/iLs/5w24oI/1K0gy8Rzdl5qu4cvqcwkmxQMvtWWHoT86iHFff/
+pp9drPctFfUetHDIKY+1U9VLilaSeCcDx7PCgxazCb7gmtTQWdM4wHcH5+69EEeG
+lP4N9efoTRlU+m5ZSuqOtEvLXtP2Gnd/Tgn3lBjHjA+hQ65G96fv40dbYiiHxluo
+cFkcwz118alx4cXStfAi6nCsDid2Y9NgWfHrJTs33QARAQABtDlMb3VpcyBIb2xi
+cm9vayA8YWNjb3VudHMtZ3Jhc3Nyb290c2Vjb25vbWljc0Bob2xicm9vay5ubz6J
+AdMEEwEIAD4WIQTFMBghgDfv6cesuTGwIKs7vZC0mAUCX6UhqgIbAwUJA8JnAAUL
+CQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRCwIKs7vZC0mEnCC/ih5hk43xHu6KBA
+on/ox6wcYLltSuaJawJbPmznrOK6aIJGUDx6E/VEjFeU/+bySrm8y3gk3jqeoPRk
+holZmvRW5/mJ/uud+7B83TOfLAHM1EgZtqCqq+Z61yrt46cjqXuQbonP5dFmnOee
+Zpg4TP7FRCjAYThy/NtOXY7Ob3OC+MNnDPo71R2se+x4Ac6NKsmNKAETZjZg01R/
+2w5Ns77pxub+tihAVasLzmqlGNjqRzLCemR4osMwd0XrtziKiIvPYFlHhysgkpYh
+oqbs2bjEdbsac21460j+3wRcvspfDEiLmI0n91s5uK3zCke0tI8BbU5/7IfnflBw
+9SVvcu5s6DqFjy3tuRVkVKs0h92YCEH6gfui1RkXPdFheGyZwvZujlfCTU0c2V8U
+pmy2TvdUSsWiHSNgY8nimWbxU1fXt7fnWnQk59/Nov4zRO4AJQhXgrb/IyhjKMVa
+UYUV/yQgVOqbv5VJnQFTuZrTm5yF47em99wmlZ06cJk0Q6I3QLkBjQRfpSGqAQwA
+0WyxXsatq9/kfN8Pd/tRjjUQlo0r9GuAKds8mKyWqk+hsOGYaTczL4qjne4Euwt1
+lWg2cC3jsX/9Ai4IX79Kkt4hOk0RbW76+YJJiL6CwsyfyPJASEq2ZqoVBgUJuBbw
+uEpMe0OL9ciEJ5oRLwZNgLXZZoHQaYlthHycvC3vEPeTtpGwYj+DfGQmt3af77i9
+0xQa1uor3QcvmmkDUb7/6Xv0Qwn8/sQ+GKyPhFia/OOZ50gnGVv2qKCFK1oaWNGz
+I5ywhD2Ij2j9ah9M08CEgWVFFibf7PRIq78yRoV9+ZCjWlIq0m2LjCykV9aEnWuw
+rWJaUtLn3i2roMieOIILhUUgDemAIV+vlbIlxZDp+XGsbhZ+MJpMUwpfEKK3Q7sD
+8tPbH6/2QpPnAezVUnwJJAg8pMLo+QzhTAd3PyPdIkc3yVQQuCycU9El7ysKhmiS
+AgqZjqrtPnU4y7SY1II1y/XDFDIuw2MggdolNahzx5VTKrNm5LYUT0m5XQmCehtJ
+ABEBAAGJAbwEGAEIACYWIQTFMBghgDfv6cesuTGwIKs7vZC0mAUCX6UhqgIbDAUJ
+A8JnAAAKCRCwIKs7vZC0mI4eC/0cMG9+fZfyq8V7wB47L/qmfS0O+bSE4AlZjtoa
+30UqbW8Yp2oa1uZXaF8loC3RW9d7VdnSh6K5vSnHh/0+OkwqAbpYDaF6Kuk/zVHX
+r4vQ1FaE1uzZisaqEORW/LG+oWiOFDhF7lXGVKj7iXwfFVVudDxHLHj34dC9rrsm
+5cTNdHalP0OW00H17nM/R4CR62mkhIM7zUuA1Z8MxSa8I3A9SL35G9iaWRYXE892
+KcPYSAgLna7rW+gHD1QI0sqsR6qdaojO5BDVrEYnP58D5aCOTeZ50ACO43JaZlYm
+o3jdqBvvYKpYuJ2is1T3unnrY6ztblz78OE+37d9gyAp1j0dhIzOfdpSHCyUYUQL
+4YNGLs/3yfelr1XXLwYXzKlioNDu7k4rggwN3td1122p3U1vfVY4qb2eTyjDPizb
+NdtbiWRXKFibzG0OoiAaq0ZC8nZsP6xy7vmI05hN7PocAWllJVdpaXVh75OViGaj
+KuNFGWsKI1qTd1aEMRQzT4s+JJM=
+=8257
+-----END PGP PUBLIC KEY BLOCK-----

docker-compose.yml

@@ -84,7 +84,7 @@ services:
       # And these two are for wait-for-it (could parse this)
       ETH_PROVIDER_HOST: eth 
       ETH_PROVIDER_PORT: 8545
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
     command: ["./reset.sh"]
     depends_on:
@@ -102,7 +102,7 @@ services:
       # And these two are for wait-for-it (could parse this)
       ETH_PROVIDER_HOST: eth 
       ETH_PROVIDER_PORT: 8545
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
@@ -146,7 +146,7 @@ services:
       DATABASE_DEBUG: 1
       ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
       CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CELERY_BROKER_URL: redis://redis:6379
       CELERY_RESULT_URL: redis://redis:6379
     deploy: 
@@ -214,7 +214,7 @@ services:
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
       DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       PGPASSWORD: ${DATABASE_PASSWORD:-tralala}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor}
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
@@ -253,9 +253,8 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-        #DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       DATABASE_DEBUG: 1
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
@@ -275,7 +274,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_manager.sh head -vv
+        ./start_manager.sh head -v
     # command: "/root/start_manager.sh head -vv"
 
   cic-eth-manager-history:
@@ -292,7 +291,7 @@ services:
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
       DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
@@ -312,7 +311,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_manager.sh history -vv
+        ./start_manager.sh history -v
     # command: "/root/start_manager.sh history -vv"
 
   cic-eth-dispatcher:
@@ -329,7 +328,7 @@ services:
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
       DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
@@ -351,7 +350,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_dispatcher.sh -q cic-eth -vv
+        ./start_dispatcher.sh -q cic-eth -v
     # command: "/root/start_dispatcher.sh -q cic-eth -vv"
 
 
@@ -368,7 +367,8 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
+      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
+      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
@@ -389,7 +389,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        ./start_retry.sh -vv
+        ./start_retry.sh -v
     # command: "/root/start_retry.sh -q cic-eth -vv"