Add missing script name

Sovereign import part II

See merge request grassrootseconomics/cic-internal-integration!81

apps/cic-eth/cic_eth/api/api_admin.py

@@ -31,7 +31,10 @@ from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import Nonce
 from cic_eth.db.enum import (
         StatusEnum,
+        StatusBits,
         is_alive,
+        is_error_status,
+        status_str,
     )
 from cic_eth.error import InitializationError
 from cic_eth.db.error import TxStateChangeError
@@ -42,6 +45,8 @@ app = celery.current_app
 #logg = logging.getLogger(__file__)
 logg = logging.getLogger()
 
+local_fail = StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.UNKNOWN_ERROR
+
 
 class AdminApi:
     """Provides an interface to view and manipulate existing transaction tasks and system runtime settings.
@@ -195,6 +200,7 @@ class AdminApi:
         blocking_tx = None
         blocking_nonce = None
         nonce_otx = 0
+        last_nonce = -1
         for k in txs.keys():
             s_get_tx = celery.signature(
                     'cic_eth.queue.tx.get_tx',
@@ -205,18 +211,25 @@ class AdminApi:
                     )
             tx = s_get_tx.apply_async().get()
             #tx = get_tx(k)
-            logg.debug('checking nonce {}'.format(tx['nonce']))
-            if tx['status'] in [StatusEnum.REJECTED, StatusEnum.FUBAR]:
-                blocking_tx = k
-                blocking_nonce = tx['nonce']
+            logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
             nonce_otx = tx['nonce']
+            if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
+                logg.info('permanently errored {} nonce {} status {}'.format(k, nonce_otx, status_str(tx['status'])))
+                blocking_tx = k
+                blocking_nonce = nonce_otx
+            elif nonce_otx - last_nonce > 1:
+                logg.error('nonce gap; {} followed {}'.format(nonce_otx, last_nonce))
+                blocking_tx = k
+                blocking_nonce = nonce_otx
+                break
+            last_nonce = nonce_otx
 
-        nonce_cache = Nonce.get(address)
+        #nonce_cache = Nonce.get(address)
         #nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
         
         return {
             'nonce': {
-                'network': nonce_cache,
+                #'network': nonce_cache,
                 'queue': nonce_otx,
                 #'cache': nonce_cache,
                 'blocking': blocking_nonce,
@@ -270,16 +283,15 @@ class AdminApi:
 #        self.w3.eth.sign(addr, text='666f6f')
 
 
-    def account(self, chain_spec, address, cols=['tx_hash', 'sender', 'recipient', 'nonce', 'block', 'tx_index', 'status', 'network_status', 'date_created'], include_sender=True, include_recipient=True):
+    def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
         """Lists locally originated transactions for the given Ethereum address.
 
         Performs a synchronous call to the Celery task responsible for performing the query.
 
         :param address: Ethereum address to return transactions for
         :type address: str, 0x-hex
-        :param cols: Data columns to include
-        :type cols: list of str
         """
+        last_nonce = -1
         s = celery.signature(
                 'cic_eth.queue.tx.get_account_tx',
                 [
@@ -291,33 +303,45 @@ class AdminApi:
 
         tx_dict_list = []
         for tx_hash in txs.keys():
+            errors = []
             s = celery.signature(
                     'cic_eth.queue.tx.get_tx_cache',
                     [tx_hash],
                     queue=self.queue,
                     )
             tx_dict = s.apply_async().get()
-            if tx_dict['sender'] == address and not include_sender:
-                logg.debug('skipping sender tx {}'.format(tx_dict['tx_hash']))
-                continue
+            if tx_dict['sender'] == address:
+                if tx_dict['nonce'] - last_nonce > 1:
+                    logg.error('nonce gap; {} followed {} for tx {}'.format(tx_dict['nonce'], last_nonce, tx_dict['hash']))
+                    errors.append('nonce')
+                elif tx_dict['nonce'] == last_nonce:
+                    logg.warning('nonce {} duplicate in tx {}'.format(tx_dict['nonce'], tx_dict['hash']))
+                last_nonce = tx_dict['nonce']
+                if not include_sender:
+                    logg.debug('skipping sender tx {}'.format(tx_dict['tx_hash']))
+                    continue
             elif tx_dict['recipient'] == address and not include_recipient:
                 logg.debug('skipping recipient tx {}'.format(tx_dict['tx_hash']))
                 continue
 
-            logg.debug(tx_dict)
             o = {
                 'nonce': tx_dict['nonce'], 
                 'tx_hash': tx_dict['tx_hash'],
                 'status': tx_dict['status'],
                 'date_updated': tx_dict['date_updated'],
+                'errors': errors,
                     }
-            tx_dict_list.append(o)
+            if renderer != None:
+                r = renderer(o)
+                w.write(r + '\n')
+            else:
+                tx_dict_list.append(o)
 
         return tx_dict_list
 
 
     # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None):
+    def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
         """Output local and network details about a given transaction with local origin.
 
         If the transaction hash is given, the raw trasnaction data will be retrieved from the local transaction queue backend. Otherwise the raw transaction data must be provided directly. Only one of transaction hash and transaction data can be passed.
@@ -511,4 +535,9 @@ class AdminApi:
             for p in problems:
                 sys.stderr.write('!!!{}\n'.format(p))
 
-        return tx
+        if renderer == None:
+            return tx
+
+        r = renderer(tx)
+        w.write(r + '\n')
+        return None

apps/cic-eth/cic_eth/eth/tx.py

@@ -281,6 +281,7 @@ def send(self, txs, chain_spec_dict):
     o = raw(tx_hex)
     conn = RPCConnection.connect(chain_spec, 'default')
     conn.do(o)
+
     s_set_sent.apply_async()
 
     tx_tail = txs[1:]

apps/cic-eth/cic_eth/registry.py

@@ -0,0 +1,33 @@
+# standard imports
+import logging
+
+# external imports
+from cic_eth_registry import CICRegistry
+from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
+from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
+
+logg = logging.getLogger()
+
+
+def connect_token_registry(rpc, chain_spec):
+    registry = CICRegistry(chain_spec, rpc)
+    token_registry_address = registry.by_name('TokenRegistry')
+    logg.debug('using token registry address {}'.format(token_registry_address))
+    lookup = TokenIndexLookup(token_registry_address)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect_declarator(rpc, chain_spec, trusted_addresses):
+    registry = CICRegistry(chain_spec, rpc)
+    declarator_address = registry.by_name('AddressDeclarator')
+    logg.debug('using declarator address {}'.format(declarator_address))
+    lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses)
+    CICRegistry.add_lookup(lookup)
+
+
+def connect(rpc, chain_spec, registry_address):
+    CICRegistry.address = registry_address
+    registry = CICRegistry(chain_spec, rpc)
+    registry_address = registry.by_name('ContractRegistry')
+    
+    return registry

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -16,8 +16,6 @@ from chainlib.eth.connection import EthUnixSignerConnection
 from chainlib.chain import ChainSpec
 
 # local imports
-from cic_eth_registry import CICRegistry
-
 from cic_eth.eth import erc20
 from cic_eth.eth import tx
 from cic_eth.eth import account
@@ -33,6 +31,11 @@ from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.otx import Otx
 from cic_eth.db import dsn_from_config
 from cic_eth.ext import tx
+from cic_eth.registry import (
+        connect as connect_registry,
+        connect_declarator,
+        connect_token_registry,
+        )
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -121,8 +124,6 @@ RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 's
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
-CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
-
 
 def main():
     argv = ['worker']
@@ -145,8 +146,8 @@ def main():
 #        Callback.ssl_ca_file = config.get('SSL_CA_FILE')
 
     rpc = RPCConnection.connect(chain_spec, 'default')
-    registry = CICRegistry(chain_spec, rpc)
-    registry_address = registry.by_name('ContractRegistry')
+
+    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
@@ -155,6 +156,8 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
+    connect_declarator(rpc, chain_spec, trusted_addresses)
+    connect_token_registry(rpc, chain_spec)
     
     current_app.worker_main(argv)
 

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -0,0 +1,100 @@
+#!/usr/bin/python
+import sys
+import os
+import logging
+import uuid
+import json
+import argparse
+
+# external imports
+import celery
+import confini
+import redis
+from xdg.BaseDirectory import xdg_config_home
+from chainlib.eth.address import to_checksum_address
+
+# local imports
+from cic_eth.api import Api
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger('create_account_script')
+logging.getLogger('confini').setLevel(logging.WARNING)
+logging.getLogger('gnupg').setLevel(logging.WARNING)
+
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Symbol of token to transfer')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('sender', type=str, help='Transaction sender')
+argparser.add_argument('recipient', type=str, help='Transaction recipient')
+argparser.add_argument('value', type=int, help='Transaction value with decimals')
+args = argparser.parse_args()
+
+if args.vv:
+    logg.setLevel(logging.DEBUG)
+if args.v:
+    logg.setLevel(logging.INFO)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        }
+config.dict_override(args_override, 'cli')
+config.add(to_checksum_address(args.sender), '_SENDER', True)
+config.add(to_checksum_address(args.recipient), '_RECIPIENT', True)
+config.add(args.value, '_VALUE', True)
+config.add(args.token_symbol, '_SYMBOL', True)
+if config.get('_SYMBOL') == None:
+    raise ValueError('gas transfers not yet supported; token symbol required')
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+
+def main():
+    redis_host = config.get('REDIS_HOST')
+    redis_port = config.get('REDIS_PORT')
+    redis_db = config.get('REDIS_DB')
+    redis_channel = str(uuid.uuid4())
+    r = redis.Redis(redis_host, redis_port, redis_db)
+
+    ps = r.pubsub()
+    ps.subscribe(redis_channel)
+    ps.get_message()
+
+    api = Api(
+            config.get('CIC_CHAIN_SPEC'),
+            queue=args.q,
+            callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+            callback_task='cic_eth.callbacks.redis.redis',
+            callback_queue=args.q,
+            )
+
+    #register = not args.no_register
+    #logg.debug('register {}'.format(register))
+    #t = api.create_account(register=register)
+    t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL'))
+
+    ps.get_message()
+    o = ps.get_message(timeout=args.timeout)
+    m = json.loads(o['data'])
+    print(m['result'])
+
+
+if __name__ == '__main__':
+    main()

apps/cic-eth/cic_eth/runnable/view.py

@@ -14,8 +14,6 @@ import datetime
 # external imports
 import confini
 import celery
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
 from hexathon import add_0x
@@ -27,6 +25,7 @@ from cic_eth.db.enum import (
     status_str,
     LockEnum,
 )
+from cic_eth.registry import connect as connect_registry
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -116,12 +115,16 @@ def render_tx(o, **kwargs):
     return content
 
 def render_account(o, **kwargs):
-    return '{} {} {} {}'.format(
+    s = '{} {} {} {}'.format(
             o['date_updated'],
             o['nonce'],
             o['tx_hash'],
             o['status'],
             )
+    if len(o['errors']) > 0:
+        s += ' !{}'.format(','.join(o['errors']))
+
+    return s
 
 
 def render_lock(o, **kwargs):
@@ -143,44 +146,31 @@ def render_lock(o, **kwargs):
     return s
 
 
-def connect_registry(registry_address, chain_spec, rpc):
-    CICRegistry.address = registry_address
-    registry = CICRegistry(chain_spec, rpc)
-    declarator_address = registry.by_name('AddressDeclarator')
-    lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses)
-    registry.add_lookup(lookup)
-    return registry
-
-
 # TODO: move each command to submodule
 def main():
     txs  = []
     renderer = render_tx
     if len(config.get('_QUERY')) > 66:
-        registry = connect_registry(registry_address, chain_spec, rpc)
-        txs = [admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry)]
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
     elif len(config.get('_QUERY')) > 42:
-        registry = connect_registry(registry_address, chain_spec, rpc)
-        txs = [admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry)]
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
+
     elif len(config.get('_QUERY')) == 42:
-        registry = connect_registry(registry_address, chain_spec, rpc)
-        txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False)
+        registry = connect_registry(rpc, chain_spec, registry_address)
+        txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
         renderer = render_account
     elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
         t = admin_api.get_lock()
         txs = t.get()
         renderer = render_lock
+        for tx in txs:
+            r = renderer(txs)
+            sys.stdout.write(r + '\n')
     else:
         raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
+                   
 
-    if len(txs) == 0:
-        logg.info('no matches found')
-    else:
-        if fmt == 'json':
-            sys.stdout.write(json.dumps(txs))
-        else:
-            m = map(renderer, txs)
-            print(*m, sep="\n")
-                    
 if __name__ == '__main__':
     main()

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         1,
-        'beta.1',
+        'beta.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/requirements.txt

@@ -1,14 +1,14 @@
-cic-base~=0.1.2a46
+cic-base~=0.1.2a51
 celery==4.4.7
 crypto-dev-signer~=0.4.14a16
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.4a7
+cic-eth-registry~=0.5.4a8
 #cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a3
+eth_accounts_index~=0.0.11a6
 erc20-transfer-authorization~=0.3.1a2
 #simple-rlp==0.1.2
 uWSGI==2.0.19.1
@@ -16,9 +16,9 @@ semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.1a5
-chainlib~=0.0.1a42
+chainlib~=0.0.1a43
 hexathon~=0.0.1a7
 chainsyncer~=0.0.1a20
 pysha3==1.0.2
 coincurve==15.0.0
-sarafu-faucet==0.0.2a13
+sarafu-faucet~=0.0.2a15

apps/cic-eth/setup.cfg

@@ -54,3 +54,4 @@ console_scripts =
 	# TODO: Merge this with ctl when subcmds sorted to submodules
 	cic-eth-tag = cic_eth.runnable.tag:main
 	cic-eth-resend = cic_eth.runnable.resend:main
+	cic-eth-transfer = cic_eth.runnable.transfer:main

apps/cic-notify/requirements.txt

@@ -1,5 +1 @@
-alembic~=1.4.2
-celery~=4.4.7
-confini~=0.3.6rc3
-redis~=3.5.3
-semver==2.13.0
+cic_base[full_graph]~=0.1.2a46

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 0, 'alpha.5')
+version = (0, 3, 0, 'alpha.7')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/requirements.txt

@@ -1,49 +1,4 @@
-#cic_base[full_graph]~=0.1.1a24
-africastalking==1.2.3
-alembic==1.4.2
-amqp==2.6.1
-attrs==20.2.0
-bcrypt==3.2.0
-betterpath==0.2.2
-billiard==3.6.3.0
-celery==4.4.7
-cffi==1.14.3
+cic_base[full_graph]~=0.1.2a46
 cic-eth~=0.10.1b1
 cic-notify~=0.4.0a3
-cic-types~=0.1.0a8
-click==7.1.2
-confini~=0.3.6rc3
-cryptography==3.2.1
-faker==4.17.1
-iniconfig==1.1.1
-kombu==4.6.11
-Mako==1.1.3
-MarkupSafe==1.1.1
-mirakuru==2.3.0
-more-itertools==8.5.0
-packaging==20.4
-phonenumbers==8.12.12
-pluggy==0.13.1
-port-for==0.4
-psutil==5.7.3
-psycopg2==2.8.6
-py==1.9.0
-pycparser==2.20
-pyparsing==2.4.7
-python-dateutil==2.8.1
-python-editor==1.0.4
-python-gnupg==0.4.6
-python-i18n==0.3.9
-pytz==2020.1
-PyYAML==5.3.1
-redis==3.5.3
-semver==2.13.0
-six==1.15.0
-SQLAlchemy==1.3.20
-tinydb==4.2.0
-toml==0.10.1
-transitions==0.8.4
-uWSGI==2.0.19.1
-vcversioner==2.16.0.0
-vine==1.3.0
-zope.interface==5.1.2
+cic-types~=0.1.0a8

apps/contract-migration/reset.sh

@@ -40,7 +40,10 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 
 	#BANCOR_REGISTRY_ADDRESS=`cic-bancor-deploy --bancor-dir /usr/local/share/cic/bancor -z $DEV_ETH_RESERVE_ADDRESS -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
 
+	>&2 echo "deploy account index contract"
 	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $keystore_file -vv -w`
+	>&2 echo "add deployer address as account index writer"
+	eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy -i $CIC_CHAIN_SPEC -y $keystore_file --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
 	eth-contract-registry-set -w -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS

apps/contract-migration/scripts/README.md

@@ -21,9 +21,13 @@ Make sure the following is running in the cluster:
 	* eth
 	* postgres
 	* redis
+	* cic-meta-server
+
+
+If using the _custodial_ alternative for user imports, also run:
 	* cic-eth-tasker
 	* cic-eth-dispatcher
-	* cic-eth-manager-head
+	* cic-eth-tracker
 
 
 You will want to run these in sequence:
@@ -40,19 +44,32 @@ Monitors a folder for output from the `import_users.py` script, adding the metad
 
 (Only if you used the `--gift-threshold` option above)
 
-`python -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> --head -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
+`python import_balance.py -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> --head -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` 
 
 This will monitor new mined blocks and send balances to the newly created accounts.
 
 
 ### 3. Users
 
+Only use **one** of the following
+
+#### Custodial
+
+This alternative generates accounts using the `cic-eth` custodial engine
+
 Without any modifications to the cluster and config files:
 
 `python import_users.py -c config --redis-host-callback redis <datadir>`
 
 ** A note on the The callback**:  The script uses a redis callback to retrieve the newly generated custodial address. This is the redis server _from the perspective of the cic-eth component_.
 
+#### Sovereign
+
+This alternative generates keystore files, while  registering corresponding addresses in the accounts registry directly
+
+`python import_sovereign_users.py -c config -i <newchain:id> -r <cic_registry_address> -p <eth_provider> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
+
+A `keystore` sub-directory in the data path is created, with ethereum keystore files for all generated private keys. Passphrase is set to empty string for all of them.
 
 ## VERIFY
 

apps/contract-migration/scripts/import_sovereign_users.py

@@ -0,0 +1,179 @@
+# standard imports
+import os
+import sys
+import json
+import logging
+import argparse
+import uuid
+import datetime
+import time
+from glob import glob
+
+# external imports
+import confini
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from cic_types.models.person import Person
+from chainlib.eth.address import to_checksum_address
+from chainlib.chain import ChainSpec
+from chainlib.eth.connection import EthHTTPConnection
+from chainlib.eth.gas import RPCGasOracle
+from chainlib.eth.nonce import RPCNonceOracle
+from cic_types.processor import generate_metadata_pointer
+from eth_accounts_index import AccountRegistry
+from cic_eth_registry import CICRegistry
+from crypto_dev_signer.keystore.dict import DictKeystore
+from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
+argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
+argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
+argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = argparser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.dict_override(args_override, 'cli')
+config.add(args.user_dir, '_USERDIR', True)
+
+user_new_dir = os.path.join(args.user_dir, 'new')
+os.makedirs(user_new_dir)
+
+meta_dir = os.path.join(args.user_dir, 'meta')
+os.makedirs(meta_dir)
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
+
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_str = str(chain_spec)
+
+batch_size = args.batch_size
+batch_delay = args.batch_delay
+
+rpc = EthHTTPConnection(args.p)
+
+signer_address = None
+keystore = DictKeystore()
+if args.y != None:
+    logg.debug('loading keystore file {}'.format(args.y))
+    signer_address = keystore.import_keystore_file(args.y)
+    logg.debug('now have key for signer address {}'.format(signer_address))
+signer = EIP155Signer(keystore)
+
+nonce_oracle = RPCNonceOracle(signer_address, rpc)
+
+CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
+registry = CICRegistry(chain_spec, rpc)
+account_registry_address = registry.by_name('AccountRegistry')
+
+keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
+os.makedirs(keyfile_dir)
+
+def register_eth(i, u):
+
+    address_hex = keystore.new()
+    address = add_0x(to_checksum_address(address_hex))
+
+    gas_oracle = RPCGasOracle(rpc, code_callback=AccountRegistry.gas)
+    c = AccountRegistry(signer=signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, chain_id=chain_spec.chain_id())
+    (tx_hash_hex, o) = c.add(account_registry_address, signer_address, address)
+    logg.debug('o {}'.format(o))
+    rpc.do(o)
+
+    pk = keystore.get(address)
+    keyfile_content = to_keyfile_dict(pk, '')
+    keyfile_path = os.path.join(keyfile_dir, '{}.json'.format(address))
+    f = open(keyfile_path, 'w')
+    json.dump(keyfile_content, f)
+    f.close()
+
+    logg.debug('[{}] register eth {} {} tx {} keyfile {}'.format(i, u, address, tx_hash_hex, keyfile_path))
+
+    return address
+   
+
+def register_ussd(u):
+    pass
+
+
+if __name__ == '__main__':
+
+
+    i = 0
+    j = 0
+    for x in os.walk(user_old_dir):
+        for y in x[2]:
+            if y[len(y)-5:] != '.json':
+                continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
+                f.close()
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person.deserialize(o)
+
+            new_address = register_eth(i, u)
+            if u.identities.get('evm') == None:
+                u.identities['evm'] = {}
+            sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
+            u.identities['evm'][sub_chain_str] = [new_address]
+
+            register_ussd(u)
+
+            new_address_clean = strip_0x(new_address)
+            filepath = os.path.join(
+                    user_new_dir,
+                    new_address_clean[:2].upper(),
+                    new_address_clean[2:4].upper(),
+                    new_address_clean.upper() + '.json',
+                    )
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
+            o = u.serialize()
+            f = open(filepath, 'w')
+            f.write(json.dumps(o))
+            f.close()
+
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
+            meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
+            os.symlink(os.path.realpath(filepath), meta_filepath)
+
+            i += 1
+            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+        
+            j += 1
+            if j == batch_size:
+                time.sleep(batch_delay)
+                j = 0
+
+    #fi.close()

apps/contract-migration/scripts/requirements.txt

@@ -1,3 +1,5 @@
-cic-base[full_graph]==0.1.2a40
-cic-eth==0.10.1b1
+cic-base[full_graph]==0.1.2a52
+sarafu-faucet==0.0.2a15
+cic-eth==0.10.1b2+build.9c750e70
 cic-types==0.1.0a8
+crypto-dev-signer==0.4.14a17

apps/contract-migration/scripts/verify.py

@@ -32,7 +32,10 @@ from chainlib.eth.block import (
 from chainlib.eth.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
-from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.gas import (
+        OverrideGasOracle,
+        balance,
+        )
 from chainlib.eth.tx import TxFactory
 from chainlib.eth.rpc import jsonrpc_template
 from chainlib.eth.error import EthException
@@ -41,6 +44,7 @@ from cic_types.models.person import (
         Person,
         generate_metadata_pointer,
         )
+from erc20_single_shot_faucet import SingleShotFaucet
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -127,17 +131,19 @@ class VerifierError(Exception):
 class Verifier:
 
     # TODO: what an awful function signature
-    def __init__(self, conn, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, data_dir, exit_on_error=False):
+    def __init__(self, conn, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
         self.conn = conn
         self.gas_oracle = gas_oracle
         self.chain_spec = chain_spec
         self.index_address = index_address
         self.token_address = token_address
+        self.faucet_address = faucet_address
         self.erc20_tx_factory = ERC20(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
         self.tx_factory = TxFactory(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
         self.api = cic_eth_api
         self.data_dir = data_dir
         self.exit_on_error = exit_on_error
+        self.faucet_tx_factory = SingleShotFaucet(chain_id=chain_spec.chain_id(), gas_oracle=gas_oracle)
 
         verifymethods = []
         for k in dir(self):
@@ -182,6 +188,21 @@ class Verifier:
             raise VerifierError((address, r), 'local key')
 
 
+    def verify_gas(self, address, balance_token=None):
+        o = balance(address)
+        r = self.conn.do(o)
+        actual_balance = int(strip_0x(r), 16)
+        if actual_balance == 0:
+            raise VerifierError((address, actual_balance), 'gas')
+
+
+    def verify_faucet(self, address, balance_token=None):
+        o = self.faucet_tx_factory.usable_for(self.faucet_address, address)
+        r = self.conn.do(o)
+        if self.faucet_tx_factory.parse_usable_for(r):
+            raise VerifierError((address, r), 'faucet')
+
+
     def verify_metadata(self, address, balance=None):
         k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person')
         url = os.path.join(meta_url, k)
@@ -220,6 +241,8 @@ class Verifier:
                 'accounts_index',
                 'balance',
                 'metadata',
+                'gas',
+                'faucet',
                 ]
 
         for k in methods:
@@ -257,6 +280,7 @@ def main():
     txf = TxFactory(signer=None, gas_oracle=gas_oracle, nonce_oracle=None, chain_id=chain_spec.chain_id())
     tx = txf.template(ZERO_ADDRESS, config.get('CIC_REGISTRY_ADDRESS'))
 
+    # TODO: replace with cic-eth-registry
     registry_addressof_method = keccak256_string_to_hex('addressOf(bytes32)')[:8]
     data = add_0x(registry_addressof_method)
     data += eth_abi.encode_single('bytes32', b'TokenRegistry').hex()
@@ -283,6 +307,18 @@ def main():
     account_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
     logg.info('found account index address {}'.format(account_index_address))
 
+    data = add_0x(registry_addressof_method)
+    data += eth_abi.encode_single('bytes32', b'Faucet').hex()
+    txf.set_code(tx, data)
+    
+    o = jsonrpc_template()
+    o['method'] = 'eth_call'
+    o['params'].append(txf.normalize(tx))
+    o['params'].append('latest')
+    r = conn.do(o)
+    faucet_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
+    logg.info('found faucet {}'.format(faucet_address))
+
 
     # Get Sarafu token address
     tx = txf.template(ZERO_ADDRESS, token_index_address)
@@ -323,7 +359,7 @@ def main():
 
     api = AdminApi(MockClient())
 
-    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, user_dir, exit_on_error)
+    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, faucet_address, user_dir, exit_on_error)
 
     user_new_dir = os.path.join(user_dir, 'new')
     for x in os.walk(user_new_dir):

apps/contract-migration/seed_cic_eth.sh

@@ -22,7 +22,7 @@ abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
 gas_amount=100000000000000000000000
 token_amount=${gas_amount}
 #faucet_amount=1000000000
-faucet_amount=0
+faucet_amount=${DEV_FAUCET_AMOUNT:-0}
 env_out_file=${CIC_DATA_DIR}/.env_seed
 init_level_file=${CIC_DATA_DIR}/.init
 truncate $env_out_file -s 0
@@ -32,11 +32,11 @@ set -a
 
 # get required addresses from registries
 DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
-DEV_ACCOUNTS_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
+DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
 DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief SRF`
 cat <<EOF
 Token registry: $DEV_TOKEN_INDEX_ADDRESS
-Account reigstry: $DEV_ACCOUNTS_INDEX_ADDRESS
+Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
 Reserve address: $DEV_RESERVE_ADDRESS
 EOF
 
@@ -66,7 +66,7 @@ DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host-call
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
-eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNTS_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
+eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
 >&2 echo gift gas to gas gifter

docker-compose.yml

@@ -102,6 +102,7 @@ services:
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
+      DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
     command: ["./run_job.sh"]
         #command: ["./reset.sh"]
     depends_on: