Merge remote-tracking branch 'origin/master' into lash/emergency-shutdown

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -2,7 +2,7 @@
 import datetime
 import logging
 
-# external imports
+# third-party imports
 import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
@@ -145,9 +145,3 @@ def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
     session.flush()
     session.close()
     return chained_input
-
-
-@celery_app.task()
-def shutdown(message):
-    logg.critical('shutdown called: {}'.format(message))
-    celery_app.control.shutdown() #broadcast('shutdown')

apps/cic-eth/cic_eth/admin/token.py

@@ -1,19 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-# local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            }

apps/cic-eth/cic_eth/api/api_admin.py

@@ -60,29 +60,6 @@ class AdminApi:
         self.call_address = call_address
 
 
-    def proxy_do(self, chain_spec, o):
-        s_proxy = celery.signature(
-                'cic_eth.task.rpc_proxy',
-                [
-                    chain_spec.asdict(),
-                    o,
-                    'default',
-                    ],
-                queue=self.queue
-                )
-        return s_proxy.apply_async()
-
-
-    
-    def registry(self):
-        s_registry = celery.signature(
-                'cic_eth.task.registry',
-                [],
-                queue=self.queue
-                )
-        return s_registry.apply_async()
-
-
     def unlock(self, chain_spec, address, flags=None):
         s_unlock = celery.signature(
             'cic_eth.admin.ctrl.unlock',
@@ -169,6 +146,7 @@ class AdminApi:
 
         # TODO: This check should most likely be in resend task itself
         tx_dict = s_get_tx_cache.apply_async().get()
+        #if tx_dict['status'] in [StatusEnum.REVERTED, StatusEnum.SUCCESS, StatusEnum.CANCELLED, StatusEnum.OBSOLETED]: 
         if not is_alive(getattr(StatusEnum, tx_dict['status']).value):
             raise TxStateChangeError('Cannot resend mined or obsoleted transaction'.format(txold_hash_hex))
         
@@ -248,6 +226,9 @@ class AdminApi:
                 break
             last_nonce = nonce_otx
 
+        #nonce_cache = Nonce.get(address)
+        #nonce_w3 = self.w3.eth.getTransactionCount(address, 'pending') 
+        
         return {
             'nonce': {
                 #'network': nonce_cache,
@@ -291,6 +272,20 @@ class AdminApi:
         return s_nonce.apply_async()
 
 
+#    # TODO: this is a stub, complete all checks
+#    def ready(self):
+#        """Checks whether all required initializations have been performed.
+#
+#        :raises cic_eth.error.InitializationError: At least one setting pre-requisite has not been met.
+#        :raises KeyError: An address provided for initialization is not known by the keystore.
+#        """
+#        addr = AccountRole.get_address('ETH_GAS_PROVIDER_ADDRESS')
+#        if addr == ZERO_ADDRESS:
+#            raise InitializationError('missing account ETH_GAS_PROVIDER_ADDRESS')
+#
+#        self.w3.eth.sign(addr, text='666f6f')
+
+
     def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
         """Lists locally originated transactions for the given Ethereum address.
 
@@ -353,7 +348,6 @@ class AdminApi:
 
 
     # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    # TODO: This method is WAY too long
     def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
         """Output local and network details about a given transaction with local origin.
 
@@ -376,6 +370,7 @@ class AdminApi:
 
         if tx_raw != None:
             tx_hash = add_0x(keccak256_hex_to_hex(tx_raw))
+            #tx_hash = self.w3.keccak(hexstr=tx_raw).hex()
 
         s = celery.signature(
             'cic_eth.queue.query.get_tx_cache',
@@ -391,78 +386,38 @@ class AdminApi:
   
         source_token = None
         if tx['source_token'] != ZERO_ADDRESS:
-            if registry != None:
+            try:
-                try:
+                source_token = registry.by_address(tx['source_token'])
-                    source_token = registry.by_address(tx['source_token'])
+                #source_token = CICRegistry.get_address(chain_spec, tx['source_token']).contract
-                except UnknownContractError:
+            except UnknownContractError:
-                    logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
+                #source_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
-            else:
+                #source_token = CICRegistry.add_token(chain_spec, source_token_contract)
-                s = celery.signature(
+                logg.warning('unknown source token contract {}'.format(tx['source_token']))
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['source_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                source_token = t.get()
-                if source_token == None:
-                    logg.warning('unknown source token contract {} (task pool)'.format(tx['source_token']))
-
 
         destination_token = None
-        if tx['destination_token'] != ZERO_ADDRESS:
+        if tx['source_token'] != ZERO_ADDRESS:
-            if registry != None:
+            try:
-                try:
+                #destination_token = CICRegistry.get_address(chain_spec, tx['destination_token'])
-                    destination_token = registry.by_address(tx['destination_token'])
+                destination_token = registry.by_address(tx['destination_token'])
-                except UnknownContractError:
+            except UnknownContractError:
-                    logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
+                #destination_token_contract = self.w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=tx['source_token'])
-            else:
+                #destination_token = CICRegistry.add_token(chain_spec, destination_token_contract)
-                s = celery.signature(
+                logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['destination_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                destination_token = t.get()
-                if destination_token == None:
-                    logg.warning('unknown destination token contract {} (task pool)'.format(tx['destination_token']))
-
 
         tx['sender_description'] = 'Custodial account'
         tx['recipient_description'] = 'Custodial account'
 
         o = code(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
+        r = self.rpc.do(o)
-        r = t.get()
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
+            try:
-                try:
+                #sender_contract = CICRegistry.get_address(chain_spec, tx['sender'])
-                    sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
+                sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
-                    tx['sender_description'] = 'Contract at {}'.format(tx['sender'])
+                tx['sender_description'] = 'Contract at {}'.format(tx['sender']) #sender_contract)
-                except UnknownContractError:
+            except UnknownContractError:
-                    tx['sender_description'] = 'Unknown contract'
+                tx['sender_description'] = 'Unknown contract'
-                except KeyError as e:
+            except KeyError as e:
-                    tx['sender_description'] = 'Unknown contract'
+                tx['sender_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['sender'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['sender_description'] = t.get()
-                if tx['sender_description'] == None:
-                    tx['sender_description'] = 'Unknown contract'
-
-
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -491,31 +446,16 @@ class AdminApi:
                     tx['sender_description'] = role
 
         o = code(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
+        r = self.rpc.do(o)
-        r = t.get()
         if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
+            try:
-                try:
+                #recipient_contract = CICRegistry.by_address(tx['recipient'])
-                    recipient_contract = registry.by_address(tx['recipient'])
+                recipient_contract = registry.by_address(tx['recipient'])
-                    tx['recipient_description'] = 'Contract at {}'.format(tx['recipient'])
+                tx['recipient_description'] = 'Contract at {}'.format(tx['recipient']) #recipient_contract)
-                except UnknownContractError as e:
+            except UnknownContractError as e:
-                    tx['recipient_description'] = 'Unknown contract'
+                tx['recipient_description'] = 'Unknown contract'
-                except KeyError as e:
+            except KeyError as e:
-                    tx['recipient_description'] = 'Unknown contract'
+                tx['recipient_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['recipient'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['recipient_description'] = t.get()
-                if tx['recipient_description'] == None:
-                    tx['recipient_description'] = 'Unknown contract'
-
         else:
             s = celery.signature(
                     'cic_eth.eth.account.have',
@@ -557,8 +497,7 @@ class AdminApi:
         r = None
         try:
             o = transaction(tx_hash)
-            t = self.proxy_do(chain_spec, o)
+            r = self.rpc.do(o)
-            r = t.get()
             if r != None:
                 tx['network_status'] = 'Mempool'
         except Exception as e:
@@ -567,8 +506,7 @@ class AdminApi:
         if r != None:
             try:
                 o = receipt(tx_hash)
-                t = self.proxy_do(chain_spec, o)
+                r = self.rpc.do(o)
-                r = t.get()
                 logg.debug('h {} o {}'.format(tx_hash, o))
                 if int(strip_0x(r['status'])) == 1:
                     tx['network_status'] = 'Confirmed'
@@ -583,13 +521,11 @@ class AdminApi:
                 pass
 
         o = balance(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
+        r = self.rpc.do(o)
-        r = t.get()
         tx['sender_gas_balance'] = r
 
         o = balance(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
+        r = self.rpc.do(o)
-        r = t.get()
         tx['recipient_gas_balance'] = r
 
         tx_unpacked = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)

apps/cic-eth/cic_eth/api/api_task.py

@@ -62,18 +62,6 @@ class Api:
                     )       
 
 
-    def default_token(self):
-        s_token = celery.signature(
-                'cic_eth.admin.token.default_token',
-                [],
-                queue=self.queue,
-                )
-        if self.callback_param != None:
-            s_token.link(self.callback_success)
-
-        return s_token.apply_async()
-
-
     def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
         """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 

apps/cic-eth/cic_eth/check/redis.py

@@ -1,18 +0,0 @@
-# external imports
-import redis
-import os
-
-
-def health(*args, **kwargs):
-    r = redis.Redis(
-            host=kwargs['config'].get('REDIS_HOST'),
-            port=kwargs['config'].get('REDIS_PORT'),
-            db=kwargs['config'].get('REDIS_DB'),
-            )
-    try:
-        r.set(kwargs['unit'], os.getpid())
-    except redis.connection.ConnectionError:
-        return False
-    except redis.connection.ResponseError:
-        return False
-    return True

apps/cic-eth/cic_eth/error.py

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
     pass
 
 
-
-
 class IntegrityError(Exception):
     """Exception raised to signal irregularities with deduplication and ordering of tasks
 
@@ -64,19 +62,15 @@ class LockedError(Exception):
     pass
 
 
-class SeppukuError(Exception):
+class SignerError(Exception):
-    """Exception base class for all errors that should cause system shutdown
-
-    """
-
-
-class SignerError(SeppukuError):
     """Exception raised when signer is unavailable or generates an error
 
     """
     pass
 
 
-class RoleAgencyError(SeppukuError):
+class EthError(Exception):
-    """Exception raise when a role cannot perform its function. This is a critical exception
+    """Exception raised when unspecified error from evm node is encountered
+
     """
+    pass

apps/cic-eth/cic_eth/eth/account.py

@@ -4,10 +4,10 @@ import logging
 # external imports
 import celery
 from erc20_single_shot_faucet import SingleShotFaucet as Faucet
+from chainlib.eth.constant import ZERO_ADDRESS
 from hexathon import (
         strip_0x,
         )
-from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.connection import RPCConnection
 from chainlib.eth.sign import (
         new_account,
@@ -19,7 +19,6 @@ from chainlib.eth.tx import (
         unpack,
         )
 from chainlib.chain import ChainSpec
-from chainlib.error import JSONRPCException
 from eth_accounts_index import AccountRegistry
 from sarafu_faucet import MinterFaucet as Faucet
 from chainqueue.db.models.tx import TxCache
@@ -71,18 +70,11 @@ def create(self, password, chain_spec_dict):
     a = None
     conn = RPCConnection.connect(chain_spec, 'signer')
     o = new_account()
-    try:
+    a = conn.do(o)
-        a = conn.do(o)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
     conn.disconnect()
 
-    # TODO: It seems infeasible that a can be None in any case, verify
     if a == None:
         raise SignerError('create account')
-        
     logg.debug('created account {}'.format(a))
 
     # Initialize nonce provider record for account
@@ -227,22 +219,21 @@ def have(self, account, chain_spec_dict):
     """
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     o = sign_message(account, '0x2a')
-    conn = RPCConnection.connect(chain_spec, 'signer')
+    try:
+        conn = RPCConnection.connect(chain_spec, 'signer')
+    except Exception as e:
+        logg.debug('cannot sign with {}: {}'.format(account, e))
+        return None
 
     try:
         conn.do(o)
-    except ConnectionError as e:
+        conn.disconnect()
-        raise SignerError(e)
+        return account
-    except FileNotFoundError as e:
+    except Exception as e:
-        raise SignerError(e)
-    except JSONRPCException as e:
         logg.debug('cannot sign with {}: {}'.format(account, e))
         conn.disconnect()
         return None
    
-    conn.disconnect()
-    return account
-
 
 @celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
 def set_role(self, tag, address, chain_spec_dict):

apps/cic-eth/cic_eth/eth/erc20.py

@@ -108,13 +108,7 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
+    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-        (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
-    
 
     rpc_signer.disconnect()
     rpc.disconnect()
@@ -177,12 +171,7 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
-    try:
+    (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
-        (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
-    except FileNotFoundError as e:
-        raise SignerError(e)
-    except ConnectionError as e:
-        raise SignerError(e)
 
     rpc_signer.disconnect()
     rpc.disconnect()

apps/cic-eth/cic_eth/eth/gas.py

@@ -328,12 +328,7 @@ def refill_gas(self, recipient_address, chain_spec_dict):
 
     # build and add transaction
     logg.debug('tx send gas amount {} from provider {} to {}'.format(refill_amount, gas_provider, recipient_address))
-    try:
+    (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
-        (tx_hash_hex, tx_signed_raw_hex) = c.create(gas_provider, recipient_address, refill_amount, tx_format=TxFormat.RLP_SIGNED)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
     logg.debug('adding queue refill gas tx {}'.format(tx_hash_hex))
     cache_task = 'cic_eth.eth.gas.cache_gas_data'
     register_tx(tx_hash_hex, tx_signed_raw_hex, chain_spec, queue, cache_task=cache_task, session=session)
@@ -409,12 +404,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
     c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
     logg.debug('change gas price from old {} to new {} for tx {}'.format(tx['gasPrice'], new_gas_price, tx))
     tx['gasPrice'] = new_gas_price
-    try:
+    (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx)
-    except ConnectionError as e:
-        raise SignerError(e)
-    except FileNotFoundError as e:
-        raise SignerError(e)
     queue_create(
         chain_spec,
         tx['nonce'],

apps/cic-eth/cic_eth/ext/tx.py

@@ -114,7 +114,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
 
                         # TODO: pass through registry to validate declarator entry of token
                         #token = registry.by_address(tx['to'], sender_address=self.call_address)
-                        token = ERC20Token(chain_spec, rpc, tx['to'])
+                        token = ERC20Token(rpc, tx['to'])
                         token_symbol = token.symbol
                         token_decimals = token.decimals
                         times = tx_times(tx['hash'], chain_spec)

apps/cic-eth/cic_eth/queue/time.py

@@ -12,7 +12,6 @@ from chainqueue.error import NotLocalTxError
 
 # local imports
 from cic_eth.task import CriticalSQLAlchemyAndWeb3Task
-from cic_eth.db.models.base import SessionBase
 
 celery_app = celery.current_app
 

apps/cic-eth/cic_eth/registry.py

@@ -29,5 +29,5 @@ def connect(rpc, chain_spec, registry_address):
     CICRegistry.address = registry_address
     registry = CICRegistry(chain_spec, rpc)
     registry_address = registry.by_name('ContractRegistry')
+    
     return registry
-

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -23,6 +23,7 @@ default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
 argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -11,14 +11,8 @@ import websocket
 # external imports
 import celery
 import confini
-from chainlib.connection import (
+from chainlib.connection import RPCConnection
-        RPCConnection,
+from chainlib.eth.connection import EthUnixSignerConnection
-        ConnType,
-        )
-from chainlib.eth.connection import (
-        EthUnixSignerConnection,
-        EthHTTPSignerConnection,
-        )
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
 from cic_eth_registry.error import UnknownContractError
@@ -60,7 +54,6 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
-from cic_eth.task import BaseTask
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -73,7 +66,6 @@ argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provide
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
-argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -93,7 +85,6 @@ config.process()
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
@@ -148,10 +139,8 @@ else:
         })
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
-RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
 RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
+#RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
 RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
@@ -159,7 +148,7 @@ Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 #import cic_eth.checks.gas
 #if not cic_eth.checks.gas.health(config=config):
 #    raise RuntimeError()
-liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker')
+liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config)
 
 def main():
     argv = ['worker']
@@ -184,7 +173,7 @@ def main():
     rpc = RPCConnection.connect(chain_spec, 'default')
 
     try:
-        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+        connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
@@ -199,15 +188,10 @@ def main():
 
     connect_declarator(rpc, chain_spec, trusted_addresses)
     connect_token_registry(rpc, chain_spec)
-
-    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
-    BaseTask.run_dir = config.get('CIC_RUN_DIR')
-    logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    
-    liveness.linux.set(rundir=config.get('CIC_RUN_DIR'))
+    liveness.linux.set()
     current_app.worker_main(argv)
-    liveness.linux.reset(rundir=config.get('CIC_RUN_DIR'))
+    liveness.linux.reset()
 
 
 @celery.signals.eventlet_pool_postshutdown.connect

apps/cic-eth/cic_eth/runnable/info.py

@@ -1,65 +0,0 @@
-#!python3
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-# standard imports
-import logging
-import argparse
-import os
-
-# external imports
-import confini
-import celery
-
-# local imports
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_format = 'terminal'
-default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-args = argparser.parse_args()
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli args')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-queue = args.q
-
-api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
-
-def main():
-    t = api.default_token()
-    token_info = t.get()
-    print('Default token symbol: {}'.format(token_info['symbol']))
-    print('Default token address: {}'.format(token_info['address']))
-
-
-if __name__ == '__main__':
-    main()

apps/cic-eth/cic_eth/runnable/transfer.py

@@ -85,6 +85,9 @@ def main():
             callback_queue=args.q,
             )
 
+    #register = not args.no_register
+    #logg.debug('register {}'.format(register))
+    #t = api.create_account(register=register)
     t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL'))
 
     ps.get_message()

apps/cic-eth/cic_eth/runnable/view.py

@@ -81,14 +81,10 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 rpc = EthHTTPConnection(args.p)
 
-#registry_address = config.get('CIC_REGISTRY_ADDRESS')
+registry_address = config.get('CIC_REGISTRY_ADDRESS')
 
 admin_api = AdminApi(rpc)
 
-t = admin_api.registry()
-registry_address = t.get()
-logg.info('got registry address from task pool: {}'.format(registry_address))
-
 trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
 if trusted_addresses_src == None:
     logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -155,16 +151,14 @@ def main():
     txs  = []
     renderer = render_tx
     if len(config.get('_QUERY')) > 66:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
+        registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
+        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
     elif len(config.get('_QUERY')) > 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
+        registry = connect_registry(rpc, chain_spec, registry_address)
-        #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
+        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
-        admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
 
     elif len(config.get('_QUERY')) == 42:
-        #registry = connect_registry(rpc, chain_spec, registry_address)
+        registry = connect_registry(rpc, chain_spec, registry_address)
         txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
         renderer = render_account
     elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':

apps/cic-eth/cic_eth/task.py

@@ -7,20 +7,18 @@ import uuid
 # external imports
 import celery
 import sqlalchemy
-from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.gas import RPCGasOracle
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.error import UnknownContractError
-import liveness.linux
 
 # local imports
-from cic_eth.error import SeppukuError
+from cic_eth.error import (
+        SignerError,
+        EthError,
+        )
 from cic_eth.db.models.base import SessionBase
 
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__name__)
 
 celery_app = celery.current_app
 
@@ -31,9 +29,6 @@ class BaseTask(celery.Task):
     call_address = ZERO_ADDRESS
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
-    default_token_address = None
-    default_token_symbol = None
-    run_dir = '/run'
 
     def create_session(self):
         return BaseTask.session_func()
@@ -43,19 +38,6 @@ class BaseTask(celery.Task):
         logg.debug('task {} root uuid {}'.format(self.__class__.__name__, self.request.root_id))
         return
 
-
-    def on_failure(self, exc, task_id, args, kwargs, einfo):
-        if isinstance(exc, SeppukuError):
-            liveness.linux.reset(rundir=self.run_dir)
-            logg.critical(einfo)
-            msg = 'received critical exception {}, calling shutdown'.format(str(exc))
-            s = celery.signature(
-                'cic_eth.admin.ctrl.shutdown',
-                [msg],
-                queue=self.request.delivery_info.get('routing_key'),
-                    )
-            s.apply_async()
-
     
 class CriticalTask(BaseTask):
     retry_jitter = True
@@ -85,6 +67,7 @@ class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
         sqlalchemy.exc.TimeoutError,
         requests.exceptions.ConnectionError,
         sqlalchemy.exc.ResourceClosedError,
+        EthError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
@@ -95,45 +78,24 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
         sqlalchemy.exc.DatabaseError,
         sqlalchemy.exc.TimeoutError,
         sqlalchemy.exc.ResourceClosedError,
+        SignerError,
         ) 
 
 class CriticalWeb3AndSignerTask(CriticalTask):
     autoretry_for = (
         requests.exceptions.ConnectionError,
+        SignerError,
         )
     safe_gas_threshold_amount = 2000000000 * 60000 * 3
     safe_gas_refill_amount = safe_gas_threshold_amount * 5 
 
 
+@celery_app.task(bind=True, base=BaseTask)
+def hello(self):
+    time.sleep(0.1)
+    return id(SessionBase.create_session)
+
+
 @celery_app.task()
 def check_health(self):
-    pass
+    celery.app.control.shutdown()
-
-
-# TODO: registry / rpc methods should perhaps be moved to better named module
-@celery_app.task()
-def registry():
-    return CICRegistry.address
-
-
-@celery_app.task()
-def registry_address_lookup(chain_spec_dict, address, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_address(address)
-
-
-@celery_app.task(throws=(UnknownContractError,))
-def registry_name_lookup(chain_spec_dict, name, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    registry = CICRegistry(chain_spec, conn)
-    return registry.by_name(name)
-
-
-@celery_app.task()
-def rpc_proxy(chain_spec_dict, o, connection_tag='default'):
-    chain_spec = ChainSpec.from_dict(chain_spec_dict)
-    conn = RPCConnection.connect(chain_spec, tag=connection_tag)
-    return conn.do(o)

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.11',
+        'beta.8',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -3,6 +3,5 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =
-default_token_symbol = GFT
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/config/docker/cic.ini

@@ -3,6 +3,5 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 tx_retry_delay = 20
-default_token_symbol = GFT
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
-health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
 run_dir = /run

apps/cic-eth/config/docker/eth.ini

@@ -1,3 +1,4 @@
 [eth]
 provider = http://localhost:63545
+health_modules = cic_eth.check.db,cic_eth.check.gas
 gas_gifter_minimum_balance = 10000000000000000000000

apps/cic-eth/config/eth.ini

@@ -1,3 +1,4 @@
 [eth]
 provider = http://localhost:8545
 gas_gifter_minimum_balance = 10000000000000000000000
+health_modules = cic_eth.check.db,cic_eth.check.gas

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base==0.1.2b5
+cic-base==0.1.2b1
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
@@ -15,10 +15,10 @@ semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
 eth-address-index~=0.1.1a9
-chainlib~=0.0.2a20
+chainlib~=0.0.2a13
 hexathon~=0.0.1a7
 chainsyncer[sql]~=0.0.2a2
-chainqueue~=0.0.2a2
+chainqueue~=0.0.1a7
 pysha3==1.0.2
 coincurve==15.0.0
 sarafu-faucet==0.0.2a28

apps/cic-eth/setup.cfg

@@ -53,7 +53,6 @@ console_scripts =
 	cic-eth-create = cic_eth.runnable.create:main 
 	cic-eth-inspect = cic_eth.runnable.view:main
 	cic-eth-ctl = cic_eth.runnable.ctrl:main
-	cic-eth-info = cic_eth.runnable.info:main
 	# TODO: Merge this with ctl when subcmds sorted to submodules
 	cic-eth-tag = cic_eth.runnable.tag:main
 	cic-eth-resend = cic_eth.runnable.resend:main

apps/cic-eth/tests/conftest.py

@@ -3,12 +3,8 @@ import os
 import sys
 import logging
 
-# external imports
-from chainlib.eth.erc20 import ERC20
-
 # local imports
 from cic_eth.api import Api
-from cic_eth.task import BaseTask
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -32,26 +28,3 @@ def api(
     ):
     chain_str = str(default_chain_spec)
     return Api(chain_str, queue=None, callback_param='foo')
-
-
-@pytest.fixture(scope='function')
-def foo_token_symbol(
-    default_chain_spec,
-    foo_token,
-    eth_rpc,
-    contract_roles,
-    ):
-    
-    c = ERC20(default_chain_spec)
-    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
-@pytest.fixture(scope='function')
-def default_token(
-        foo_token,
-        foo_token_symbol,
-        ):
-    BaseTask.default_token_symbol = foo_token_symbol
-    BaseTask.default_token_address = foo_token   

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -210,11 +210,9 @@ def test_callback_filter(
 
         def __init__(self):
             self.results = {}
-            self.queue = 'test'
 
         def call_back(self, transfer_type, result):
             self.results[transfer_type] = result
-            return self
 
     mock = CallbackMock()
     fltr.call_back = mock.call_back

apps/cic-eth/tests/filters/test_tx_filter.py

@@ -65,7 +65,6 @@ def test_tx(
     tx_hash_hex_orig = tx_hash_hex
 
     gas_oracle = OverrideGasOracle(price=1100000000, limit=21000)
-    c = Gas(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 100 * (10 ** 6), tx_format=TxFormat.RLP_SIGNED)
     queue_create(
             default_chain_spec,

apps/cic-eth/tests/fixtures_celery.py

@@ -34,7 +34,6 @@ def celery_includes():
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
         'cic_eth.admin.debug',
-        'cic_eth.admin.token',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',

apps/cic-eth/tests/fixtures_database.py

@@ -53,9 +53,6 @@ def init_database(
     alembic.command.downgrade(ac, 'base')
     alembic.command.upgrade(ac, 'head')
 
-    session.execute('DELETE FROM lock')
-    session.commit()
-
     yield session
     session.commit()
     session.close()

apps/cic-eth/tests/task/api/test_admin.py

@@ -273,7 +273,7 @@ def test_tx(
     eth_signer,
     agent_roles,
     contract_roles,
-    celery_session_worker,
+    celery_worker,
     ):
 
     nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)

apps/cic-eth/tests/task/api/test_list.py

@@ -35,7 +35,7 @@ def test_list_tx(
         foo_token,
         register_tokens,
         init_eth_tester,
-        celery_session_worker,
+        celery_worker,
         ):
 
     tx_hashes = []

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -1,21 +0,0 @@
-# external imports
-import celery
-
-
-def test_default_token(
-        default_token,
-        celery_session_worker,
-        foo_token,
-        foo_token_symbol,
-        ):
-      
-    s = celery.signature(
-            'cic_eth.admin.token.default_token',
-            [],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get()
-
-    assert r['address'] == foo_token
-    assert r['symbol'] == foo_token_symbol

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b2
+cic_base[full_graph]~=0.1.2a79
-cic-eth~=0.11.0b9
+cic-eth~=0.11.0b7
 cic-notify~=0.4.0a4
 cic-types~=0.1.0a10

apps/contract-migration/reset.sh

@@ -3,7 +3,7 @@
 set -a
 
 CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
-CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -19,8 +19,8 @@ if [ ! -z $DEV_ETH_GAS_PRICE ]; then
 	>&2 echo using static gas price $DEV_ETH_GAS_PRICE
 fi
 
-if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
+if [[ $DEV_TOKEN_TYPE != 'GFT' && $DEV_TOKEN_TYPE != 'SRF' ]]; then
-	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
+	>&2 echo DEV_TOKEN_TYPE must be one of [GFT,SRF], but was $DEV_TOKEN_TYPE
 	exit 1
 fi
 
@@ -60,7 +60,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ $CIC_DEFAULT_TOKEN_SYMBOL == 'GFT' ]; then
+	if [ $DEV_TOKEN_TYPE == 'GFT' ]; then
 		>&2 echo "deploying 'giftable token'"
 		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Giftable Token" --symbol "GFT" --decimals 6 -vv`
 	else

apps/contract-migration/scripts/cic_eth/import_users.py

@@ -32,7 +32,6 @@ default_config_dir = '/usr/local/etc/cic'
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
-argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
 argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
 argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -77,11 +76,6 @@ os.makedirs(user_new_dir)
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
-custom_dir = os.path.join(args.user_dir, 'custom')
-os.makedirs(custom_dir)
-os.makedirs(os.path.join(custom_dir, 'new'))
-os.makedirs(os.path.join(custom_dir, 'meta'))
-
 phone_dir = os.path.join(args.user_dir, 'phone')
 os.makedirs(os.path.join(phone_dir, 'meta'))
 
@@ -91,11 +85,6 @@ os.stat(user_old_dir)
 txs_dir = os.path.join(args.user_dir, 'txs')
 os.makedirs(txs_dir)
 
-user_dir = args.user_dir
-
-old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
-old_chain_str = str(old_chain_spec)
-
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 
@@ -144,17 +133,7 @@ def register_eth(i, u):
 
 if __name__ == '__main__':
 
-    user_tags = {}
+    #fi = open(os.path.join(user_out_dir, 'addresses.csv'), 'a')
-    f = open(os.path.join(user_dir, 'tags.csv'), 'r')
-    while True:
-        r = f.readline().rstrip()
-        if len(r) == 0:
-            break
-        (old_address, tags_csv) = r.split(':')
-        old_address = strip_0x(old_address)
-        user_tags[old_address] = tags_csv.split(',')
-        logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
-
 
     i = 0
     j = 0
@@ -219,29 +198,6 @@ if __name__ == '__main__':
             os.symlink(os.path.realpath(filepath), meta_phone_filepath)
 
 
-            # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
-            custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
-
-            filepath = os.path.join(
-                    custom_dir,
-                    'new',
-                    custom_key[:2].upper(),
-                    custom_key[2:4].upper(),
-                    custom_key.upper() + '.json',
-                    )
-            os.makedirs(os.path.dirname(filepath), exist_ok=True)
-           
-            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
-            f = open(filepath, 'w')
-            k = u.identities['evm'][sub_old_chain_str][0]
-            tag_data = {'tags': user_tags[strip_0x(k)]}
-            f.write(json.dumps(tag_data))
-            f.close()
-
-            os.symlink(os.path.realpath(filepath), custom_filepath)
-
-
             i += 1
             sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
         

apps/contract-migration/scripts/cic_meta/import_meta_custom.js

@@ -1,139 +0,0 @@
-const fs = require('fs');
-const path = require('path');
-const http = require('http');
-
-const cic = require('cic-client-meta');
-const vcfp = require('vcard-parser');
-
-//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
-
-const config = new cic.Config('./config'); 
-config.process();
-console.log(config);
-
-
-function sendit(uid, envelope) {
-	const d = envelope.toJSON();
-
-	const contentLength = (new TextEncoder().encode(d)).length;
-	const opts = {
-		method: 'PUT',
-		headers: {
-			'Content-Type': 'application/json',
-			'Content-Length': contentLength,
-			'X-CIC-AUTOMERGE': 'client',
-
-		},
-	};
-	let url = config.get('META_URL');
-	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
-	console.log('posting to url: ' + url + uid);
-	const req = http.request(url + uid, opts, (res) => {
-		res.on('data', process.stdout.write);
-		res.on('end', () => {
-			console.log('result', res.statusCode, res.headers);
-		});
-	});
-	if (!req.write(d)) {
-		console.error('foo', d);
-		process.exit(1);
-	}
-	req.end();
-}
-
-function doOne(keystore, filePath, identifier) {
-	const signer = new cic.PGPSigner(keystore);
-
-	const o = JSON.parse(fs.readFileSync(filePath).toString());
-	//const b = Buffer.from(j['vcard'], 'base64');
-	//const s = b.toString();
-	//const o = vcfp.parse(s);
-	//const phone = o.tel[0].value;
-
-	//cic.Phone.toKey(phone).then((uid) => {
-	//const o = fs.readFileSync(filePath, 'utf-8');
-
-	const s = new cic.Syncable(identifier, o);
-	s.setSigner(signer);
-	s.onwrap = (env) => {
-		sendit(identifier, env);
-	};
-	s.sign();
-	//});
-}
-
-const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
-const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
-pk = fs.readFileSync(privateKeyPath);
-pubk = fs.readFileSync(publicKeyPath);
-
-new cic.PGPKeyStore(
-	config.get('PGP_PASSPHRASE'),
-	pk,
-	pubk,
-	undefined,
-	undefined,
-	importMetaCustom,
-);
-
-const batchSize = 16;
-const batchDelay = 1000;
-const total = parseInt(process.argv[3]);
-const dataDir = process.argv[2];
-const workDir = path.join(dataDir, 'custom/meta');
-const userDir = path.join(dataDir, 'custom/new');
-let count = 0;
-let batchCount = 0;
-
-
-function importMetaCustom(keystore) {
-	let err;
-	let files;
-
-	try {
-		err, files = fs.readdirSync(workDir);
-	} catch {
-		console.error('source directory not yet ready', workDir);
-		setTimeout(importMetaPhone, batchDelay, keystore);
-		return;
-	}
-	let limit = batchSize;
-	if (files.length < limit) {
-		limit = files.length;
-	}
-	for (let i = 0; i < limit; i++) {
-		const file = files[i];
-		if (file.length < 3) {
-			console.debug('skipping file', file);	
-			continue;
-		}
-		//const identifier = file.substr(0,file.length-5);
-		const identifier = file;
-		const filePath = path.join(workDir, file);
-		console.log(filePath);
-
-		//const address = fs.readFileSync(filePath).toString().substring(2).toUpperCase();
-		const custom = JSON.parse(fs.readFileSync(filePath).toString());
-		const customFilePath = path.join(
-			userDir,
-			identifier.substring(0, 2),
-			identifier.substring(2, 4),
-			identifier + '.json',
-		);
-
-		doOne(keystore, filePath, identifier);
-		fs.unlinkSync(filePath);
-		count++;
-		batchCount++;
-		if (batchCount == batchSize) {
-			console.debug('reached batch size, breathing');
-			batchCount=0;
-			setTimeout(importMeta, batchDelay, keystore);
-			return;
-		}
-	}
-	if (count == total) {
-		return;
-	}
-	setTimeout(importMetaCustom, 100, keystore);
-}

apps/contract-migration/scripts/create_import_users.py

@@ -37,7 +37,6 @@ config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
-argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record')
 argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -82,10 +81,6 @@ phone_idx = []
 user_dir = args.dir
 user_count = args.user_count
 
-tags = args.tag
-if tags == None or len(tags) == 0:
-    tags = ['individual']
-
 random.seed()
 
 def genPhoneIndex(phone):
@@ -194,7 +189,6 @@ if __name__ == '__main__':
     os.makedirs(base_dir, exist_ok=True)
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
-    ft = open(os.path.join(user_dir, 'tags.csv'), 'w')
 
     i = 0
     while i < user_count:
@@ -221,12 +215,10 @@ if __name__ == '__main__':
         f.write(eth)
         f.close()
 
-        ft.write('{}:{}\n'.format(eth, ','.join(tags)))
         amount = genAmount()
         fa.write('{},{}\n'.format(eth,amount))
         logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone))
         
         i += 1
 
-    ft.close()
     fa.close()

apps/contract-migration/scripts/eth/import_users.py

@@ -38,7 +38,6 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
 argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
-argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
 argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
 argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
@@ -69,11 +68,6 @@ os.makedirs(user_new_dir)
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
-custom_dir = os.path.join(args.user_dir, 'custom')
-os.makedirs(custom_dir)
-os.makedirs(os.path.join(custom_dir, 'new'))
-os.makedirs(os.path.join(custom_dir, 'meta'))
-
 phone_dir = os.path.join(args.user_dir, 'phone')
 os.makedirs(os.path.join(phone_dir, 'meta'))
 
@@ -83,14 +77,9 @@ os.stat(user_old_dir)
 txs_dir = os.path.join(args.user_dir, 'txs')
 os.makedirs(txs_dir)
 
-user_dir = args.user_dir
-
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 
-old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
-old_chain_str = str(old_chain_spec)
-
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
@@ -138,18 +127,12 @@ def register_eth(i, u):
     return address
    
 
+def register_ussd(u):
+    pass
+
+
 if __name__ == '__main__':
 
-    user_tags = {}
-    f = open(os.path.join(user_dir, 'tags.csv'), 'r')
-    while True:
-        r = f.readline().rstrip()
-        if len(r) == 0:
-            break
-        (old_address, tags_csv) = r.split(':')
-        old_address = strip_0x(old_address)
-        user_tags[old_address] = tags_csv.split(',')
-        logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
 
     i = 0
     j = 0
@@ -167,7 +150,6 @@ if __name__ == '__main__':
                 continue
             f.close()
             u = Person.deserialize(o)
-            logg.debug('u {}'.format(o))
 
             new_address = register_eth(i, u)
             if u.identities.get('evm') == None:
@@ -175,6 +157,8 @@ if __name__ == '__main__':
             sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
             u.identities['evm'][sub_chain_str] = [new_address]
 
+            register_ussd(u)
+
             new_address_clean = strip_0x(new_address)
             filepath = os.path.join(
                     user_new_dir,
@@ -214,29 +198,6 @@ if __name__ == '__main__':
 
             os.symlink(os.path.realpath(filepath), meta_phone_filepath)
 
-
-            # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
-            custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
-
-            filepath = os.path.join(
-                    custom_dir,
-                    'new',
-                    custom_key[:2].upper(),
-                    custom_key[2:4].upper(),
-                    custom_key.upper() + '.json',
-                    )
-            os.makedirs(os.path.dirname(filepath), exist_ok=True)
-           
-            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
-            f = open(filepath, 'w')
-            k = u.identities['evm'][sub_old_chain_str][0]
-            tag_data = {'tags': user_tags[strip_0x(k)]}
-            f.write(json.dumps(tag_data))
-            f.close()
-
-            os.symlink(os.path.realpath(filepath), custom_filepath)
-
             i += 1
             sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
         

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b2
+cic-base[full_graph]==0.1.2a77
 sarafu-faucet==0.0.2a28
-cic-eth==0.11.0b10
+cic-eth==0.11.0b6
 cic-types==0.1.0a10
-crypto-dev-signer==0.4.14b3
+crypto-dev-signer==0.4.14b2

apps/contract-migration/seed_cic_eth.sh

@@ -13,10 +13,11 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi
 DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
 CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} 
 ETH_PASSPHRASE=''
-CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
-if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
+if [ $DEV_TOKEN_TYPE = 'giftable' ]; then
-	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
+	token_symbol='GFT'
-	exit 1
+else
+	token_symbol='SRF'
 fi
 
 # Debug flag
@@ -38,11 +39,11 @@ set -a
 # get required addresses from registries
 DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
 DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
-DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL`
+DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $DEV_TOKEN_TYPE`
 cat <<EOF
 Token registry: $DEV_TOKEN_INDEX_ADDRESS
 Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
-Reserve address: $DEV_RESERVE_ADDRESS ($CIC_DEFAULT_TOKEN_SYMBOL)
+Reserve address: $DEV_RESERVE_ADDRESS ($DEV_TOKEN_TYPE)
 EOF
 
 >&2 echo "create account for gas gifter"

apps/util/liveness/liveness/linux.py

@@ -42,7 +42,6 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs
 
 
 def set(error=0, namespace=default_namespace, rundir='/run'):
-    logg.info('liveness SET error {} for namespace {}'.format(error, namespace))
     app_rundir = os.path.join(rundir, namespace)
     f = open(os.path.join(app_rundir, 'error'), 'w')
     f.write(str(error))
@@ -50,13 +49,6 @@ def set(error=0, namespace=default_namespace, rundir='/run'):
 
 
 def reset(namespace=default_namespace, rundir='/run'):
-    logg.info('liveness RESET for namespace {}'.format(namespace))
     app_rundir = os.path.join(rundir, namespace)
-    try:
+    os.unlink(os.path.join(app_rundir, 'pid'))
-        os.unlink(os.path.join(app_rundir, 'pid'))
+    os.unlink(os.path.join(app_rundir, 'error'))
-    except FileNotFoundError:
-        pass
-    try:
-        os.unlink(os.path.join(app_rundir, 'error'))
-    except FileNotFoundError:
-        pass

docker-compose.yml

@@ -104,7 +104,7 @@ services:
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
-      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
+      DEV_TOKEN_TYPE: ${DEV_TOKEN_TYPE:-GFT}
       DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
     command: ["./run_job.sh"]
@@ -240,8 +240,6 @@ services:
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
       DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       DATABASE_POOL_SIZE: 0
-      REDIS_PORT: 6379
-      REDIS_HOST: redis
       PGPASSWORD: ${DATABASE_PASSWORD:-tralala}
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996}
       BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor}
@@ -251,7 +249,6 @@ services:
       SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
       ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
       TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
-      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
     depends_on:
       - eth
       - postgres