Bump ussd cic-eth requirement

Add custom meta tags

See merge request grassrootseconomics/cic-internal-integration!114

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -32,7 +32,9 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = '::'
+    if chain_spec_dict != None:
+        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
     logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -51,7 +53,9 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = '::'
+    if chain_spec_dict != None:
+        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, flags, address=address)
     logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
@@ -127,7 +131,9 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
 
 @celery_app.task(base=CriticalSQLAlchemyTask)
 def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
+    chain_str = '::'
+    if chain_spec_dict != None:
+        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     session = SessionBase.create_session()
     r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
     if address != None:

apps/cic-eth/cic_eth/admin/token.py

@@ -0,0 +1,19 @@
+# standard imports
+import logging
+
+# external imports
+import celery
+
+# local imports
+from cic_eth.task import BaseTask
+
+celery_app = celery.current_app
+logg = logging.getLogger()
+
+
+@celery_app.task(bind=True, base=BaseTask)
+def default_token(self):
+    return {
+            'symbol': self.default_token_symbol,
+            'address': self.default_token_address,
+            }

apps/cic-eth/cic_eth/api/api_task.py

@@ -62,6 +62,18 @@ class Api:
                     )       
 
 
+    def default_token(self):
+        s_token = celery.signature(
+                'cic_eth.admin.token.default_token',
+                [],
+                queue=self.queue,
+                )
+        if self.callback_param != None:
+            s_token.link(self.callback_success)
+
+        return s_token.apply_async()
+
+
     def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
         """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
 

apps/cic-eth/cic_eth/check/db.py

@@ -1,5 +1,6 @@
 from cic_eth.db.models.base import SessionBase
 
+
 def health(*args, **kwargs):
     session = SessionBase.create_session()
     session.execute('SELECT count(*) from alembic_version')

apps/cic-eth/cic_eth/check/gas.py

@@ -0,0 +1,48 @@
+# standard imports
+import logging
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.chain import ChainSpec
+from chainlib.eth.gas import balance
+
+# local imports
+from cic_eth.db.models.role import AccountRole
+from cic_eth.db.models.base import SessionBase
+from cic_eth.db.enum import LockEnum
+from cic_eth.error import LockedError
+from cic_eth.admin.ctrl import check_lock
+
+logg = logging.getLogger().getChild(__name__)
+
+
+def health(*args, **kwargs):
+
+    session = SessionBase.create_session()
+
+    config = kwargs['config']
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
+
+    try:
+        check_lock(None, None, LockEnum.INIT)
+    except LockedError:
+        logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
+        return True
+
+    gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
+    session.close()
+
+    rpc = RPCConnection.connect(chain_spec, 'default')
+    o = balance(gas_provider)
+    r = rpc.do(o)
+    try: 
+        r = int(r, 16)
+    except TypeError:
+        r = int(r)
+    gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
+    if r < gas_min:
+        logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
+        return False
+
+    return True

apps/cic-eth/cic_eth/check/signer.py

@@ -0,0 +1,37 @@
+# standard imports
+import time
+import logging
+from urllib.error import URLError
+
+# external imports
+from chainlib.connection import RPCConnection
+from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.sign import sign_message
+from chainlib.error import JSONRPCException
+
+logg = logging.getLogger().getChild(__name__)
+
+
+def health(*args, **kwargs):
+    blocked = True
+    max_attempts = 5
+    conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
+    for i in range(max_attempts):
+        idx = i + 1
+        logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
+        try:
+            conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
+        except FileNotFoundError:
+            pass
+        except ConnectionError:
+            pass
+        except URLError:
+            pass
+        except JSONRPCException:
+            logg.debug('signer connection succeeded')
+            return True
+
+        if idx < max_attempts:
+            time.sleep(0.5)
+
+    return False

apps/cic-eth/cic_eth/db/enum.py

@@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum):
     QUEUE: Disable queueing new or modified transactions
     """
     STICKY=1
-    CREATE=2
+    INIT=2
-    SEND=4
+    CREATE=4
-    QUEUE=8
+    SEND=8
-    QUERY=16
+    QUEUE=16
+    QUERY=32
     ALL=int(0xfffffffffffffffe)
 
 

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -5,8 +5,11 @@ Revises: 1f1b3b641d08
 Create Date: 2021-04-02 18:41:20.864265
 
 """
+import datetime
 from alembic import op
 import sqlalchemy as sa
+from chainlib.eth.constant import ZERO_ADDRESS
+from cic_eth.db.enum import LockEnum
 
 
 # revision identifiers, used by Alembic.
@@ -23,10 +26,11 @@ def upgrade():
             sa.Column("address", sa.String(42), nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
-            sa.Column("date_created", sa.DateTime, nullable=False),
+            sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
             sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             )
     op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
+    op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
 
 
 def downgrade():

apps/cic-eth/cic_eth/runnable/ctrl.py

@@ -59,6 +59,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         }
 # override args
+config.dict_override(args_override, 'cli')
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@@ -67,7 +68,9 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
 
 queue = args.q
 
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+chain_spec = None
+if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 admin_api = AdminApi(None)
 
 
@@ -82,6 +85,9 @@ def lock_names_to_flag(s):
 
 # TODO: move each command to submodule
 def main():
+    chain_spec_dict = None
+    if chain_spec != None:
+        chain_spec_dict = chain_spec.asdict()
     if args.command == 'unlock':
         flags = lock_names_to_flag(args.flags)
         if not is_checksum_address(args.address):
@@ -91,7 +97,7 @@ def main():
             'cic_eth.admin.ctrl.unlock',
             [
                 None,
-                chain_spec.asdict(),
+                chain_spec_dict,
                 args.address,
                 flags,
                 ],
@@ -110,7 +116,7 @@ def main():
             'cic_eth.admin.ctrl.lock',
             [
                 None,
-                chain_spec.asdict(),
+                chain_spec_dict,
                 args.address,
                 flags,
                 ],

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -15,8 +15,10 @@ from chainlib.connection import RPCConnection
 from chainlib.eth.connection import EthUnixSignerConnection
 from chainlib.chain import ChainSpec
 from chainqueue.db.models.otx import Otx
+from cic_eth_registry.error import UnknownContractError
 import liveness.linux
 
+
 # local imports
 from cic_eth.eth import (
         erc20,
@@ -28,6 +30,7 @@ from cic_eth.eth import (
 from cic_eth.admin import (
         debug,
         ctrl,
+        token,
         )
 from cic_eth.queue import (
         query,
@@ -52,6 +55,7 @@ from cic_eth.registry import (
         connect_declarator,
         connect_token_registry,
         )
+from cic_eth.task import BaseTask
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -64,6 +68,7 @@ argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provide
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
 argparser.add_argument('-r', type=str, help='CIC registry address')
+argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
 argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
 argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
@@ -83,6 +88,7 @@ config.process()
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
         }
@@ -138,11 +144,15 @@ else:
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
-RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
+#RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
+RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
 
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
-liveness.linux.load(health_modules)
+#import cic_eth.checks.gas
+#if not cic_eth.checks.gas.health(config=config):
+#    raise RuntimeError()
+liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config)
 
 def main():
     argv = ['worker']
@@ -166,7 +176,11 @@ def main():
 
     rpc = RPCConnection.connect(chain_spec, 'default')
 
-    connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    try:
+        registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
+    except UnknownContractError as e:
+        logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
+        sys.exit(1)
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
@@ -175,8 +189,13 @@ def main():
     trusted_addresses = trusted_addresses_src.split(',')
     for address in trusted_addresses:
         logg.info('using trusted address {}'.format(address))
+
     connect_declarator(rpc, chain_spec, trusted_addresses)
     connect_token_registry(rpc, chain_spec)
+
+    BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
+    BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
+    logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
    
     liveness.linux.set()
     current_app.worker_main(argv)

apps/cic-eth/cic_eth/runnable/info.py

@@ -0,0 +1,65 @@
+#!python3
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# standard imports
+import logging
+import argparse
+import os
+
+# external imports
+import confini
+import celery
+
+# local imports
+from cic_eth.api import Api
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_format = 'terminal'
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+args = argparser.parse_args()
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.dict_override(args_override, 'cli args')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+queue = args.q
+
+api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue)
+
+def main():
+    t = api.default_token()
+    token_info = t.get()
+    print('Default token symbol: {}'.format(token_info['symbol']))
+    print('Default token address: {}'.format(token_info['address']))
+
+
+if __name__ == '__main__':
+    main()

apps/cic-eth/cic_eth/task.py

@@ -29,6 +29,8 @@ class BaseTask(celery.Task):
     call_address = ZERO_ADDRESS
     create_nonce_oracle = RPCNonceOracle
     create_gas_oracle = RPCGasOracle
+    default_token_address = None
+    default_token_symbol = None
 
     def create_session(self):
         return BaseTask.session_func()
@@ -94,3 +96,8 @@ class CriticalWeb3AndSignerTask(CriticalTask):
 def hello(self):
     time.sleep(0.1)
     return id(SessionBase.create_session)
+
+
+@celery_app.task()
+def check_health(self):
+    celery.app.control.shutdown()

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.7',
+        'beta.9',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -3,4 +3,6 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address =
-health_modules = cic_eth.k8s.db
+default_token_symbol = GFT
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
+run_dir = /run

apps/cic-eth/config/docker/cic.ini

@@ -3,3 +3,6 @@ registry_address =
 chain_spec = evm:bloxberg:8996
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
 tx_retry_delay = 20
+default_token_symbol = GFT
+health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
+run_dir = /run

apps/cic-eth/config/docker/eth.ini

@@ -1,8 +1,3 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_provider_address = 
+gas_gifter_minimum_balance = 10000000000000000000000
-#chain_id = 
-abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
-account_accounts_index_writer = 

apps/cic-eth/config/docker/signer.ini

@@ -1,5 +1,5 @@
 [signer]
-socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
+socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
 secret = deedbeef
 database_name = signer_test
 dev_keys_path = 

apps/cic-eth/config/eth.ini

@@ -1,8 +1,3 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:8545
-gas_provider_address = 
+gas_gifter_minimum_balance = 10000000000000000000000
-#chain_id = 
-abi_dir = /usr/local/share/cic/solidity/abi
-account_accounts_index_writer = 

apps/cic-eth/requirements.txt

@@ -1,6 +1,6 @@
-cic-base==0.1.2a79+build.35e442bc
+cic-base==0.1.2b1
 celery==4.4.7
-crypto-dev-signer~=0.4.14b2
+crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
 cic-eth-registry~=0.5.4a16
 #cic-bancor~=0.0.6

apps/cic-eth/setup.cfg

@@ -38,7 +38,7 @@ packages =
 	cic_eth.runnable.daemons.filters
 	cic_eth.callbacks
 	cic_eth.sync
-	cic_eth.k8s
+	cic_eth.check
 scripts =
 	./scripts/migrate.py
 
@@ -53,6 +53,7 @@ console_scripts =
 	cic-eth-create = cic_eth.runnable.create:main 
 	cic-eth-inspect = cic_eth.runnable.view:main
 	cic-eth-ctl = cic_eth.runnable.ctrl:main
+	cic-eth-info = cic_eth.runnable.info:main
 	# TODO: Merge this with ctl when subcmds sorted to submodules
 	cic-eth-tag = cic_eth.runnable.tag:main
 	cic-eth-resend = cic_eth.runnable.resend:main

apps/cic-eth/tests/conftest.py

@@ -3,8 +3,12 @@ import os
 import sys
 import logging
 
+# external imports
+from chainlib.eth.erc20 import ERC20
+
 # local imports
 from cic_eth.api import Api
+from cic_eth.task import BaseTask
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -28,3 +32,26 @@ def api(
     ):
     chain_str = str(default_chain_spec)
     return Api(chain_str, queue=None, callback_param='foo')
+
+
+@pytest.fixture(scope='function')
+def foo_token_symbol(
+    default_chain_spec,
+    foo_token,
+    eth_rpc,
+    contract_roles,
+    ):
+    
+    c = ERC20(default_chain_spec)
+    o = c.symbol(foo_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
+    r = eth_rpc.do(o)
+    return c.parse_symbol(r)
+
+
+@pytest.fixture(scope='function')
+def default_token(
+        foo_token,
+        foo_token_symbol,
+        ):
+    BaseTask.default_token_symbol = foo_token_symbol
+    BaseTask.default_token_address = foo_token   

apps/cic-eth/tests/fixtures_celery.py

@@ -34,6 +34,7 @@ def celery_includes():
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
         'cic_eth.admin.debug',
+        'cic_eth.admin.token',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',

apps/cic-eth/tests/unit/admin/test_default_token.py

@@ -0,0 +1,21 @@
+# external imports
+import celery
+
+
+def test_default_token(
+        default_token,
+        celery_session_worker,
+        foo_token,
+        foo_token_symbol,
+        ):
+      
+    s = celery.signature(
+            'cic_eth.admin.token.default_token',
+            [],
+            queue=None,
+            )
+    t = s.apply_async()
+    r = t.get()
+
+    assert r['address'] == foo_token
+    assert r['symbol'] == foo_token_symbol

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2a79
+cic_base[full_graph]~=0.1.2ab2
-cic-eth~=0.11.0b7
+cic-eth~=0.11.0b9
 cic-notify~=0.4.0a4
 cic-types~=0.1.0a10

apps/contract-migration/docker/Dockerfile

@@ -57,8 +57,8 @@ WORKDIR /home/grassroots
 USER grassroots
 
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a77
+ARG cic_base_version=0.1.2a79
-ARG cic_eth_version=0.11.0b6
+ARG cic_eth_version=0.11.0b8+build.c2286e5c
 ARG sarafu_faucet_version=0.0.2a28
 ARG sarafu_token_version==0.0.1a6
 ARG cic_contracts_version=0.0.2a2

apps/contract-migration/reset.sh

@@ -3,7 +3,7 @@
 set -a
 
 CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995}
-DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-GFT}
+CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
 DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER}
 DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000}
@@ -19,8 +19,8 @@ if [ ! -z $DEV_ETH_GAS_PRICE ]; then
 	>&2 echo using static gas price $DEV_ETH_GAS_PRICE
 fi
 
-if [[ $DEV_TOKEN_TYPE != 'GFT' && $DEV_TOKEN_TYPE != 'SRF' ]]; then
+if [[ $CIC_DEFAULT_TOKEN_SYMBOL != 'GFT' && $CIC_DEFAULT_TOKEN_SYMBOL != 'SRF' ]]; then
-	>&2 echo DEV_TOKEN_TYPE must be one of [GFT,SRF], but was $DEV_TOKEN_TYPE
+	>&2 echo CIC_DEFAULT_TOKEN_SYMBOL must be one of [GFT,SRF], but was $CIC_DEFAULT_TOKEN_SYMBOL
 	exit 1
 fi
 
@@ -60,7 +60,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 	fi
 
-	if [ $DEV_TOKEN_TYPE == 'GFT' ]; then
+	if [ $CIC_DEFAULT_TOKEN_SYMBOL == 'GFT' ]; then
 		>&2 echo "deploying 'giftable token'"
 		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w --name "Giftable Token" --symbol "GFT" --decimals 6 -vv`
 	else

apps/contract-migration/scripts/cic_eth/import_users.py

@@ -32,6 +32,7 @@ default_config_dir = '/usr/local/etc/cic'
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
 argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
 argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@@ -76,6 +77,11 @@ os.makedirs(user_new_dir)
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
+custom_dir = os.path.join(args.user_dir, 'custom')
+os.makedirs(custom_dir)
+os.makedirs(os.path.join(custom_dir, 'new'))
+os.makedirs(os.path.join(custom_dir, 'meta'))
+
 phone_dir = os.path.join(args.user_dir, 'phone')
 os.makedirs(os.path.join(phone_dir, 'meta'))
 
@@ -85,6 +91,11 @@ os.stat(user_old_dir)
 txs_dir = os.path.join(args.user_dir, 'txs')
 os.makedirs(txs_dir)
 
+user_dir = args.user_dir
+
+old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
+old_chain_str = str(old_chain_spec)
+
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 
@@ -133,7 +144,17 @@ def register_eth(i, u):
 
 if __name__ == '__main__':
 
-    #fi = open(os.path.join(user_out_dir, 'addresses.csv'), 'a')
+    user_tags = {}
+    f = open(os.path.join(user_dir, 'tags.csv'), 'r')
+    while True:
+        r = f.readline().rstrip()
+        if len(r) == 0:
+            break
+        (old_address, tags_csv) = r.split(':')
+        old_address = strip_0x(old_address)
+        user_tags[old_address] = tags_csv.split(',')
+        logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
+
 
     i = 0
     j = 0
@@ -198,6 +219,29 @@ if __name__ == '__main__':
             os.symlink(os.path.realpath(filepath), meta_phone_filepath)
 
 
+            # custom data
+            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
+
+            filepath = os.path.join(
+                    custom_dir,
+                    'new',
+                    custom_key[:2].upper(),
+                    custom_key[2:4].upper(),
+                    custom_key.upper() + '.json',
+                    )
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+           
+            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
+            f = open(filepath, 'w')
+            k = u.identities['evm'][sub_old_chain_str][0]
+            tag_data = {'tags': user_tags[strip_0x(k)]}
+            f.write(json.dumps(tag_data))
+            f.close()
+
+            os.symlink(os.path.realpath(filepath), custom_filepath)
+
+
             i += 1
             sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
         

apps/contract-migration/scripts/cic_meta/import_meta_custom.js

@@ -0,0 +1,139 @@
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+
+const cic = require('cic-client-meta');
+const vcfp = require('vcard-parser');
+
+//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
+
+const config = new cic.Config('./config'); 
+config.process();
+console.log(config);
+
+
+function sendit(uid, envelope) {
+	const d = envelope.toJSON();
+
+	const contentLength = (new TextEncoder().encode(d)).length;
+	const opts = {
+		method: 'PUT',
+		headers: {
+			'Content-Type': 'application/json',
+			'Content-Length': contentLength,
+			'X-CIC-AUTOMERGE': 'client',
+
+		},
+	};
+	let url = config.get('META_URL');
+	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
+	console.log('posting to url: ' + url + uid);
+	const req = http.request(url + uid, opts, (res) => {
+		res.on('data', process.stdout.write);
+		res.on('end', () => {
+			console.log('result', res.statusCode, res.headers);
+		});
+	});
+	if (!req.write(d)) {
+		console.error('foo', d);
+		process.exit(1);
+	}
+	req.end();
+}
+
+function doOne(keystore, filePath, identifier) {
+	const signer = new cic.PGPSigner(keystore);
+
+	const o = JSON.parse(fs.readFileSync(filePath).toString());
+	//const b = Buffer.from(j['vcard'], 'base64');
+	//const s = b.toString();
+	//const o = vcfp.parse(s);
+	//const phone = o.tel[0].value;
+
+	//cic.Phone.toKey(phone).then((uid) => {
+	//const o = fs.readFileSync(filePath, 'utf-8');
+
+	const s = new cic.Syncable(identifier, o);
+	s.setSigner(signer);
+	s.onwrap = (env) => {
+		sendit(identifier, env);
+	};
+	s.sign();
+	//});
+}
+
+const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+pk = fs.readFileSync(privateKeyPath);
+pubk = fs.readFileSync(publicKeyPath);
+
+new cic.PGPKeyStore(
+	config.get('PGP_PASSPHRASE'),
+	pk,
+	pubk,
+	undefined,
+	undefined,
+	importMetaCustom,
+);
+
+const batchSize = 16;
+const batchDelay = 1000;
+const total = parseInt(process.argv[3]);
+const dataDir = process.argv[2];
+const workDir = path.join(dataDir, 'custom/meta');
+const userDir = path.join(dataDir, 'custom/new');
+let count = 0;
+let batchCount = 0;
+
+
+function importMetaCustom(keystore) {
+	let err;
+	let files;
+
+	try {
+		err, files = fs.readdirSync(workDir);
+	} catch {
+		console.error('source directory not yet ready', workDir);
+		setTimeout(importMetaPhone, batchDelay, keystore);
+		return;
+	}
+	let limit = batchSize;
+	if (files.length < limit) {
+		limit = files.length;
+	}
+	for (let i = 0; i < limit; i++) {
+		const file = files[i];
+		if (file.length < 3) {
+			console.debug('skipping file', file);	
+			continue;
+		}
+		//const identifier = file.substr(0,file.length-5);
+		const identifier = file;
+		const filePath = path.join(workDir, file);
+		console.log(filePath);
+
+		//const address = fs.readFileSync(filePath).toString().substring(2).toUpperCase();
+		const custom = JSON.parse(fs.readFileSync(filePath).toString());
+		const customFilePath = path.join(
+			userDir,
+			identifier.substring(0, 2),
+			identifier.substring(2, 4),
+			identifier + '.json',
+		);
+
+		doOne(keystore, filePath, identifier);
+		fs.unlinkSync(filePath);
+		count++;
+		batchCount++;
+		if (batchCount == batchSize) {
+			console.debug('reached batch size, breathing');
+			batchCount=0;
+			setTimeout(importMeta, batchDelay, keystore);
+			return;
+		}
+	}
+	if (count == total) {
+		return;
+	}
+	setTimeout(importMetaCustom, 100, keystore);
+}

apps/contract-migration/scripts/create_import_users.py

@@ -37,6 +37,7 @@ config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
 
 argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
+argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record')
 argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)')
 argparser.add_argument('-v', action='store_true', help='Be verbose')
 argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@@ -81,6 +82,10 @@ phone_idx = []
 user_dir = args.dir
 user_count = args.user_count
 
+tags = args.tag
+if tags == None or len(tags) == 0:
+    tags = ['individual']
+
 random.seed()
 
 def genPhoneIndex(phone):
@@ -189,6 +194,7 @@ if __name__ == '__main__':
     os.makedirs(base_dir, exist_ok=True)
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
+    ft = open(os.path.join(user_dir, 'tags.csv'), 'w')
 
     i = 0
     while i < user_count:
@@ -215,10 +221,12 @@ if __name__ == '__main__':
         f.write(eth)
         f.close()
 
+        ft.write('{}:{}\n'.format(eth, ','.join(tags)))
         amount = genAmount()
         fa.write('{},{}\n'.format(eth,amount))
         logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone))
         
         i += 1
 
+    ft.close()
     fa.close()

apps/contract-migration/scripts/eth/import_users.py

@@ -38,6 +38,7 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
 argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
 argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
 argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
 argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
 argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
@@ -68,6 +69,11 @@ os.makedirs(user_new_dir)
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
+custom_dir = os.path.join(args.user_dir, 'custom')
+os.makedirs(custom_dir)
+os.makedirs(os.path.join(custom_dir, 'new'))
+os.makedirs(os.path.join(custom_dir, 'meta'))
+
 phone_dir = os.path.join(args.user_dir, 'phone')
 os.makedirs(os.path.join(phone_dir, 'meta'))
 
@@ -77,9 +83,14 @@ os.stat(user_old_dir)
 txs_dir = os.path.join(args.user_dir, 'txs')
 os.makedirs(txs_dir)
 
+user_dir = args.user_dir
+
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 
+old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
+old_chain_str = str(old_chain_spec)
+
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
@@ -127,12 +138,18 @@ def register_eth(i, u):
     return address
    
 
-def register_ussd(u):
-    pass
-
-
 if __name__ == '__main__':
 
+    user_tags = {}
+    f = open(os.path.join(user_dir, 'tags.csv'), 'r')
+    while True:
+        r = f.readline().rstrip()
+        if len(r) == 0:
+            break
+        (old_address, tags_csv) = r.split(':')
+        old_address = strip_0x(old_address)
+        user_tags[old_address] = tags_csv.split(',')
+        logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
 
     i = 0
     j = 0
@@ -150,6 +167,7 @@ if __name__ == '__main__':
                 continue
             f.close()
             u = Person.deserialize(o)
+            logg.debug('u {}'.format(o))
 
             new_address = register_eth(i, u)
             if u.identities.get('evm') == None:
@@ -157,8 +175,6 @@ if __name__ == '__main__':
             sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
             u.identities['evm'][sub_chain_str] = [new_address]
 
-            register_ussd(u)
-
             new_address_clean = strip_0x(new_address)
             filepath = os.path.join(
                     user_new_dir,
@@ -198,6 +214,29 @@ if __name__ == '__main__':
 
             os.symlink(os.path.realpath(filepath), meta_phone_filepath)
 
+
+            # custom data
+            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
+
+            filepath = os.path.join(
+                    custom_dir,
+                    'new',
+                    custom_key[:2].upper(),
+                    custom_key[2:4].upper(),
+                    custom_key.upper() + '.json',
+                    )
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+           
+            sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
+            f = open(filepath, 'w')
+            k = u.identities['evm'][sub_old_chain_str][0]
+            tag_data = {'tags': user_tags[strip_0x(k)]}
+            f.write(json.dumps(tag_data))
+            f.close()
+
+            os.symlink(os.path.realpath(filepath), custom_filepath)
+
             i += 1
             sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
         

apps/contract-migration/seed_cic_eth.sh

@@ -13,6 +13,12 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi
 DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
 CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} 
 ETH_PASSPHRASE=''
+DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
+if [ $DEV_TOKEN_TYPE = 'giftable' ]; then
+	token_symbol='GFT'
+else
+	token_symbol='SRF'
+fi
 
 # Debug flag
 DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@@ -33,11 +39,11 @@ set -a
 # get required addresses from registries
 DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
 DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
-DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $DEV_TOKEN_TYPE`
+DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL`
 cat <<EOF
 Token registry: $DEV_TOKEN_INDEX_ADDRESS
 Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
-Reserve address: $DEV_RESERVE_ADDRESS ($DEV_TOKEN_TYPE)
+Reserve address: $DEV_RESERVE_ADDRESS ($CIC_DEFAULT_TOKEN_SYMBOL)
 EOF
 
 >&2 echo "create account for gas gifter"
@@ -46,6 +52,7 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H
 echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
 cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
 
+
 >&2 echo "create account for sarafu gifter"
 DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
 echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
@@ -97,5 +104,10 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 #echo -n 0 > $init_level_file
 
+# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
+cic-eth-ctl -i :: unlock INIT
+cic-eth-ctl -i :: unlock SEND
+cic-eth-ctl -i :: unlock QUEUE
+
 set +a
 set +e

apps/util/liveness/liveness/linux.py

@@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs
         checks.append(module)
 
     for check in checks:
-        r = check.health(args, kwargs)
+        r = check.health(*args, **kwargs)
         if r == False:
             raise RuntimeError('liveness check {} failed'.format(str(check)))
         logg.info('liveness check passed: {}'.format(str(check)))

apps/util/liveness/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 setup(
         name='liveness',
-        version='0.0.1a6',
+        version='0.0.1a7',
         packages=['liveness'],
         include_package_data=True,
         )

apps/util/liveness/tests/test_imports.py

@@ -116,7 +116,9 @@ class TestImports(unittest.TestCase):
 
     def test_args(self):
         checks = ['tests.imports.import_args']
-        liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42})
+        aargs=['foo']
+        kwaargs={'bar': 42}
+        liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs)
         f = open(self.pid_path, 'r')
         r = f.read()
         f.close()

docker-compose.yml

@@ -104,7 +104,7 @@ services:
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
       RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data
       DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0}
-      DEV_TOKEN_TYPE: ${DEV_TOKEN_TYPE:-GFT}
+      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
       DEV_SARAFU_DEMURRAGE_LEVEL: ${DEV_SARAFU_DEMURRAGE_LEVEL:-196454828847045000000000000000000}
       DEV_ETH_GAS_PRICE: ${DEV_ETH_GAS_PRICE:-1}
     command: ["./run_job.sh"]
@@ -249,6 +249,7 @@ services:
       SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef}
       ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA}
       TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1}
+      CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT}
     depends_on:
       - eth
       - postgres