move files out of scripts folder to their own dir

apps/data-seeding/cic_eth/traffic/common/__init__.py

@@ -0,0 +1,8 @@
+from . import (
+    log,
+    argparse,
+    config,
+    signer,
+    rpc,
+    registry,
+    )

apps/data-seeding/cic_eth/traffic/common/argparse.py

@@ -0,0 +1,73 @@
+# standard imports
+import logging
+import argparse
+import os
+import sys
+
+default_config_dir = os.environ.get('CONFINI_DIR')
+full_template = { 
+        # (long arg and key name, short var, type, default, help,)
+        'provider': ('p', str, None, 'RPC provider url',),
+        'registry_address': ('r', str, None, 'CIC registry address',),
+        'keystore_file': ('y', str, None, 'Keystore file',),
+        'config_dir': ('c', str, default_config_dir, 'Configuration directory',),
+        'queue': ('q', str, 'cic-eth', 'Celery task queue',),
+        'chain_spec': ('i', str, None, 'Chain spec string',),
+        'abi_dir': (None, str, None, 'Smart contract ABI search path',),
+        'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
+        }
+default_include_args = [
+    'config_dir',
+    'provider',
+    'env_prefix',
+    ]
+
+sub = None
+
+def create(caller_dir, include_args=default_include_args):
+
+    argparser = argparse.ArgumentParser()
+
+    for k in include_args:
+        a = full_template[k]
+        long_flag = '--' + k.replace('_', '-')
+        short_flag = None
+        dest = None
+        if a[0] != None:
+            short_flag = '-' + a[0]
+            dest = a[0]
+        else:
+            dest = k
+        default = a[2]
+        if default == None and k == 'config_dir':
+            default = os.path.join(caller_dir, 'config')
+        
+        if short_flag == None:
+            argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
+        else:
+            argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
+
+    argparser.add_argument('-v', action='store_true', help='Be verbose')
+    argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+
+    return argparser
+
+
+def add(argparser, processor, name, description=None):
+    processor(argparser)
+
+    return argparser
+
+
+def parse(argparser, logger=None):
+
+    args = argparser.parse_args(sys.argv[1:])
+
+    # handle logging input
+    if logger != None:
+        if args.vv:
+            logger.setLevel(logging.DEBUG)
+        elif args.v:
+            logger.setLevel(logging.INFO)
+
+    return args

apps/data-seeding/cic_eth/traffic/common/config.py

@@ -0,0 +1,39 @@
+# external imports
+import logging
+import confini
+
+logg = logging.getLogger(__name__)
+
+default_arg_overrides = {
+    'abi_dir': 'ETH_ABI_DIR',
+    'p': 'ETH_PROVIDER',
+    'i': 'CIC_CHAIN_SPEC',
+    'r': 'CIC_REGISTRY_ADDRESS',
+    }
+
+
+def override(config, override_dict, label):
+    config.dict_override(override_dict, label)
+    config.validate()
+    return config
+
+
+def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
+    # handle config input
+    config = confini.Config(config_dir, env_prefix)
+    config.process()
+    if arg_overrides != None and args != None:
+        override_dict = {}
+        for k in arg_overrides:
+            v = getattr(args, k)
+            if v != None:
+                override_dict[arg_overrides[k]] = v
+        config = override(config, override_dict, 'args')
+    else:
+        config.validate()
+
+    return config
+
+
+def log(config):
+    logg.debug('config loaded:\n{}'.format(config))

apps/data-seeding/cic_eth/traffic/common/log.py

@@ -0,0 +1,18 @@
+# standard imports
+import logging
+
+logging.basicConfig(level=logging.WARNING)
+
+default_mutelist = [
+        'urllib3',
+        'websockets.protocol',
+        'web3.RequestManager',
+        'web3.providers.WebsocketProvider',
+        'web3.providers.HTTPProvider',
+        ]
+
+def create(name=None, mutelist=default_mutelist):
+    logg = logging.getLogger(name)
+    for m in mutelist:
+        logging.getLogger(m).setLevel(logging.CRITICAL)
+    return logg

apps/data-seeding/cic_eth/traffic/common/registry.py

@@ -0,0 +1,86 @@
+# standard imports
+import logging
+import copy
+
+# external imports
+from cic_registry import CICRegistry
+from eth_token_index import TokenUniqueSymbolIndex
+from eth_accounts_index import AccountRegistry
+from chainlib.chain import ChainSpec
+from cic_registry.chain import ChainRegistry
+from cic_registry.helper.declarator import DeclaratorOracleAdapter
+
+logg = logging.getLogger(__name__)
+
+
+class TokenOracle:
+
+    def __init__(self, conn, chain_spec, registry):
+        self.tokens = []
+        self.chain_spec = chain_spec
+        self.registry = registry
+
+        token_registry_contract = CICRegistry.get_contract(chain_spec, 'TokenRegistry', 'Registry')
+        self.getter = TokenUniqueSymbolIndex(conn, token_registry_contract.address())
+
+
+    def get_tokens(self):
+        token_count = self.getter.count()
+        if token_count == len(self.tokens):
+            return self.tokens
+
+        for i in range(len(self.tokens), token_count):
+            token_address = self.getter.get_index(i)
+            t = self.registry.get_address(self.chain_spec, token_address)
+            token_symbol = t.symbol()
+            self.tokens.append(t)
+
+            logg.debug('adding token idx {} symbol {} address {}'.format(i, token_symbol, token_address))
+
+        return copy.copy(self.tokens)
+
+
+class AccountsOracle:
+
+    def __init__(self, conn, chain_spec, registry):
+        self.accounts = []
+        self.chain_spec = chain_spec
+        self.registry = registry
+
+        accounts_registry_contract = CICRegistry.get_contract(chain_spec, 'AccountRegistry', 'Registry')
+        self.getter = AccountRegistry(conn, accounts_registry_contract.address())
+
+
+    def get_accounts(self):
+        accounts_count = self.getter.count()
+        if accounts_count == len(self.accounts):
+            return self.accounts
+
+        for i in range(len(self.accounts), accounts_count):
+            account = self.getter.get_index(i)
+            self.accounts.append(account)
+            logg.debug('adding account {}'.format(account))
+
+        return copy.copy(self.accounts)
+
+
+def init_legacy(config, w3):
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    CICRegistry.init(w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
+    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
+
+    chain_registry = ChainRegistry(chain_spec)
+    CICRegistry.add_chain_registry(chain_registry, True)
+
+    declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
+    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
+    if trusted_addresses_src == None:
+        raise ValueError('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
+    trusted_addresses = trusted_addresses_src.split(',')
+    for address in trusted_addresses:
+        logg.info('using trusted address {}'.format(address))
+
+    oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
+    chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
+
+    return CICRegistry

apps/data-seeding/cic_eth/traffic/common/rpc.py

@@ -0,0 +1,18 @@
+# standard imports
+import re
+
+# external imports
+import web3
+
+def create(url):
+    # web3 input
+    # TODO: Replace with chainlib
+    re_websocket = r'^wss?:'
+    re_http = r'^https?:'
+    blockchain_provider = None
+    if re.match(re_websocket, url):
+        blockchain_provider = web3.Web3.WebsocketProvider(url)
+    elif re.match(re_http, url):
+        blockchain_provider = web3.Web3.HTTPProvider(url)
+    w3 = web3.Web3(blockchain_provider)
+    return w3

apps/data-seeding/cic_eth/traffic/common/signer.py

@@ -0,0 +1,23 @@
+# standard imports
+import logging
+
+# external imports 
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+
+logg = logging.getLogger(__name__)
+
+keystore = DictKeystore()
+
+def from_keystore(keyfile):
+    global keystore
+
+    # signer
+    if keyfile == None:
+        raise ValueError('please specify signer keystore file')
+
+    logg.debug('loading keystore file {}'.format(keyfile))
+    address = keystore.import_keystore_file(keyfile)
+
+    signer = EIP155Signer(keystore)
+    return (address, signer,)