cic-stack/apps/cic-eth/cic_eth/runnable/daemons/tasker.py

# standard imports
import sys
import os
import logging
import argparse
import tempfile
import re
import urllib
import websocket

# third-party imports
import celery
import confini
from crypto_dev_signer.eth.web3ext import Web3 as Web3Ext
from web3 import HTTPProvider, WebsocketProvider
from gas_proxy.web3 import GasMiddleware

# local imports
from cic_registry.registry import CICRegistry
from cic_registry.registry import ChainRegistry
from cic_registry.registry import ChainSpec
from cic_registry.helper.declarator import DeclaratorOracleAdapter

from cic_bancor.bancor import BancorRegistryClient
from cic_eth.eth import bancor
from cic_eth.eth import token
from cic_eth.eth import tx
from cic_eth.eth import account
from cic_eth.eth import request
from cic_eth.admin import debug
from cic_eth.admin import ctrl
from cic_eth.eth.rpc import RpcClient
from cic_eth.eth.rpc import GasOracle
from cic_eth.queue import tx
from cic_eth.callbacks import Callback
from cic_eth.callbacks import http
from cic_eth.callbacks import tcp
from cic_eth.callbacks import redis
from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.otx import Otx
from cic_eth.db import dsn_from_config

logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()

config_dir = os.path.join('/usr/local/etc/cic-eth')

argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', type=str, help='web3 provider')
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
args = argparser.parse_args()

if args.vv:
    logging.getLogger().setLevel(logging.DEBUG)
elif args.v:
    logging.getLogger().setLevel(logging.INFO)

config = confini.Config(args.c, args.env_prefix)
config.process()
# override args
args_override = {
        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
        'CIC_CHAIN_SPEC': getattr(args, 'i'),
        'ETH_PROVIDER': getattr(args, 'p'),
        'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
        }
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(args.c, config))

# connect to database
dsn = dsn_from_config(config)
SessionBase.connect(dsn, pool_size=8, debug=config.true('DATABASE_DEBUG'))

# verify database connection with minimal sanity query
session = SessionBase.create_session()
session.execute('select version_num from alembic_version')
session.close()

# set up celery
current_app = celery.Celery(__name__)

broker = config.get('CELERY_BROKER_URL')
if broker[:4] == 'file':
    bq = tempfile.mkdtemp()
    bp = tempfile.mkdtemp()
    current_app.conf.update({
            'broker_url': broker,
            'broker_transport_options': {
                'data_folder_in': bq,
                'data_folder_out': bq,
                'data_folder_processed': bp,
            },
            },
            )
    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
else:
    current_app.conf.update({
        'broker_url': broker,
        })

result = config.get('CELERY_RESULT_URL')
if result[:4] == 'file':
    rq = tempfile.mkdtemp()
    current_app.conf.update({
        'result_backend': 'file://{}'.format(rq),
        })
    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
else:
    current_app.conf.update({
        'result_backend': result,
        })


# set up web3
# TODO: web3 socket wrapping is now a lot of code. factor out
class JSONRPCHttpSocketAdapter:

    def __init__(self, url):
        self.response = None
        self.url = url

    def send(self, data):
        logg.debug('redirecting socket send to jsonrpc http socket adapter {} {}'.format(self.url, data))
        req = urllib.request.Request(self.url, method='POST')
        req.add_header('Content-type', 'application/json')
        req.add_header('Connection', 'close')
        res = urllib.request.urlopen(req, data=data.encode('utf-8'))
        self.response = res.read().decode('utf-8')
        logg.debug('setting jsonrpc http socket adapter response to {}'.format(self.response))

    def recv(self, n=0):
        return self.response


re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
socket_constructor = None
if re.match(re_websocket, blockchain_provider) != None:
    def socket_constructor_ws():
        return websocket.create_connection(config.get('ETH_PROVIDER'))
    socket_constructor = socket_constructor_ws
    blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
    def socket_constructor_http():
        return JSONRPCHttpSocketAdapter(config.get('ETH_PROVIDER'))
    socket_constructor = socket_constructor_http
    blockchain_provider = HTTPProvider(blockchain_provider)
else:
    raise ValueError('unknown provider url {}'.format(blockchain_provider))


def web3ext_constructor():
    w3 = Web3Ext(blockchain_provider, config.get('SIGNER_SOCKET_PATH'))
    GasMiddleware.socket_constructor = socket_constructor
    w3.middleware_onion.add(GasMiddleware)

    def sign_transaction(tx):
        r = w3.eth.signTransaction(tx)
        d = r.__dict__
        for k in d.keys():
            if k == 'tx':
                d[k] = d[k].__dict__ 
            else:
                d[k] = d[k].hex()
        return d

    setattr(w3.eth, 'sign_transaction', sign_transaction)
    setattr(w3.eth, 'send_raw_transaction', w3.eth.sendRawTransaction)
    return (blockchain_provider, w3)
RpcClient.set_constructor(web3ext_constructor) 

Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')


def main():
    argv = ['worker']
    if args.vv:
        argv.append('--loglevel=DEBUG')
    elif args.v:
        argv.append('--loglevel=INFO')
    argv.append('-Q')
    argv.append(args.q)
    argv.append('-n')
    argv.append(args.q)

    if config.true('SSL_ENABLE_CLIENT'):
        Callback.ssl = True
        Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
        Callback.ssl_key_file = config.get('SSL_KEY_FILE')
        Callback.ssl_password = config.get('SSL_PASSWORD')

    if config.get('SSL_CA_FILE') != '':
        Callback.ssl_ca_file = config.get('SSL_CA_FILE')

    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))

    c = RpcClient(chain_spec)
    CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
    CICRegistry.add_path(config.get('ETH_ABI_DIR'))

    chain_registry = ChainRegistry(chain_spec)
    CICRegistry.add_chain_registry(chain_registry, True)
    try:
        CICRegistry.get_contract(chain_spec, 'CICRegistry')
    except Exception as e:
        logg.exception('Eek, registry failure is baaad juju {}'.format(e))
        sys.exit(1)

    if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
        CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)

    declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
    if trusted_addresses_src == None:
        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
        sys.exit(1)
    trusted_addresses = trusted_addresses_src.split(',')
    for address in trusted_addresses:
        logg.info('using trusted address {}'.format(address))
    oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
    chain_registry.add_oracle('naive_erc20_oracle', oracle)


    #chain_spec = CICRegistry.default_chain_spec
    #bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
    #bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
    #bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
    #bancor_registry.load(True)
    current_app.worker_main(argv)


if __name__ == '__main__':
    main()