# standard imports import sys import os import logging import argparse import tempfile import re import urllib import websocket # third-party imports import celery import confini from crypto_dev_signer.eth.web3ext import Web3 as Web3Ext from web3 import HTTPProvider, WebsocketProvider from gas_proxy.web3 import GasMiddleware # local imports from cic_registry.registry import CICRegistry from cic_registry.registry import ChainRegistry from cic_registry.registry import ChainSpec from cic_registry.helper.declarator import DeclaratorOracleAdapter from cic_bancor.bancor import BancorRegistryClient from cic_eth.eth import bancor from cic_eth.eth import token from cic_eth.eth import tx from cic_eth.eth import account from cic_eth.eth import request from cic_eth.admin import debug from cic_eth.admin import ctrl from cic_eth.eth.rpc import RpcClient from cic_eth.eth.rpc import GasOracle from cic_eth.queue import tx from cic_eth.callbacks import Callback from cic_eth.callbacks import http from cic_eth.callbacks import tcp from cic_eth.callbacks import redis from cic_eth.db.models.base import SessionBase from cic_eth.db.models.otx import Otx from cic_eth.db import dsn_from_config logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() config_dir = os.path.join('/usr/local/etc/cic-eth') argparser = argparse.ArgumentParser() argparser.add_argument('-p', '--provider', dest='p', type=str, help='web3 provider') argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks') argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi') argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() if args.vv: logging.getLogger().setLevel(logging.DEBUG) elif args.v: logging.getLogger().setLevel(logging.INFO) config = confini.Config(args.c, args.env_prefix) config.process() # override args args_override = { 'ETH_ABI_DIR': getattr(args, 'abi_dir'), 'CIC_CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'), } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug('config loaded from {}:\n{}'.format(args.c, config)) # connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn) # verify database connection with minimal sanity query session = SessionBase.create_session() session.execute('select version_num from alembic_version') session.close() # set up celery current_app = celery.Celery(__name__) broker = config.get('CELERY_BROKER_URL') if broker[:4] == 'file': bq = tempfile.mkdtemp() bp = tempfile.mkdtemp() current_app.conf.update({ 'broker_url': broker, 'broker_transport_options': { 'data_folder_in': bq, 'data_folder_out': bq, 'data_folder_processed': bp, }, }, ) logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) else: current_app.conf.update({ 'broker_url': broker, }) result = config.get('CELERY_RESULT_URL') if result[:4] == 'file': rq = tempfile.mkdtemp() current_app.conf.update({ 'result_backend': 'file://{}'.format(rq), }) logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) else: current_app.conf.update({ 'result_backend': result, }) # set up web3 # TODO: web3 socket wrapping is now a lot of code. factor out class JSONRPCHttpSocketAdapter: def __init__(self, url): self.response = None self.url = url def send(self, data): logg.debug('redirecting socket send to jsonrpc http socket adapter {} {}'.format(self.url, data)) req = urllib.request.Request(self.url, method='POST') req.add_header('Content-type', 'application/json') req.add_header('Connection', 'close') res = urllib.request.urlopen(req, data=data.encode('utf-8')) self.response = res.read().decode('utf-8') logg.debug('setting jsonrpc http socket adapter response to {}'.format(self.response)) def recv(self, n=0): return self.response re_websocket = re.compile('^wss?://') re_http = re.compile('^https?://') blockchain_provider = config.get('ETH_PROVIDER') socket_constructor = None if re.match(re_websocket, blockchain_provider) != None: def socket_constructor_ws(): return websocket.create_connection(config.get('ETH_PROVIDER')) socket_constructor = socket_constructor_ws blockchain_provider = WebsocketProvider(blockchain_provider) elif re.match(re_http, blockchain_provider) != None: def socket_constructor_http(): return JSONRPCHttpSocketAdapter(config.get('ETH_PROVIDER')) socket_constructor = socket_constructor_http blockchain_provider = HTTPProvider(blockchain_provider) else: raise ValueError('unknown provider url {}'.format(blockchain_provider)) def web3ext_constructor(): w3 = Web3Ext(blockchain_provider, config.get('SIGNER_SOCKET_PATH')) GasMiddleware.socket_constructor = socket_constructor w3.middleware_onion.add(GasMiddleware) def sign_transaction(tx): r = w3.eth.signTransaction(tx) d = r.__dict__ for k in d.keys(): if k == 'tx': d[k] = d[k].__dict__ else: d[k] = d[k].hex() return d setattr(w3.eth, 'sign_transaction', sign_transaction) setattr(w3.eth, 'send_raw_transaction', w3.eth.sendRawTransaction) return (blockchain_provider, w3) RpcClient.set_constructor(web3ext_constructor) logg.info('ccc {}'.format(config.store['TASKS_TRACE_QUEUE_STATUS'])) Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') def main(): argv = ['worker'] if args.vv: argv.append('--loglevel=DEBUG') elif args.v: argv.append('--loglevel=INFO') argv.append('-Q') argv.append(args.q) argv.append('-n') argv.append(args.q) if config.true('SSL_ENABLE_CLIENT'): Callback.ssl = True Callback.ssl_cert_file = config.get('SSL_CERT_FILE') Callback.ssl_key_file = config.get('SSL_KEY_FILE') Callback.ssl_password = config.get('SSL_PASSWORD') if config.get('SSL_CA_FILE') != '': Callback.ssl_ca_file = config.get('SSL_CA_FILE') chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) c = RpcClient(chain_spec) CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec) CICRegistry.add_path(config.get('ETH_ABI_DIR')) chain_registry = ChainRegistry(chain_spec) CICRegistry.add_chain_registry(chain_registry, True) if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None: CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True) if config.get('CIC_DECLARATOR_ADDRESS') != None: abi_path = os.path.join(config.get('ETH_ABI_DIR'), '{}.json'.format(interface)) f = open(abi_path) abi = json.load(abi_path) f.close() c = w3.eth.contract(abi=abi, address=address) trusted_addresses = config.get('CIC_TRUSTED_ADDRESSES', []).split(',') oracle = DeclaratorOracleAdapter(contract, trusted_addresses) chain_registry.add_oracle(oracle) #chain_spec = CICRegistry.default_chain_spec #bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry') #bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec) #bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR')) #bancor_registry.load(True) current_app.worker_main(argv) if __name__ == '__main__': main()