kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

WIP refactor cic-cache to use chainlib

Browse Source
This commit is contained in:
nolash 2021-03-31 16:49:58 +02:00
parent 3f8d7fc10a
commit 3c7afc53da
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
2 changed files with 137 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

259
apps/cic-cache/cic_cache/runnable/tracker.py
View File

@ -9,23 +9,34 @@ import re
# third-party imports # third-party imports
import confini import confini
from cic_registry import CICRegistry from cic_eth_registry import CICRegistry
from cic_registry.chain import ( from chainlib.chain import ChainSpec
ChainRegistry, from chainlib.eth.tx import (
ChainSpec, transaction,
transaction_by_block,
) )
#from cic_registry.bancor import BancorRegistryClient from chainlib.eth.block import (
from cic_registry.token import Token transaction_count,
from cic_registry.error import ( block_by_number,
block_by_hash,
)
from chainlib.eth.contract import (
code,
)
from chainlib.connection import RPCConnection
#from cic_registry.token import Token
from cic_eth_registry.error import (
UnknownContractError, UnknownContractError,
UnknownDeclarationError, # UnknownDeclarationError,
) )
from cic_registry.declaration import to_token_declaration from hexathon import (
from web3.exceptions import BlockNotFound, TransactionNotFound strip_0x,
from websockets.exceptions import ConnectionClosedError add_0x,
from requests.exceptions import ConnectionError )
import web3 #from cic_registry.declaration import to_token_declaration
from web3 import HTTPProvider, WebsocketProvider #from web3.exceptions import BlockNotFound, TransactionNotFound
#from websockets.exceptions import ConnectionClosedError
#from requests.exceptions import ConnectionError
# local imports # local imports
from cic_cache import db from cic_cache import db
@ -33,11 +44,6 @@ from cic_cache.db.models.base import SessionBase
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
log_topics = { log_topics = {
'transfer': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef', 'transfer': '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef',
@ -48,11 +54,12 @@ log_topics = {
config_dir = os.path.join('/usr/local/etc/cic-cache') config_dir = os.path.join('/usr/local/etc/cic-cache')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--trust-address', default=[], type=str, dest='trust_address', action='append', help='Set address as trust') argparser.add_argument('--trust-address', default=[], type=str, dest='trust_address', action='append', help='Set address as trust')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
argparser.add_argument('-v', help='be verbose', action='store_true') argparser.add_argument('-v', help='be verbose', action='store_true')
argparser.add_argument('-vv', help='be more verbose', action='store_true') argparser.add_argument('-vv', help='be more verbose', action='store_true')
args = argparser.parse_args(sys.argv[1:]) args = argparser.parse_args(sys.argv[1:])
@ -60,7 +67,6 @@ args = argparser.parse_args(sys.argv[1:])
config_dir = os.path.join(args.c) config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True) os.makedirs(config_dir, 0o777, True)
if args.v == True: if args.v == True:
logging.getLogger().setLevel(logging.INFO) logging.getLogger().setLevel(logging.INFO)
elif args.vv == True: elif args.vv == True:
@ -69,7 +75,9 @@ elif args.vv == True:
config = confini.Config(config_dir, args.env_prefix) config = confini.Config(config_dir, args.env_prefix)
config.process() config.process()
args_override = { args_override = {
'ETH_ABI_DIR': getattr(args, 'abi_dir'), 'ETH_PROVIDER': getattr(args, 'p'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'CIC_TRUST_ADDRESS': ",".join(getattr(args, 'trust_address', [])), 'CIC_TRUST_ADDRESS': ",".join(getattr(args, 'trust_address', [])),
} }
config.dict_override(args_override, 'cli flag') config.dict_override(args_override, 'cli flag')
@ -79,37 +87,34 @@ logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
# connect to database # connect to database
dsn = db.dsn_from_config(config) dsn = db.dsn_from_config(config)
SessionBase.connect(dsn) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
registry_address = config.get('CIC_REGISTRY_ADDRESS')
re_websocket = re.compile('^wss?://') def rpc_connect():
re_http = re.compile('^https?://') rpc = RPCConnection.connect(chain_spec, 'default')
blockchain_provider = config.get('ETH_PROVIDER') return rpc
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
rpc = rpc_connect()
CICRegistry.init(registry_address, chain_spec, rpc)
class RunStateEnum(enum.IntEnum): class RunStateEnum(enum.IntEnum):
INIT = 0 INIT = 0
RUN = 1 RUN = 1
TERMINATE = 9 TERMINATE = 9
def rubberstamp(src): def rubberstamp(src):
return True return True
class Tracker: class Tracker:
def __init__(self, chain_spec, trusts=[]): def __init__(self, chain_spec, rpc, trusts=[]):
self.block_height = 0 self.block_height = 0
self.tx_height = 0 self.tx_height = 0
self.state = RunStateEnum.INIT self.state = RunStateEnum.INIT
@ -117,7 +122,9 @@ class Tracker:
self.convert_enabled = False self.convert_enabled = False
self.trusts = trusts self.trusts = trusts
self.chain_spec = chain_spec self.chain_spec = chain_spec
self.declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', 'Declarator')
registry = CICRegistry(chain_spec, rpc)
self.declarator_address = registry.by_name('AddressDeclarator')
def __process_tx(self, w3, session, t, r, l, b): def __process_tx(self, w3, session, t, r, l, b):
@ -198,67 +205,79 @@ class Tracker:
# session.flush() # session.flush()
def check_token(self, address): def check_token(self, registry, address):
t = None return registry.by_address(self.chain_spec, address)
try: #except UnknownContractError:
t = CICRegistry.get_address(CICRegistry.default_chain_spec, address) # logg.debug('contract {} not in registry'.format(address))
return t
except UnknownContractError:
logg.debug('contract {} not in registry'.format(address))
# If nothing was returned, we look up the token in the declarator # If nothing was returned, we look up the token in the declarator
for trust in self.trusts: # for trust in self.trusts:
logg.debug('look up declaration for contract {} with trust {}'.format(address, trust)) # logg.debug('look up declaration for contract {} with trust {}'.format(address, trust))
fn = self.declarator.function('declaration') # fn = self.declarator.function('declaration')
# TODO: cache trust in LRUcache # # TODO: cache trust in LRUcache
declaration_array = fn(trust, address).call() # declaration_array = fn(trust, address).call()
try: # try:
declaration = to_token_declaration(trust, address, declaration_array, [rubberstamp]) # declaration = to_token_declaration(trust, address, declaration_array, [rubberstamp])
logg.debug('found declaration for token {} from trust address {}'.format(address, trust)) # logg.debug('found declaration for token {} from trust address {}'.format(address, trust))
except UnknownDeclarationError: # except UnknownDeclarationError:
continue # continue
#
try: # try:
c = w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=address) # c = w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=address)
t = CICRegistry.add_token(self.chain_spec, c) # t = CICRegistry.add_token(self.chain_spec, c)
break # break
except ValueError: # except ValueError:
logg.error('declaration for {} validates as token, but location is not ERC20 compatible'.format(address)) # logg.error('declaration for {} validates as token, but location is not ERC20 compatible'.format(address))
#
return t # return t
# TODO use input data instead of logs # TODO use input data instead of logs
def process(self, w3, session, block): def process(self, rpc, session, block):
#self.refresh_registry(w3) #self.refresh_registry(w3)
tx_count = w3.eth.getBlockTransactionCount(block.hash) o = transaction_count(block['hash'])
b = w3.eth.getBlock(block.hash) r = rpc.do(o)
tx_count = int(r, 16)
o = block_by_hash(block['hash'])
block = rpc.do(o)
for i in range(self.tx_height, tx_count): for i in range(self.tx_height, tx_count):
tx = w3.eth.getTransactionByBlock(block.hash, i) o = transaction_by_block(block['hash'], i)
if tx.to == None: tx = rpc.do(o)
logg.debug('block {} tx {} is contract creation tx, skipping'.format(block.number, i))
continue if tx['to'] == None:
if len(w3.eth.getCode(tx.to)) == 0: logg.debug('block {} tx {} is contract creation tx, skipping'.format(block['number'], i))
logg.debug('block {} tx {} not a contract tx, skipping'.format(block.number, i))
continue continue
t = self.check_token(tx.to) o = code(tx['to'])
if t != None and isinstance(t, Token): r = rpc.do(o)
r = w3.eth.getTransactionReceipt(tx.hash)
for l in r.logs:
logg.debug('block {} tx {} {} token log {} {}'.format(block.number, i, tx.hash.hex(), l.logIndex, l.topics[0].hex()))
if l.topics[0].hex() == log_topics['transfer']:
self.__process_tx(w3, session, t, r, l, b)
# TODO: cache contracts in LRUcache #if len(w3.eth.getCode(tx.to)) == 0:
elif self.convert_enabled and tx.to == CICRegistry.get_contract(CICRegistry.default_chain_spec, 'Converter').address: if len(strip_0x(r)) == 0:
r = w3.eth.getTransactionReceipt(tx.hash) logg.debug('block {} tx {} not a contract tx, skipping'.format(block['number'], i))
for l in r.logs: continue
logg.info('block {} tx {} {} bancornetwork log {} {}'.format(block.number, i, tx.hash.hex(), l.logIndex, l.topics[0].hex()))
if l.topics[0].hex() == log_topics['convert']: registry = CICRegistry(self.chain_spec, rpc)
self.__process_convert(w3, session, t, r, l, b) t = self.check_token(registry, tx['to'])
logg.debug('token? {}'.format(t))
# t = self.check_token(tx.to)
# if t != None and isinstance(t, Token):
# r = w3.eth.getTransactionReceipt(tx.hash)
# for l in r.logs:
# logg.debug('block {} tx {} {} token log {} {}'.format(block.number, i, tx.hash.hex(), l.logIndex, l.topics[0].hex()))
# if l.topics[0].hex() == log_topics['transfer']:
# self.__process_tx(w3, session, t, r, l, b)
#
# # TODO: cache contracts in LRUcache
# elif self.convert_enabled and tx.to == CICRegistry.get_contract(CICRegistry.default_chain_spec, 'Converter').address:
# r = w3.eth.getTransactionReceipt(tx.hash)
# for l in r.logs:
# logg.info('block {} tx {} {} bancornetwork log {} {}'.format(block.number, i, tx.hash.hex(), l.logIndex, l.topics[0].hex()))
# if l.topics[0].hex() == log_topics['convert']:
# self.__process_convert(w3, session, t, r, l, b)
session.execute("UPDATE tx_sync SET tx = '{}'".format(tx.hash.hex())) session.execute("UPDATE tx_sync SET tx = '{}'".format(tx['hash']))
session.commit() session.commit()
self.tx_height += 1 self.tx_height += 1
@ -271,67 +290,59 @@ class Tracker:
logg.info('starting at block {} tx index {}'.format(self.block_height, self.tx_height)) logg.info('starting at block {} tx index {}'.format(self.block_height, self.tx_height))
self.state = RunStateEnum.RUN self.state = RunStateEnum.RUN
while self.state == RunStateEnum.RUN: while self.state == RunStateEnum.RUN:
(provider, w3) = web3_constructor()
session = SessionBase.create_session() session = SessionBase.create_session()
try: try:
block = w3.eth.getBlock(self.block_height) rpc = rpc_connect()
self.process(w3, session, block) o = block_by_number(self.block_height)
block = rpc.do(o)
#block = w3.eth.getBlock(self.block_height)
self.process(rpc, session, block)
self.block_height += 1 self.block_height += 1
self.tx_height = 0 self.tx_height = 0
except BlockNotFound as e: # except BlockNotFound as e:
logg.debug('no block {} yet, zZzZ...'.format(self.block_height)) # logg.debug('no block {} yet, zZzZ...'.format(self.block_height))
time.sleep(self.__get_next_retry()) # time.sleep(self.__get_next_retry())
except ConnectionClosedError as e: # except ConnectionClosedError as e:
logg.info('connection gone, retrying') # logg.info('connection gone, retrying')
time.sleep(self.__get_next_retry(True)) # time.sleep(self.__get_next_retry(True))
except OSError as e: # except OSError as e:
logg.error('cannot connect {}'.format(e)) # logg.error('cannot connect {}'.format(e))
time.sleep(self.__get_next_retry(True)) # time.sleep(self.__get_next_retry(True))
except Exception as e: except Exception as e:
session.close() session.close()
raise(e) raise(e)
session.close() session.close()
def load(self, w3): def load(self, conn):
session = SessionBase.create_session() session = SessionBase.create_session()
r = session.execute('SELECT tx FROM tx_sync').first() r = session.execute('SELECT tx FROM tx_sync').first()
if r != None: if r != None:
if r[0] == '0x{0:0{1}X}'.format(0, 64): if r[0] == '0x{0:0{1}X}'.format(0, 64):
logg.debug('last tx was zero-address, starting from scratch') logg.debug('last tx was zero-address, starting from scratch')
return return
t = w3.eth.getTransaction(r[0]) o = transaction(r[0])
tx = conn.do(o)
self.block_height = t.blockNumber
self.tx_height = t.transactionIndex+1
c = w3.eth.getBlockTransactionCount(t.blockHash.hex()) self.block_height = int(tx['blockNumber'], 16)
logg.debug('last tx processed {} index {} (max index {})'.format(t.blockNumber, t.transactionIndex, c-1)) self.tx_height = int(tx['transactionIndex'], 16) + 1
o = transaction_count(tx['blockHash'])
r = conn.do(o)
c = int(r, 16)
logg.debug('last tx processed {} index {} (max index {})'.format(tx['blockNumber'], tx['transactionIndex'], c-1))
if c == self.tx_height: if c == self.tx_height:
self.block_height += 1 self.block_height += 1
self.tx_height = 0 self.tx_height = 0
session.close() session.close()
(provider, w3) = web3_constructor()
trust = config.get('CIC_TRUST_ADDRESS', "").split(",") trust = config.get('CIC_TRUST_ADDRESS', "").split(",")
chain_spec = args.i
try:
w3.eth.chainId
except Exception as e:
logg.exception(e)
sys.stderr.write('cannot connect to evm node\n')
sys.exit(1)
def main(): def main():
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
CICRegistry.init(w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec) t = Tracker(chain_spec, rpc, trust)
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
chain_registry = ChainRegistry(chain_spec)
CICRegistry.add_chain_registry(chain_registry)
t = Tracker(chain_spec, trust) t.load(rpc)
t.load(w3)
t.loop() t.loop()

4
apps/cic-cache/requirements.txt
View File

@ -1,8 +1,8 @@
alembic==1.4.2 alembic==1.4.2
confini~=0.3.6b2 confini~=0.3.6rc3
uwsgi==2.0.19.1 uwsgi==2.0.19.1
moolb~=0.1.0 moolb~=0.1.0
cic-registry~=0.5.3a4 cic-eth-registry~=0.5.4a8
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
semver==2.13.0 semver==2.13.0
psycopg2==2.8.6 psycopg2==2.8.6