kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rehabilitate transfer, approve

Browse Source 
Signed-off-by: nolash <dev@holbrook.no>
This commit is contained in:
Louis Holbrook
2021-03-29 13:27:53 +00:00
parent 299385f320
commit b65ab8a0ca
222 changed files with 3272 additions and 809800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
apps/cic-eth/cic_eth/runnable/create.py
View File

@@ -1,17 +1,19 @@
#!/usr/bin/python
#import socket
import sys
import os
import logging
import uuid
import json
import argparse
# external imports
import celery
import confini
import redis
from xdg.BaseDirectory import xdg_config_home
import celery
# local imports
from cic_eth.api import Api
import confini
import argparse
import redis
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger('create_account_script')
@@ -50,6 +52,7 @@ args_override = {
'REDIS_DB': getattr(args, 'redis_db'),
}
config.dict_override(args_override, 'cli')
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))

46
apps/cic-eth/cic_eth/runnable/ctrl.py
View File

@@ -3,34 +3,28 @@ import argparse
import sys
import os
import logging
import re
# third-party imports
import confini
import celery
import web3
from cic_registry.chain import ChainSpec
from cic_registry import zero_address
from chainlib.chain import ChainSpec
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.address import is_checksum_address
# local imports
from cic_eth.api import AdminApi
from cic_eth.eth.rpc import RpcClient
from cic_eth.db.enum import LockEnum
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
logging.getLogger('web3').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
default_abi_dir = '/usr/share/local/cic/solidity/abi'
default_format = 'terminal'
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-r', '--registry-address', type=str, help='CIC registry address')
argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format')
argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -40,7 +34,7 @@ argparser.add_argument('-vv', help='be more verbose', action='store_true')
def process_lock_args(argparser):
argparser.add_argument('flags', type=str, help='Flags to manipulate')
argparser.add_argument('address', default=zero_address, nargs='?', type=str, help='Ethereum address to unlock,')
argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,')
sub = argparser.add_subparsers()
sub.dest = "command"
@@ -69,30 +63,12 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = web3.Web3.WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = web3.Web3.HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3_constructor)
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
queue = args.q
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec)
c = RpcClient(chain_spec)
admin_api = AdminApi(c)
admin_api = AdminApi(None)
def lock_names_to_flag(s):
@@ -108,14 +84,14 @@ def lock_names_to_flag(s):
def main():
if args.command == 'unlock':
flags = lock_names_to_flag(args.flags)
if not web3.Web3.isChecksumAddress(args.address):
if not is_checksum_address(args.address):
raise ValueError('Invalid checksum address {}'.format(args.address))
s = celery.signature(
'cic_eth.admin.ctrl.unlock',
[
None,
chain_str,
chain_spec.asdict(),
args.address,
flags,
],
@@ -127,14 +103,14 @@ def main():
if args.command == 'lock':
flags = lock_names_to_flag(args.flags)
if not web3.Web3.isChecksumAddress(args.address):
if not is_checksum_address(args.address):
raise ValueError('Invalid checksum address {}'.format(args.address))
s = celery.signature(
'cic_eth.admin.ctrl.lock',
[
None,
chain_str,
chain_spec.asdict(),
args.address,
flags,
],

55
apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py
View File

@@ -11,16 +11,15 @@ import datetime
# third-party imports
import confini
import celery
import web3
from web3 import HTTPProvider, WebsocketProvider
from cic_registry import CICRegistry
from cic_registry.chain import ChainSpec
from cic_eth_registry import CICRegistry
from chainlib.chain import ChainSpec
from chainlib.eth.tx import unpack
from chainlib.connection import RPCConnection
from chainsyncer.error import SyncDone
from hexathon import strip_0x
# local imports
import cic_eth
from cic_eth.eth import RpcClient
from cic_eth.db import SessionBase
from cic_eth.db.enum import StatusEnum
from cic_eth.db.enum import StatusBits
@@ -31,7 +30,6 @@ from cic_eth.queue.tx import (
set_dequeue,
)
from cic_eth.admin.ctrl import lock_send
from cic_eth.sync.error import LoopDone
from cic_eth.eth.tx import send as task_tx_send
from cic_eth.error import (
PermanentTxError,
@@ -42,16 +40,14 @@ from cic_eth.error import (
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
config_dir = os.path.join('/usr/local/etc/cic-eth')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
argparser.add_argument('-v', help='be verbose', action='store_true')
@@ -68,6 +64,11 @@ os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config.process()
# override args
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@@ -79,25 +80,12 @@ queue = args.q
dsn = dsn_from_config(config)
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3_constructor)
RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default')
run = True
class DispatchSyncer:
yield_delay = 0.005
@@ -117,7 +105,6 @@ class DispatchSyncer:
chain_str = str(self.chain_spec)
for k in txs.keys():
tx_raw = txs[k]
#tx = unpack_signed_raw_tx_hex(tx_raw, self.chain_spec.chain_id())
tx_raw_bytes = bytes.fromhex(strip_0x(tx_raw))
tx = unpack(tx_raw_bytes, self.chain_spec.chain_id())
@@ -131,7 +118,7 @@ class DispatchSyncer:
'cic_eth.admin.ctrl.check_lock',
[
[tx_raw],
chain_str,
self.chain_spec.asdict(),
LockEnum.QUEUE,
tx['from'],
],
@@ -140,7 +127,7 @@ class DispatchSyncer:
s_send = celery.signature(
'cic_eth.eth.tx.send',
[
chain_str,
self.chain_spec.asdict(),
],
queue=queue,
)
@@ -165,17 +152,11 @@ class DispatchSyncer:
def main():
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
c = RpcClient(chain_spec)
CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
syncer = DispatchSyncer(chain_spec)
conn = RPCConnection.connect(chain_spec, 'default')
try:
syncer.loop(c.w3, float(config.get('DISPATCHER_LOOP_INTERVAL')))
except LoopDone as e:
syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
except SyncDone as e:
sys.stderr.write("dispatcher done at block {}\n".format(e))
sys.exit(0)

99
apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py
View File

@@ -2,29 +2,61 @@
import logging
# third-party imports
import web3
import celery
from cic_registry.error import UnknownContractError
from cic_eth_registry.error import UnknownContractError
from chainlib.status import Status as TxStatus
from chainlib.eth.address import to_checksum
from chainlib.eth.address import to_checksum_address
from chainlib.eth.error import RequestMismatchException
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.erc20 import ERC20
from hexathon import strip_0x
# local imports
from .base import SyncFilter
from cic_eth.eth.token import (
unpack_transfer,
unpack_transferfrom,
)
from cic_eth.eth.account import unpack_gift
from cic_eth.eth.token import ExtendedTx
from .base import SyncFilter
from cic_eth.eth.meta import ExtendedTx
logg = logging.getLogger(__name__)
logg = logging.getLogger().getChild(__name__)
transfer_method_signature = 'a9059cbb' # keccak256(transfer(address,uint256))
transferfrom_method_signature = '23b872dd' # keccak256(transferFrom(address,address,uint256))
giveto_method_signature = '63e4bff4' # keccak256(giveTo(address))
def parse_transfer(tx):
r = ERC20.parse_transfer_request(tx.payload)
transfer_data = {}
transfer_data['to'] = r[0]
transfer_data['value'] = r[1]
transfer_data['from'] = tx['from']
transfer_data['token_address'] = tx['to']
return ('transfer', transfer_data)
def parse_transferfrom(tx):
r = ERC20.parse_transfer_request(tx.payload)
transfer_data = unpack_transferfrom(tx.payload)
transfer_data['from'] = r[0]
transfer_data['to'] = r[1]
transfer_data['value'] = r[2]
transfer_data['token_address'] = tx['to']
return ('transferfrom', transfer_data)
def parse_giftto(tx):
# TODO: broken
logg.error('broken')
return
transfer_data = unpack_gift(tx.payload)
transfer_data['from'] = tx.inputs[0]
transfer_data['value'] = 0
transfer_data['token_address'] = ZERO_ADDRESS
# TODO: would be better to query the gift amount from the block state
for l in tx.logs:
topics = l['topics']
logg.debug('topixx {}'.format(topics))
if strip_0x(topics[0]) == '45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
#transfer_data['value'] = web3.Web3.toInt(hexstr=strip_0x(l['data']))
transfer_data['value'] = int.from_bytes(bytes.fromhex(strip_0x(l_data)))
#token_address_bytes = topics[2][32-20:]
token_address = strip_0x(topics[2])[64-40:]
transfer_data['token_address'] = to_checksum_address(token_address)
return ('tokengift', transfer_data)
class CallbackFilter(SyncFilter):
@@ -66,35 +98,23 @@ class CallbackFilter(SyncFilter):
def parse_data(self, tx):
transfer_type = None
transfer_data = None
# TODO: what's with the mix of attributes and dict keys
logg.debug('have payload {}'.format(tx.payload))
method_signature = tx.payload[:8]
logg.debug('tx status {}'.format(tx.status))
if method_signature == transfer_method_signature:
transfer_data = unpack_transfer(tx.payload)
transfer_data['from'] = tx['from']
transfer_data['token_address'] = tx['to']
elif method_signature == transferfrom_method_signature:
transfer_type = 'transferfrom'
transfer_data = unpack_transferfrom(tx.payload)
transfer_data['token_address'] = tx['to']
for parser in [
parse_transfer,
parse_transferfrom,
parse_giftto,
]:
try:
(transfer_type, transfer_data) = parser(tx)
break
except RequestMismatchException:
continue
# TODO: do not rely on logs here
elif method_signature == giveto_method_signature:
transfer_type = 'tokengift'
transfer_data = unpack_gift(tx.payload)
transfer_data['from'] = tx.inputs[0]
transfer_data['value'] = 0
transfer_data['token_address'] = ZERO_ADDRESS
for l in tx.logs:
topics = l['topics']
logg.debug('topixx {}'.format(topics))
if strip_0x(topics[0]) == '45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
transfer_data['value'] = web3.Web3.toInt(hexstr=strip_0x(l['data']))
#token_address_bytes = topics[2][32-20:]
token_address = strip_0x(topics[2])[64-40:]
transfer_data['token_address'] = to_checksum(token_address)
logg.debug('resolved method {}'.format(transfer_type))
@@ -105,8 +125,6 @@ class CallbackFilter(SyncFilter):
def filter(self, conn, block, tx, db_session=None):
chain_str = str(self.chain_spec)
transfer_data = None
transfer_type = None
try:
@@ -122,11 +140,10 @@ class CallbackFilter(SyncFilter):
logg.debug('checking callbacks filter input {}'.format(tx.payload[:8]))
if transfer_data != None:
logg.debug('wtfoo {}'.format(transfer_data))
token_symbol = None
result = None
try:
tokentx = ExtendedTx(tx.hash, self.chain_spec)
tokentx = ExtendedTx(conn, tx.hash, self.chain_spec)
tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses)
tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
if transfer_data['status'] == 0:

9
apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py
View File

@@ -2,7 +2,6 @@
import logging
# external imports
from cic_registry.chain import ChainSpec
from hexathon import add_0x
# local imports
@@ -11,10 +10,10 @@ from cic_eth.db.models.base import SessionBase
from cic_eth.db.models.tx import TxCache
from cic_eth.db.models.otx import Otx
from cic_eth.queue.tx import get_paused_txs
from cic_eth.eth.task import create_check_gas_and_send_task
from cic_eth.eth.gas import create_check_gas_task
from .base import SyncFilter
logg = logging.getLogger(__name__)
logg = logging.getLogger().getChild(__name__)
class GasFilter(SyncFilter):
@@ -45,9 +44,9 @@ class GasFilter(SyncFilter):
logg.info('resuming gas-in-waiting txs for {}'.format(r[0]))
if len(txs) > 0:
s = create_check_gas_and_send_task(
s = create_check_gas_task(
list(txs.values()),
str(self.chain_spec),
self.chain_spec,
r[0],
0,
tx_hashes_hex=list(txs.keys()),

10
apps/cic-eth/cic_eth/runnable/daemons/filters/register.py
View File

@@ -3,7 +3,7 @@ import logging
# third-party imports
import celery
from chainlib.eth.address import to_checksum
from chainlib.eth.address import to_checksum_address
from hexathon import (
add_0x,
strip_0x,
@@ -12,9 +12,9 @@ from hexathon import (
# local imports
from .base import SyncFilter
logg = logging.getLogger(__name__)
logg = logging.getLogger().getChild(__name__)
account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd' # keccak256(AccountAdded(address,uint256))
account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd'
class RegistrationFilter(SyncFilter):
@@ -32,7 +32,7 @@ class RegistrationFilter(SyncFilter):
# TODO: use abi conversion method instead
address_hex = strip_0x(l['topics'][1])[64-40:]
address = to_checksum(add_0x(address_hex))
address = to_checksum_address(add_0x(address_hex))
logg.info('request token gift to {}'.format(address))
s_nonce = celery.signature(
'cic_eth.eth.tx.reserve_nonce',
@@ -44,7 +44,7 @@ class RegistrationFilter(SyncFilter):
s_gift = celery.signature(
'cic_eth.eth.account.gift',
[
str(self.chain_spec),
self.chain_spec.asdict(),
],
queue=self.queue,
)

52
apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py
View File

@@ -7,41 +7,29 @@ from hexathon import (
strip_0x,
add_0x,
)
from chainlib.eth.address import to_checksum
from chainlib.eth.address import to_checksum_address
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.contract import (
ABIContractType,
abi_decode_single,
)
from cic_eth_registry import CICRegistry
from erc20_transfer_authorization import TransferAuthorization
# local imports
from .base import SyncFilter
logg = logging.getLogger(__name__)
transfer_request_signature = 'ed71262a'
def unpack_create_request(data):
data = strip_0x(data)
cursor = 0
f = data[cursor:cursor+8]
cursor += 8
if f != transfer_request_signature:
raise ValueError('Invalid create request data ({})'.format(f))
o = {}
o['sender'] = data[cursor+24:cursor+64]
cursor += 64
o['recipient'] = data[cursor+24:cursor+64]
cursor += 64
o['token'] = data[cursor+24:cursor+64]
cursor += 64
o['value'] = int(data[cursor:], 16)
return o
class TransferAuthFilter(SyncFilter):
def __init__(self, registry, chain_spec, queue=None):
def __init__(self, registry, chain_spec, conn, queue=None, call_address=ZERO_ADDRESS):
self.queue = queue
self.chain_spec = chain_spec
self.transfer_request_contract = registry.get_contract(self.chain_spec, 'TransferAuthorization')
registry = CICRegistry(chain_spec, conn)
self.transfer_request_contract = registry.by_name('TransferAuthorization', sender_address=call_address)
def filter(self, conn, block, tx, session): #rcpt, chain_str, session=None):
@@ -61,11 +49,13 @@ class TransferAuthFilter(SyncFilter):
logg.debug('not our transfer auth contract address {}'.format(recipient))
return False
o = unpack_create_request(tx.payload)
r = TransferAuthorization.parse_create_request_request(tx.payload)
sender = add_0x(to_checksum(o['sender']))
recipient = add_0x(to_checksum(recipient))
token = add_0x(to_checksum(o['token']))
sender = abi_decode_single(ABIContractType.ADDRESS, r[0])
recipient = abi_decode_single(ABIContractType.ADDRESS, r[1])
token = abi_decode_single(ABIContractType.ADDRESS, r[2])
value = abi_decode_single(ABIContractType.UINT256, r[3])
token_data = {
'address': token,
}
@@ -83,8 +73,8 @@ class TransferAuthFilter(SyncFilter):
[
sender,
recipient,
o['value'],
str(self.chain_spec),
value,
self.chain_spec.asdict(),
],
queue=self.queue,
)

3
apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py
View File

@@ -13,7 +13,7 @@ from chainsyncer.db.models.base import SessionBase
from chainlib.status import Status
from .base import SyncFilter
logg = logging.getLogger(__name__)
logg = logging.getLogger().getChild(__name__)
class TxFilter(SyncFilter):
@@ -31,6 +31,7 @@ class TxFilter(SyncFilter):
logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
return None
logg.info('tx filter match on {}'.format(otx.tx_hash))
db_session.flush()
SessionBase.release_session(db_session)
s = celery.signature(
'cic_eth.queue.tx.set_final_status',

83
apps/cic-eth/cic_eth/runnable/daemons/retry.py
View File

@@ -8,9 +8,8 @@ import datetime
import web3
import confini
import celery
from web3 import HTTPProvider, WebsocketProvider
from cic_registry import CICRegistry
from cic_registry.chain import ChainSpec
from cic_eth_registry import CICRegistry
from chainlib.chain import ChainSpec
from cic_eth.db import dsn_from_config
from cic_eth.db import SessionBase
@@ -25,19 +24,14 @@ from cic_eth.eth.util import unpack_signed_raw_tx_hex
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
config_dir = os.path.join('/usr/local/etc/cic-eth')
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('--retry-delay', dest='retry_delay', type=str, help='seconds to wait for retrying a transaction that is marked as sent')
argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
argparser.add_argument('-v', help='be verbose', action='store_true')
@@ -56,6 +50,7 @@ config = confini.Config(config_dir, args.env_prefix)
config.process()
# override args
args_override = {
'ETH_PROVIDER': getattr(args, 'p'),
'ETH_ABI_DIR': getattr(args, 'abi_dir'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'),
@@ -71,31 +66,15 @@ queue = args.q
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
RPCConnection.registry_location(args.p, chain_spec, tag='default')
dsn = dsn_from_config(config)
SessionBase.connect(dsn)
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3_constructor)
straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
# TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here
def sendfail_filter(w3, tx_hash, rcpt, chain_str):
chain_spec = ChainSpec.from_chain_str(chain_str)
def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
tx_dict = get_tx(tx_hash)
tx = unpack_signed_raw_tx_hex(tx_dict['signed_tx'], chain_spec.chain_id())
logg.debug('submitting tx {} for retry'.format(tx_hash))
@@ -137,7 +116,7 @@ def sendfail_filter(w3, tx_hash, rcpt, chain_str):
# TODO: can we merely use the dispatcher instead?
def dispatch(chain_str):
def dispatch(conn, chain_spec):
txs = get_status_tx(StatusEnum.RETRY, before=datetime.datetime.utcnow())
if len(txs) == 0:
logg.debug('no retry state txs found')
@@ -199,11 +178,49 @@ def dispatch(chain_str):
# s_send.apply_async()
def main():
class RetrySyncer(Syncer):
c = RpcClient(chain_spec)
CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
def __init__(self, chain_spec, stalled_grace_seconds, failed_grace_seconds=None, final_func=None):
self.chain_spec = chain_spec
if failed_grace_seconds == None:
failed_grace_seconds = stalled_grace_seconds
self.stalled_grace_seconds = stalled_grace_seconds
self.failed_grace_seconds = failed_grace_seconds
self.final_func = final_func
def get(self):
# before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.failed_grace_seconds)
# failed_txs = get_status_tx(
# StatusEnum.SENDFAIL.value,
# before=before,
# )
before = datetime.datetime.utcnow() - datetime.timedelta(seconds=self.stalled_grace_seconds)
stalled_txs = get_status_tx(
StatusBits.IN_NETWORK.value,
not_status=StatusBits.FINAL | StatusBits.MANUAL | StatusBits.OBSOLETE,
before=before,
)
# return list(failed_txs.keys()) + list(stalled_txs.keys())
return stalled_txs
def process(self, conn, ref):
logg.debug('tx {}'.format(ref))
for f in self.filter:
f(conn, ref, None, str(self.chain_spec))
def loop(self, interval):
while self.running and Syncer.running_global:
rpc = RPCConnection.connect(self.chain_spec, 'default')
for tx in self.get():
self.process(rpc, tx)
if self.final_func != None:
self.final_func(rpc, self.chain_spec)
time.sleep(interval)
def main():
syncer = RetrySyncer(chain_spec, straggler_delay, final_func=dispatch)
syncer.filter.append(sendfail_filter)

135
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@@ -8,28 +8,21 @@ import re
import urllib
import websocket
# third-party imports
# external imports
import celery
import confini
from crypto_dev_signer.eth.web3ext import Web3 as Web3Ext
from web3 import HTTPProvider, WebsocketProvider
from gas_proxy.web3 import GasMiddleware
from chainlib.connection import RPCConnection
from chainlib.eth.connection import EthUnixSignerConnection
from chainlib.chain import ChainSpec
# local imports
from cic_registry.registry import CICRegistry
from cic_registry.registry import ChainRegistry
from cic_registry.registry import ChainSpec
from cic_registry.helper.declarator import DeclaratorOracleAdapter
from cic_eth_registry import CICRegistry
from cic_bancor.bancor import BancorRegistryClient
from cic_eth.eth import bancor
from cic_eth.eth import token
from cic_eth.eth import erc20
from cic_eth.eth import tx
from cic_eth.eth import account
from cic_eth.admin import debug
from cic_eth.admin import ctrl
from cic_eth.eth.rpc import RpcClient
from cic_eth.eth.rpc import GasOracle
from cic_eth.queue import tx
from cic_eth.queue import balance
from cic_eth.callbacks import Callback
@@ -47,7 +40,7 @@ logg = logging.getLogger()
config_dir = os.path.join('/usr/local/etc/cic-eth')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', type=str, help='web3 provider')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider')
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks')
argparser.add_argument('-r', type=str, help='CIC registry address')
@@ -68,12 +61,12 @@ config = confini.Config(args.c, args.env_prefix)
config.process()
# override args
args_override = {
'ETH_ABI_DIR': getattr(args, 'abi_dir'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'ETH_PROVIDER': getattr(args, 'p'),
'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'),
}
config.add(args.q, '_CELERY_QUEUE', True)
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
@@ -81,7 +74,7 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
# connect to database
dsn = dsn_from_config(config)
SessionBase.connect(dsn, pool_size=8, debug=config.true('DATABASE_DEBUG'))
SessionBase.connect(dsn, pool_size=50, debug=config.true('DATABASE_DEBUG'))
# verify database connection with minimal sanity query
session = SessionBase.create_session()
@@ -122,68 +115,14 @@ else:
'result_backend': result,
})
# set up web3
# TODO: web3 socket wrapping is now a lot of code. factor out
class JSONRPCHttpSocketAdapter:
def __init__(self, url):
self.response = None
self.url = url
def send(self, data):
logg.debug('redirecting socket send to jsonrpc http socket adapter {} {}'.format(self.url, data))
req = urllib.request.Request(self.url, method='POST')
req.add_header('Content-type', 'application/json')
req.add_header('Connection', 'close')
res = urllib.request.urlopen(req, data=data.encode('utf-8'))
self.response = res.read().decode('utf-8')
logg.debug('setting jsonrpc http socket adapter response to {}'.format(self.response))
def recv(self, n=0):
return self.response
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
socket_constructor = None
if re.match(re_websocket, blockchain_provider) != None:
def socket_constructor_ws():
return websocket.create_connection(config.get('ETH_PROVIDER'))
socket_constructor = socket_constructor_ws
blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
def socket_constructor_http():
return JSONRPCHttpSocketAdapter(config.get('ETH_PROVIDER'))
socket_constructor = socket_constructor_http
blockchain_provider = HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3ext_constructor():
w3 = Web3Ext(blockchain_provider, config.get('SIGNER_SOCKET_PATH'))
GasMiddleware.socket_constructor = socket_constructor
w3.middleware_onion.add(GasMiddleware)
def sign_transaction(tx):
r = w3.eth.signTransaction(tx)
d = r.__dict__
for k in d.keys():
if k == 'tx':
d[k] = d[k].__dict__
else:
d[k] = d[k].hex()
return d
setattr(w3.eth, 'sign_transaction', sign_transaction)
setattr(w3.eth, 'send_raw_transaction', w3.eth.sendRawTransaction)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3ext_constructor)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
def main():
argv = ['worker']
@@ -196,33 +135,19 @@ def main():
argv.append('-n')
argv.append(args.q)
if config.true('SSL_ENABLE_CLIENT'):
Callback.ssl = True
Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
Callback.ssl_key_file = config.get('SSL_KEY_FILE')
Callback.ssl_password = config.get('SSL_PASSWORD')
# if config.true('SSL_ENABLE_CLIENT'):
# Callback.ssl = True
# Callback.ssl_cert_file = config.get('SSL_CERT_FILE')
# Callback.ssl_key_file = config.get('SSL_KEY_FILE')
# Callback.ssl_password = config.get('SSL_PASSWORD')
#
# if config.get('SSL_CA_FILE') != '':
# Callback.ssl_ca_file = config.get('SSL_CA_FILE')
if config.get('SSL_CA_FILE') != '':
Callback.ssl_ca_file = config.get('SSL_CA_FILE')
rpc = RPCConnection.connect(chain_spec, 'default')
registry = CICRegistry(chain_spec, rpc)
registry_address = registry.by_name('ContractRegistry')
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
c = RpcClient(chain_spec)
CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
chain_registry = ChainRegistry(chain_spec)
CICRegistry.add_chain_registry(chain_registry, True)
try:
CICRegistry.get_contract(chain_spec, 'CICRegistry')
except Exception as e:
logg.exception('Eek, registry failure is baaad juju {}'.format(e))
sys.exit(1)
if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None:
logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
@@ -230,15 +155,7 @@ def main():
trusted_addresses = trusted_addresses_src.split(',')
for address in trusted_addresses:
logg.info('using trusted address {}'.format(address))
oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
chain_registry.add_oracle(oracle, 'naive_erc20_oracle')
#chain_spec = CICRegistry.default_chain_spec
#bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
#bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
#bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
#bancor_registry.load(True)
current_app.worker_main(argv)

54
apps/cic-eth/cic_eth/runnable/daemons/tracker.py
View File

@@ -11,18 +11,15 @@ import re
import confini
import celery
import rlp
import web3
from web3 import HTTPProvider, WebsocketProvider
import cic_base.config
import cic_base.log
import cic_base.argparse
import cic_base.rpc
from cic_registry import CICRegistry
from cic_eth_registry import CICRegistry
from cic_eth_registry.error import UnknownContractError
from chainlib.chain import ChainSpec
from cic_registry import zero_address
from cic_registry.chain import ChainRegistry
from cic_registry.error import UnknownContractError
from chainlib.eth.connection import HTTPConnection
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.connection import RPCConnection
from chainlib.eth.block import (
block_latest,
)
@@ -37,22 +34,7 @@ from chainsyncer.driver import (
from chainsyncer.db.models.base import SessionBase
# local imports
from cic_eth.registry import init_registry
from cic_eth.eth import RpcClient
from cic_eth.db import Otx
from cic_eth.db import TxConvertTransfer
from cic_eth.db.models.tx import TxCache
from cic_eth.db.enum import StatusEnum
from cic_eth.db import dsn_from_config
from cic_eth.queue.tx import get_paused_txs
#from cic_eth.sync import Syncer
#from cic_eth.sync.error import LoopDone
from cic_eth.db.error import UnknownConvertError
from cic_eth.eth.util import unpack_signed_raw_tx
from cic_eth.eth.task import create_check_gas_and_send_task
from cic_eth.eth.token import unpack_transfer
from cic_eth.eth.token import unpack_transferfrom
from cic_eth.eth.account import unpack_gift
from cic_eth.runnable.daemons.filters import (
CallbackFilter,
GasFilter,
@@ -75,27 +57,25 @@ config.add(args.q, '_CELERY_QUEUE', True)
cic_base.config.log(config)
dsn = dsn_from_config(config)
SessionBase.connect(dsn, pool_size=1, debug=config.true('DATABASE_DEBUG'))
SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
def main():
# parse chain spec object
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
# connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
# set up registry
w3 = cic_base.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
registry = init_registry(config, w3)
# Connect to blockchain with chainlib
conn = HTTPConnection(config.get('ETH_PROVIDER'))
rpc = RPCConnection.connect(chain_spec, 'default')
o = block_latest()
r = conn.do(o)
r = rpc.do(o)
block_offset = int(strip_0x(r), 16) + 1
logg.debug('starting at block {}'.format(block_offset))
@@ -147,7 +127,7 @@ def main():
gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
i = 0
for syncer in syncers:
@@ -156,17 +136,15 @@ def main():
syncer.add_filter(registration_filter)
# TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
syncer.add_filter(tx_filter)
syncer.add_filter(transfer_auth_filter)
#syncer.add_filter(transfer_auth_filter)
for cf in callback_filters:
syncer.add_filter(cf)
r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), conn)
r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
i += 1
sys.exit(0)
if __name__ == '__main__':
main()

31
apps/cic-eth/cic_eth/runnable/tag.py
View File

@@ -5,14 +5,14 @@ import logging
import argparse
import re
# third-party imports
import web3
from web3 import HTTPProvider, WebsocketProvider
# external imports
import celery
import confini
from chainlib.chain import ChainSpec
from xdg.BaseDirectory import xdg_config_home
# local imports
from cic_eth.api import AdminApi
from cic_eth.eth import RpcClient
from cic_eth.db import dsn_from_config
from cic_eth.db.models.base import SessionBase
@@ -48,29 +48,14 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}\n{}'.format(args.c, config))
chain_spec = ChainSpec.from_chain_str(args.i)
dsn = dsn_from_config(config)
SessionBase.connect(dsn)
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3_constructor)
c = RpcClient(config.get('CIC_CHAIN_SPEC'))
def main():
api = AdminApi(c)
api.tag_account(args.tag, args.address)
api = AdminApi(None)
api.tag_account(args.tag, args.address, chain_spec)
if __name__ == '__main__':

85
apps/cic-eth/cic_eth/runnable/view.py
View File

@@ -11,18 +11,17 @@ import sys
import re
import datetime
# third-party imports
# external imports
import confini
import celery
import web3
from cic_registry import CICRegistry
from cic_registry.chain import ChainSpec
from cic_registry.chain import ChainRegistry
from cic_eth_registry import CICRegistry
from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
from hexathon import add_0x
# local imports
from cic_eth.api import AdminApi
from cic_eth.eth.rpc import RpcClient
from cic_eth.db.enum import (
StatusEnum,
status_str,
@@ -32,18 +31,14 @@ from cic_eth.db.enum import (
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
logging.getLogger('web3').setLevel(logging.WARNING)
logging.getLogger('urllib3').setLevel(logging.WARNING)
default_abi_dir = '/usr/share/local/cic/solidity/abi'
default_format = 'terminal'
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
argparser.add_argument('-f', '--format', dest='f', default='terminal', type=str, help='Output format')
argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output statis bit enum names only')
argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
@@ -74,38 +69,30 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
config.add(add_0x(args.query), '_QUERY', True)
re_websocket = re.compile('^wss?://')
re_http = re.compile('^https?://')
blockchain_provider = config.get('ETH_PROVIDER')
if re.match(re_websocket, blockchain_provider) != None:
blockchain_provider = web3.Web3.WebsocketProvider(blockchain_provider)
elif re.match(re_http, blockchain_provider) != None:
blockchain_provider = web3.Web3.HTTPProvider(blockchain_provider)
else:
raise ValueError('unknown provider url {}'.format(blockchain_provider))
def web3_constructor():
w3 = web3.Web3(blockchain_provider)
return (blockchain_provider, w3)
RpcClient.set_constructor(web3_constructor)
try:
config.add(add_0x(args.query), '_QUERY', True)
except:
config.add(args.query, '_QUERY', True)
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
queue = args.q
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec)
c = RpcClient(chain_spec)
admin_api = AdminApi(c)
CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
chain_registry = ChainRegistry(chain_spec)
CICRegistry.add_chain_registry(chain_registry)
CICRegistry.add_path(config.get('ETH_ABI_DIR'))
CICRegistry.load_for(chain_spec)
rpc = EthHTTPConnection(args.p)
registry_address = config.get('CIC_REGISTRY_ADDRESS')
admin_api = AdminApi(rpc)
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None:
logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
sys.exit(1)
trusted_addresses = trusted_addresses_src.split(',')
for address in trusted_addresses:
logg.info('using trusted address {}'.format(address))
fmt = 'terminal'
if args.f[:1] == 'j':
@@ -155,19 +142,33 @@ def render_lock(o, **kwargs):
return s
def connect_registry(registry_address, chain_spec, rpc):
CICRegistry.address = registry_address
registry = CICRegistry(chain_spec, rpc)
declarator_address = registry.by_name('AddressDeclarator')
lookup = AddressDeclaratorLookup(declarator_address, trusted_addresses)
registry.add_lookup(lookup)
return registry
# TODO: move each command to submodule
def main():
txs = []
renderer = render_tx
if len(config.get('_QUERY')) > 66:
txs = [admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'))]
registry = connect_registry(registry_address, chain_spec, rpc)
txs = [admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry)]
elif len(config.get('_QUERY')) > 42:
txs = [admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'))]
registry = connect_registry(registry_address, chain_spec, rpc)
txs = [admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry)]
elif len(config.get('_QUERY')) == 42:
registry = connect_registry(registry_address, chain_spec, rpc)
txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False)
renderer = render_account
elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
txs = admin_api.get_lock()
t = admin_api.get_lock()
txs = t.get()
renderer = render_lock
else:
raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))