Replace cic-base in tracker
This commit is contained in:
nolash
parent
d6157652c1
commit
f5e422cadd
GPG Key ID:
21D2E7BB88C2A746
|
|
@ -14,6 +14,12 @@ from chainlib.eth.cli import (
|
|||
ArgumentParser as BaseArgumentParser,
|
||||
Flag,
|
||||
)
|
||||
from chainlib.connection import (
|
||||
RPCConnection,
|
||||
ConnType,
|
||||
)
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.connection import EthUnixSignerConnection
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -24,14 +30,22 @@ class CICFlag(enum.IntEnum):
|
|||
|
||||
# celery - nibble 1
|
||||
CELERY = 1
|
||||
CELERY_QUEUE = 2
|
||||
|
||||
# redis - nibble 2
|
||||
REDIS = 16
|
||||
REDIS_CALLBACK = 32
|
||||
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
|
||||
# chain - nibble 3
|
||||
CHAIN = 256
|
||||
|
||||
# sync - nibble 4
|
||||
SYNCER = 4096
|
||||
|
||||
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
||||
argflag_local_chain = CICFlag.CHAIN
|
||||
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
|
||||
|
||||
|
||||
class Config(BaseConfig):
|
||||
|
|
@ -54,6 +68,10 @@ class Config(BaseConfig):
|
|||
local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
|
||||
local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
|
||||
local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
|
||||
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
||||
config.dict_override(local_args_override, 'local cli args')
|
||||
|
|
@ -65,6 +83,8 @@ class Config(BaseConfig):
|
|||
if local_arg_flags & CICFlag.CELERY:
|
||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||
|
||||
|
||||
|
||||
logg.debug('config loaded:\n{}'.format(config))
|
||||
|
||||
return config
|
||||
|
|
@ -83,6 +103,11 @@ class ArgumentParser(BaseArgumentParser):
|
|||
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
self.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
|
||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||
|
||||
|
||||
class CeleryApp:
|
||||
|
|
@ -100,3 +125,31 @@ class CeleryApp:
|
|||
logg.info('creating celery app without results backend on {}'.format(broker_url))
|
||||
|
||||
return celery_app
|
||||
|
||||
|
||||
class RPC:
|
||||
|
||||
def __init__(self, chain_spec, rpc_provider, signer_provider=None):
|
||||
self.chain_spec = chain_spec
|
||||
self.rpc_provider = rpc_provider
|
||||
self.signer_provider = signer_provider
|
||||
|
||||
|
||||
def get_default(self):
|
||||
return RPCConnection.connect(self.chain_spec, 'default')
|
||||
|
||||
|
||||
@staticmethod
|
||||
def from_config(config):
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
|
||||
if config.get('SIGNER_PROVIDER'):
|
||||
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
|
||||
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
|
||||
rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
||||
logg.info('set up rpc: {}'.format(rpc))
|
||||
return rpc
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
[cic]
|
||||
registry_address =
|
||||
trust_address =
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
[database]
|
||||
engine =
|
||||
driver =
|
||||
host =
|
||||
port =
|
||||
name =
|
||||
user =
|
||||
password =
|
||||
debug = 0
|
||||
pool_size = 0
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
[signer]
|
||||
provider =
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
[syncer]
|
||||
loop_interval = 1
|
||||
block_offset =
|
||||
no_history = 0
|
||||
|
|
@ -17,7 +17,7 @@ import cic_eth.cli
|
|||
from cic_eth.api import Api
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger('create_account_script')
|
||||
logg = logging.getLogger()
|
||||
logging.getLogger('confini').setLevel(logging.WARNING)
|
||||
logging.getLogger('gnupg').setLevel(logging.WARNING)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,13 +8,6 @@ import sys
|
|||
import re
|
||||
|
||||
# external imports
|
||||
import confini
|
||||
import celery
|
||||
import rlp
|
||||
import cic_base.config
|
||||
import cic_base.log
|
||||
import cic_base.argparse
|
||||
import cic_base.rpc
|
||||
from cic_base.eth.syncer import chain_interface
|
||||
from cic_eth_registry.error import UnknownContractError
|
||||
from chainlib.chain import ChainSpec
|
||||
|
|
@ -30,8 +23,13 @@ from chainsyncer.backend.sql import SQLBackend
|
|||
from chainsyncer.driver.head import HeadSyncer
|
||||
from chainsyncer.driver.history import HistorySyncer
|
||||
from chainsyncer.db.models.base import SessionBase
|
||||
from chainlib.eth.address import (
|
||||
is_checksum_address,
|
||||
to_checksum_address,
|
||||
)
|
||||
|
||||
# local imports
|
||||
import cic_eth.cli
|
||||
from cic_eth.db import dsn_from_config
|
||||
from cic_eth.runnable.daemons.filters import (
|
||||
CallbackFilter,
|
||||
|
|
@ -47,61 +45,52 @@ from cic_eth.registry import (
|
|||
connect_token_registry,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
logging.getLogger('confini').setLevel(logging.WARNING)
|
||||
logging.getLogger('gnupg').setLevel(logging.WARNING)
|
||||
|
||||
script_dir = os.path.realpath(os.path.dirname(__file__))
|
||||
arg_flags = cic_eth.cli.argflag_std_read
|
||||
local_arg_flags = cic_eth.cli.argflag_local_sync
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
def add_block_args(argparser):
|
||||
argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
|
||||
argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
return argparser
|
||||
# process config
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
# connect to celery
|
||||
cic_eth.cli.CeleryApp.from_config(config)
|
||||
|
||||
logg = cic_base.log.create()
|
||||
argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
|
||||
argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
|
||||
args = cic_base.argparse.parse(argparser, logg)
|
||||
|
||||
config = cic_base.config.create(args.c, args, args.env_prefix)
|
||||
|
||||
config.add(args.y, '_KEYSTORE_FILE', True)
|
||||
config.add(args.q, '_CELERY_QUEUE', True)
|
||||
config.add(args.history_start, 'SYNCER_HISTORY_START', True)
|
||||
config.add(args.no_history, '_NO_HISTORY', True)
|
||||
|
||||
cic_base.config.log(config)
|
||||
|
||||
# set up database
|
||||
dsn = dsn_from_config(config)
|
||||
|
||||
SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
# set up rpc
|
||||
rpc = cic_eth.cli.RPC.from_config(config)
|
||||
conn = rpc.get_default()
|
||||
|
||||
cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
|
||||
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
# set up chain provisions
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
registry = None
|
||||
try:
|
||||
registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
except UnknownContractError as e:
|
||||
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
|
||||
sys.exit(1)
|
||||
logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
|
||||
|
||||
|
||||
def main():
|
||||
# connect to celery
|
||||
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
|
||||
|
||||
# Connect to blockchain with chainlib
|
||||
|
||||
o = block_latest()
|
||||
r = rpc.do(o)
|
||||
r = conn.do(o)
|
||||
block_current = int(r, 16)
|
||||
block_offset = block_current + 1
|
||||
|
||||
loop_interval = config.get('SYNCER_LOOP_INTERVAL')
|
||||
if loop_interval == None:
|
||||
stat = init_chain_stat(rpc, block_start=block_current)
|
||||
stat = init_chain_stat(conn, block_start=block_current)
|
||||
loop_interval = stat.block_average()
|
||||
|
||||
logg.debug('current block height {}'.format(block_offset))
|
||||
|
|
@ -134,34 +123,39 @@ def main():
|
|||
logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
|
||||
syncers.append(HeadSyncer(syncer_backend, chain_interface))
|
||||
|
||||
connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
|
||||
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
|
||||
if trusted_addresses_src == None:
|
||||
logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
|
||||
sys.exit(1)
|
||||
trusted_addresses = trusted_addresses_src.split(',')
|
||||
for address in trusted_addresses:
|
||||
for i, address in enumerate(trusted_addresses):
|
||||
if not config.get('_UNSAFE'):
|
||||
if not is_checksum_address(address):
|
||||
raise ValueError('address {} is not a valid checksum address'.format(address))
|
||||
else:
|
||||
trusted_addresses[i] = to_checksum_address(address)
|
||||
logg.info('using trusted address {}'.format(address))
|
||||
connect_declarator(rpc, chain_spec, trusted_addresses)
|
||||
connect_token_registry(rpc, chain_spec)
|
||||
connect_declarator(conn, chain_spec, trusted_addresses)
|
||||
connect_token_registry(conn, chain_spec)
|
||||
CallbackFilter.trusted_addresses = trusted_addresses
|
||||
|
||||
callback_filters = []
|
||||
for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
|
||||
task_split = cb.split(':')
|
||||
task_queue = config.get('_CELERY_QUEUE')
|
||||
task_queue = config.get('CELERY_QUEUE')
|
||||
if len(task_split) > 1:
|
||||
task_queue = task_split[0]
|
||||
callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
|
||||
callback_filters.append(callback_filter)
|
||||
|
||||
tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE'))
|
||||
tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE'))
|
||||
|
||||
account_registry_address = registry.by_name('AccountRegistry')
|
||||
registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE'))
|
||||
registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE'))
|
||||
|
||||
gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE'))
|
||||
gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
|
||||
|
||||
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
|
||||
|
||||
|
|
@ -176,7 +170,7 @@ def main():
|
|||
for cf in callback_filters:
|
||||
syncer.add_filter(cf)
|
||||
|
||||
r = syncer.loop(int(loop_interval), rpc)
|
||||
r = syncer.loop(int(loop_interval), conn)
|
||||
sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
|
||||
|
||||
i += 1
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
[bancor]
|
||||
dir = /usr/local/share/cic/bancor
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
[celery]
|
||||
broker_url = redis://
|
||||
result_url = redis://
|
||||
debug = 0
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
[cic]
|
||||
registry_address =
|
||||
chain_spec = evm:bloxberg:8996
|
||||
tx_retry_delay =
|
||||
trust_address =
|
||||
default_token_symbol = GFT
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||
run_dir = /run
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[custody]
|
||||
account_index_address =
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[database]
|
||||
NAME=cic-eth
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
POOL_SIZE=50
|
||||
DEBUG=0
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[dispatcher]
|
||||
loop_interval = 0.9
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[bancor]
|
||||
dir = /usr/local/share/cic/bancor
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
[celery]
|
||||
broker_url = redis://localhost:63379
|
||||
result_url = redis://localhost:63379
|
||||
debug = 0
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
[cic]
|
||||
registry_address =
|
||||
chain_spec = evm:bloxberg:8996
|
||||
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
|
||||
tx_retry_delay = 20
|
||||
default_token_symbol = GFT
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||
run_dir = /run
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[custody]
|
||||
account_index_address =
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[dispatcher]
|
||||
loop_interval = 0.9
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[eth]
|
||||
provider = http://localhost:63545
|
||||
gas_gifter_minimum_balance = 10000000000000000000000
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
[redis]
|
||||
host = localhost
|
||||
port = 63379
|
||||
db = 0
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
[signer]
|
||||
socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
|
||||
secret = deedbeef
|
||||
database_name = signer_test
|
||||
dev_keys_path =
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
[SSL]
|
||||
enable_client = false
|
||||
cert_file =
|
||||
key_file =
|
||||
password =
|
||||
ca_file =
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[SYNCER]
|
||||
loop_interval =
|
||||
history_start = 0
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[eth]
|
||||
provider = http://localhost:8545
|
||||
gas_gifter_minimum_balance = 10000000000000000000000
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[bancor]
|
||||
registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549
|
||||
dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
[database]
|
||||
NAME=cic-eth
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=sqlite
|
||||
DRIVER=pysqlite
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[eth]
|
||||
gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
|
||||
provider = http://localhost:8545
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[celery]
|
||||
broker_url = redis://
|
||||
result_url = file://
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[signer]
|
||||
socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
[SSL]
|
||||
enable_client = true
|
||||
cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt
|
||||
key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key
|
||||
password = test
|
||||
ca_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[SYNCER]
|
||||
loop_interval = 1
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
[redis]
|
||||
host =
|
||||
port =
|
||||
db =
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
[signer]
|
||||
socket_path = /run/crypto-dev-signer/jsonrpc.ipc
|
||||
secret = deedbeef
|
||||
database_name = signer_test
|
||||
dev_keys_path =
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
[SSL]
|
||||
enable_client = false
|
||||
cert_file =
|
||||
key_file =
|
||||
password =
|
||||
ca_file =
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[SYNCER]
|
||||
loop_interval =
|
||||
history_start = 0
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
[tasks]
|
||||
transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
|
||||
trace_queue_status = 1
|
||||
Loading…
Reference in New Issue