Replace cic-base in tracker

This commit is contained in:
nolash 2021-07-23 06:34:18 +02:00
parent d6157652c1
commit f5e422cadd
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
37 changed files with 115 additions and 169 deletions

View File

@ -14,6 +14,12 @@ from chainlib.eth.cli import (
ArgumentParser as BaseArgumentParser, ArgumentParser as BaseArgumentParser,
Flag, Flag,
) )
from chainlib.connection import (
RPCConnection,
ConnType,
)
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthUnixSignerConnection
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
@ -24,14 +30,22 @@ class CICFlag(enum.IntEnum):
# celery - nibble 1 # celery - nibble 1
CELERY = 1 CELERY = 1
CELERY_QUEUE = 2
# redis - nibble 2 # redis - nibble 2
REDIS = 16 REDIS = 16
REDIS_CALLBACK = 32 REDIS_CALLBACK = 32
# chain - nibble 3
CHAIN = 256
# sync - nibble 4
SYNCER = 4096
argflag_local_task = CICFlag.CELERY argflag_local_task = CICFlag.CELERY
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
argflag_local_chain = CICFlag.CHAIN
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
class Config(BaseConfig): class Config(BaseConfig):
@ -54,6 +68,10 @@ class Config(BaseConfig):
local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
local_args_override['REDIS_DB'] = getattr(args, 'redis_db') local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
if local_arg_flags & CICFlag.CHAIN:
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
if local_arg_flags & CICFlag.CELERY: if local_arg_flags & CICFlag.CELERY:
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
config.dict_override(local_args_override, 'local cli args') config.dict_override(local_args_override, 'local cli args')
@ -65,6 +83,8 @@ class Config(BaseConfig):
if local_arg_flags & CICFlag.CELERY: if local_arg_flags & CICFlag.CELERY:
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
logg.debug('config loaded:\n{}'.format(config)) logg.debug('config loaded:\n{}'.format(config))
return config return config
@ -83,6 +103,11 @@ class ArgumentParser(BaseArgumentParser):
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
if local_arg_flags & CICFlag.CELERY: if local_arg_flags & CICFlag.CELERY:
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
if local_arg_flags & CICFlag.SYNCER:
self.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
if local_arg_flags & CICFlag.CHAIN:
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
class CeleryApp: class CeleryApp:
@ -100,3 +125,31 @@ class CeleryApp:
logg.info('creating celery app without results backend on {}'.format(broker_url)) logg.info('creating celery app without results backend on {}'.format(broker_url))
return celery_app return celery_app
class RPC:
def __init__(self, chain_spec, rpc_provider, signer_provider=None):
self.chain_spec = chain_spec
self.rpc_provider = rpc_provider
self.signer_provider = signer_provider
def get_default(self):
return RPCConnection.connect(self.chain_spec, 'default')
@staticmethod
def from_config(config):
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
if config.get('SIGNER_PROVIDER'):
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
logg.info('set up rpc: {}'.format(rpc))
return rpc
def __str__(self):
return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)

View File

@ -1,2 +1,3 @@
[cic] [cic]
registry_address = registry_address =
trust_address =

View File

@ -0,0 +1,10 @@
[database]
engine =
driver =
host =
port =
name =
user =
password =
debug = 0
pool_size = 0

View File

@ -0,0 +1,2 @@
[signer]
provider =

View File

@ -0,0 +1,4 @@
[syncer]
loop_interval = 1
block_offset =
no_history = 0

View File

@ -17,7 +17,7 @@ import cic_eth.cli
from cic_eth.api import Api from cic_eth.api import Api
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger('create_account_script') logg = logging.getLogger()
logging.getLogger('confini').setLevel(logging.WARNING) logging.getLogger('confini').setLevel(logging.WARNING)
logging.getLogger('gnupg').setLevel(logging.WARNING) logging.getLogger('gnupg').setLevel(logging.WARNING)

View File

@ -8,13 +8,6 @@ import sys
import re import re
# external imports # external imports
import confini
import celery
import rlp
import cic_base.config
import cic_base.log
import cic_base.argparse
import cic_base.rpc
from cic_base.eth.syncer import chain_interface from cic_base.eth.syncer import chain_interface
from cic_eth_registry.error import UnknownContractError from cic_eth_registry.error import UnknownContractError
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
@ -30,8 +23,13 @@ from chainsyncer.backend.sql import SQLBackend
from chainsyncer.driver.head import HeadSyncer from chainsyncer.driver.head import HeadSyncer
from chainsyncer.driver.history import HistorySyncer from chainsyncer.driver.history import HistorySyncer
from chainsyncer.db.models.base import SessionBase from chainsyncer.db.models.base import SessionBase
from chainlib.eth.address import (
is_checksum_address,
to_checksum_address,
)
# local imports # local imports
import cic_eth.cli
from cic_eth.db import dsn_from_config from cic_eth.db import dsn_from_config
from cic_eth.runnable.daemons.filters import ( from cic_eth.runnable.daemons.filters import (
CallbackFilter, CallbackFilter,
@ -47,61 +45,52 @@ from cic_eth.registry import (
connect_token_registry, connect_token_registry,
) )
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
logging.getLogger('confini').setLevel(logging.WARNING)
logging.getLogger('gnupg').setLevel(logging.WARNING)
script_dir = os.path.realpath(os.path.dirname(__file__)) arg_flags = cic_eth.cli.argflag_std_read
local_arg_flags = cic_eth.cli.argflag_local_sync
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
def add_block_args(argparser): # process config
argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
return argparser
# connect to celery
cic_eth.cli.CeleryApp.from_config(config)
logg = cic_base.log.create() # set up database
argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
args = cic_base.argparse.parse(argparser, logg)
config = cic_base.config.create(args.c, args, args.env_prefix)
config.add(args.y, '_KEYSTORE_FILE', True)
config.add(args.q, '_CELERY_QUEUE', True)
config.add(args.history_start, 'SYNCER_HISTORY_START', True)
config.add(args.no_history, '_NO_HISTORY', True)
cic_base.config.log(config)
dsn = dsn_from_config(config) dsn = dsn_from_config(config)
SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG')) SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG'))
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) # set up rpc
rpc = cic_eth.cli.RPC.from_config(config)
conn = rpc.get_default()
cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) # set up chain provisions
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
rpc = RPCConnection.connect(chain_spec, 'default')
registry = None registry = None
try: try:
registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
except UnknownContractError as e: except UnknownContractError as e:
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
sys.exit(1) sys.exit(1)
logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
def main(): def main():
# connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
# Connect to blockchain with chainlib # Connect to blockchain with chainlib
o = block_latest() o = block_latest()
r = rpc.do(o) r = conn.do(o)
block_current = int(r, 16) block_current = int(r, 16)
block_offset = block_current + 1 block_offset = block_current + 1
loop_interval = config.get('SYNCER_LOOP_INTERVAL') loop_interval = config.get('SYNCER_LOOP_INTERVAL')
if loop_interval == None: if loop_interval == None:
stat = init_chain_stat(rpc, block_start=block_current) stat = init_chain_stat(conn, block_start=block_current)
loop_interval = stat.block_average() loop_interval = stat.block_average()
logg.debug('current block height {}'.format(block_offset)) logg.debug('current block height {}'.format(block_offset))
@ -134,34 +123,39 @@ def main():
logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
syncers.append(HeadSyncer(syncer_backend, chain_interface)) syncers.append(HeadSyncer(syncer_backend, chain_interface))
connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None: if trusted_addresses_src == None:
logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
sys.exit(1) sys.exit(1)
trusted_addresses = trusted_addresses_src.split(',') trusted_addresses = trusted_addresses_src.split(',')
for address in trusted_addresses: for i, address in enumerate(trusted_addresses):
if not config.get('_UNSAFE'):
if not is_checksum_address(address):
raise ValueError('address {} is not a valid checksum address'.format(address))
else:
trusted_addresses[i] = to_checksum_address(address)
logg.info('using trusted address {}'.format(address)) logg.info('using trusted address {}'.format(address))
connect_declarator(rpc, chain_spec, trusted_addresses) connect_declarator(conn, chain_spec, trusted_addresses)
connect_token_registry(rpc, chain_spec) connect_token_registry(conn, chain_spec)
CallbackFilter.trusted_addresses = trusted_addresses CallbackFilter.trusted_addresses = trusted_addresses
callback_filters = [] callback_filters = []
for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','): for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
task_split = cb.split(':') task_split = cb.split(':')
task_queue = config.get('_CELERY_QUEUE') task_queue = config.get('CELERY_QUEUE')
if len(task_split) > 1: if len(task_split) > 1:
task_queue = task_split[0] task_queue = task_split[0]
callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue) callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue)
callback_filters.append(callback_filter) callback_filters.append(callback_filter)
tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE')) tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE'))
account_registry_address = registry.by_name('AccountRegistry') account_registry_address = registry.by_name('AccountRegistry')
registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE')) registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE'))
gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE')) gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
@ -176,7 +170,7 @@ def main():
for cf in callback_filters: for cf in callback_filters:
syncer.add_filter(cf) syncer.add_filter(cf)
r = syncer.loop(int(loop_interval), rpc) r = syncer.loop(int(loop_interval), conn)
sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
i += 1 i += 1

View File

@ -1,2 +0,0 @@
[bancor]
dir = /usr/local/share/cic/bancor

View File

@ -1,4 +0,0 @@
[celery]
broker_url = redis://
result_url = redis://
debug = 0

View File

@ -1,8 +0,0 @@
[cic]
registry_address =
chain_spec = evm:bloxberg:8996
tx_retry_delay =
trust_address =
default_token_symbol = GFT
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

View File

@ -1,2 +0,0 @@
[custody]
account_index_address =

View File

@ -1,10 +0,0 @@
[database]
NAME=cic-eth
USER=postgres
PASSWORD=
HOST=localhost
PORT=5432
ENGINE=postgresql
DRIVER=psycopg2
POOL_SIZE=50
DEBUG=0

View File

@ -1,2 +0,0 @@
[dispatcher]
loop_interval = 0.9

View File

@ -1,2 +0,0 @@
[bancor]
dir = /usr/local/share/cic/bancor

View File

@ -1,4 +0,0 @@
[celery]
broker_url = redis://localhost:63379
result_url = redis://localhost:63379
debug = 0

View File

@ -1,8 +0,0 @@
[cic]
registry_address =
chain_spec = evm:bloxberg:8996
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
tx_retry_delay = 20
default_token_symbol = GFT
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

View File

@ -1,2 +0,0 @@
[custody]
account_index_address =

View File

@ -1,2 +0,0 @@
[dispatcher]
loop_interval = 0.9

View File

@ -1,3 +0,0 @@
[eth]
provider = http://localhost:63545
gas_gifter_minimum_balance = 10000000000000000000000

View File

@ -1,4 +0,0 @@
[redis]
host = localhost
port = 63379
db = 0

View File

@ -1,5 +0,0 @@
[signer]
socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
secret = deedbeef
database_name = signer_test
dev_keys_path =

View File

@ -1,6 +0,0 @@
[SSL]
enable_client = false
cert_file =
key_file =
password =
ca_file =

View File

@ -1,3 +0,0 @@
[SYNCER]
loop_interval =
history_start = 0

View File

@ -1,3 +0,0 @@
[eth]
provider = http://localhost:8545
gas_gifter_minimum_balance = 10000000000000000000000

View File

@ -1,3 +0,0 @@
[bancor]
registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549
dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6

View File

@ -1,8 +0,0 @@
[database]
NAME=cic-eth
USER=postgres
PASSWORD=
HOST=localhost
PORT=5432
ENGINE=sqlite
DRIVER=pysqlite

View File

@ -1,3 +0,0 @@
[eth]
gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
provider = http://localhost:8545

View File

@ -1,3 +0,0 @@
[celery]
broker_url = redis://
result_url = file://

View File

@ -1,2 +0,0 @@
[signer]
socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc

View File

@ -1,6 +0,0 @@
[SSL]
enable_client = true
cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt
key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key
password = test
ca_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt

View File

@ -1,2 +0,0 @@
[SYNCER]
loop_interval = 1

View File

@ -1,4 +0,0 @@
[redis]
host =
port =
db =

View File

@ -1,5 +0,0 @@
[signer]
socket_path = /run/crypto-dev-signer/jsonrpc.ipc
secret = deedbeef
database_name = signer_test
dev_keys_path =

View File

@ -1,6 +0,0 @@
[SSL]
enable_client = false
cert_file =
key_file =
password =
ca_file =

View File

@ -1,3 +0,0 @@
[SYNCER]
loop_interval =
history_start = 0

View File

@ -1,3 +0,0 @@
[tasks]
transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop
trace_queue_status = 1