Import cic-eth cli sources
This commit is contained in:
lash
parent
b2421cce8a
commit
c55716cb9b
GPG Key ID:
21D2E7BB88C2A746
|
|
@ -0,0 +1,10 @@
|
|||
# local imports
|
||||
from .base import *
|
||||
from .chain import (
|
||||
EthChainInterface,
|
||||
chain_interface,
|
||||
)
|
||||
from .rpc import RPC
|
||||
from .arg import ArgumentParser
|
||||
from .config import Config
|
||||
from .celery import CeleryApp
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
# external imports
|
||||
from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
|
||||
|
||||
# local imports
|
||||
from .base import CICFlag, Flag
|
||||
|
||||
|
||||
class ArgumentParser(BaseArgumentParser):
|
||||
|
||||
def process_local_flags(self, local_arg_flags):
|
||||
if local_arg_flags & CICFlag.REDIS:
|
||||
self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
|
||||
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
|
||||
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
|
||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||
self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
|
||||
self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
|
||||
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
|
||||
self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
|
||||
self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
|
||||
self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
|
||||
self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
|
||||
self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
|
||||
self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
|
||||
self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
|
||||
self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
|
||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
|
||||
if local_arg_flags & CICFlag.SERVER:
|
||||
self.add_argument('--server-port', type=int, default=5000, help='Server port')
|
||||
self.add_argument('--server-host', type=str, default="0.0.0.0", help='Server host')
|
||||
self.add_argument('--server-workers', type=int, default=1, help='The number of worker processes for handling requests')
|
||||
self.add_argument('--server-config', type=str, default=None, help='Gunicorn config file, or python module. It will override all other server args. (see https://docs.gunicorn.org/en/19.2.1/settings.html#config-file)')
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# standard imports
|
||||
import enum
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.cli import (
|
||||
argflag_std_read,
|
||||
argflag_std_write,
|
||||
argflag_std_base,
|
||||
Flag,
|
||||
)
|
||||
|
||||
class CICFlag(enum.IntEnum):
|
||||
|
||||
# celery - nibble 1
|
||||
CELERY = 1
|
||||
|
||||
# redis - nibble 2
|
||||
REDIS = 16
|
||||
REDIS_CALLBACK = 32
|
||||
|
||||
# chain - nibble 3
|
||||
CHAIN = 256
|
||||
|
||||
# sync - nibble 4
|
||||
SYNCER = 4096
|
||||
|
||||
# server - nibble 5
|
||||
SERVER=65536
|
||||
|
||||
argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
||||
argflag_local_chain = CICFlag.CHAIN
|
||||
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
|
||||
argflag_local_server = CICFlag.SERVER | CICFlag.REDIS | CICFlag.REDIS_CALLBACK | CICFlag.CELERY | Flag.CHAIN_SPEC
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CeleryApp:
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
backend_url = config.get('CELERY_RESULT_URL')
|
||||
broker_url = config.get('CELERY_BROKER_URL')
|
||||
celery_app = None
|
||||
if backend_url != None:
|
||||
celery_app = celery.Celery(broker=broker_url, backend=backend_url)
|
||||
logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
|
||||
else:
|
||||
celery_app = celery.Celery(broker=broker_url)
|
||||
logg.info('creating celery app without results backend on {}'.format(broker_url))
|
||||
|
||||
return celery_app
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# external imports
|
||||
from chainlib.eth.block import (
|
||||
block_by_number,
|
||||
Block,
|
||||
)
|
||||
from chainlib.eth.tx import (
|
||||
receipt,
|
||||
Tx,
|
||||
)
|
||||
from chainlib.interface import ChainInterface
|
||||
|
||||
|
||||
class EthChainInterface(ChainInterface):
|
||||
|
||||
def __init__(self):
|
||||
self._tx_receipt = receipt
|
||||
self._block_by_number = block_by_number
|
||||
self._block_from_src = Block.from_src
|
||||
self._src_normalize = Tx.src_normalize
|
||||
|
||||
chain_interface = EthChainInterface()
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import urllib.parse
|
||||
import copy
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.cli import (
|
||||
Config as BaseConfig,
|
||||
Flag,
|
||||
)
|
||||
from urlybird.merge import (
|
||||
urlhostmerge,
|
||||
urlmerge,
|
||||
)
|
||||
|
||||
# local imports
|
||||
from .base import CICFlag
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Config(BaseConfig):
|
||||
|
||||
local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
|
||||
|
||||
@classmethod
|
||||
def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
|
||||
expanded_base_config_dir = [cls.local_base_config_dir]
|
||||
if base_config_dir != None:
|
||||
if isinstance(base_config_dir, str):
|
||||
base_config_dir = [base_config_dir]
|
||||
for d in base_config_dir:
|
||||
expanded_base_config_dir.append(d)
|
||||
config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
|
||||
|
||||
local_args_override = {}
|
||||
if local_arg_flags & CICFlag.REDIS:
|
||||
local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
|
||||
local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
|
||||
local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
|
||||
local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
|
||||
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
||||
if local_arg_flags & CICFlag.SERVER:
|
||||
local_args_override['SERVER_PORT'] = getattr(args, 'server_port')
|
||||
local_args_override['SERVER_HOST'] = getattr(args, 'server_host')
|
||||
local_args_override['SERVER_WORKERS'] = getattr(args, 'server_workers')
|
||||
local_args_override['SERVER_CONFIG'] = getattr(args, 'server_config')
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
||||
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
|
||||
local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
|
||||
|
||||
config.dict_override(local_args_override, 'local cli args')
|
||||
|
||||
local_celery_args_override = {}
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
hostport = urlhostmerge(
|
||||
None,
|
||||
config.get('REDIS_HOST'),
|
||||
config.get('REDIS_PORT'),
|
||||
)
|
||||
db = getattr(args, 'redis_db', None)
|
||||
if db != None:
|
||||
db = str(db)
|
||||
|
||||
redis_url = (
|
||||
'redis',
|
||||
hostport,
|
||||
db,
|
||||
)
|
||||
|
||||
|
||||
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
|
||||
hostport = urlhostmerge(
|
||||
celery_config_url[1],
|
||||
getattr(args, 'celery_host', None),
|
||||
getattr(args, 'celery_port', None),
|
||||
)
|
||||
db = getattr(args, 'redis_db', None)
|
||||
if db != None:
|
||||
db = str(db)
|
||||
celery_arg_url = (
|
||||
getattr(args, 'celery_scheme', None),
|
||||
hostport,
|
||||
db,
|
||||
)
|
||||
|
||||
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
|
||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
||||
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
|
||||
if not getattr(args, 'celery_no_result'):
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
|
||||
if local_celery_args_override['CELERY_RESULT_URL'] == None:
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
|
||||
celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
|
||||
hostport = urlhostmerge(
|
||||
celery_config_url[1],
|
||||
getattr(args, 'celery_result_host', None),
|
||||
getattr(args, 'celery_result_port', None),
|
||||
)
|
||||
celery_arg_url = (
|
||||
getattr(args, 'celery_result_scheme', None),
|
||||
hostport,
|
||||
getattr(args, 'celery_result_db', None),
|
||||
)
|
||||
celery_url = urlmerge(celery_config_url, celery_arg_url)
|
||||
logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
|
||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
|
||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||
|
||||
config.dict_override(local_celery_args_override, 'local celery cli args')
|
||||
|
||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||
redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
|
||||
redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
|
||||
config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
|
||||
config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
|
||||
|
||||
logg.debug('config loaded:\n{}'.format(config))
|
||||
|
||||
return config
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.connection import (
|
||||
RPCConnection,
|
||||
ConnType,
|
||||
)
|
||||
from chainlib.eth.connection import (
|
||||
EthUnixSignerConnection,
|
||||
EthHTTPSignerConnection,
|
||||
)
|
||||
from chainlib.chain import ChainSpec
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RPC:
|
||||
|
||||
def __init__(self, chain_spec, rpc_provider, signer_provider=None):
|
||||
self.chain_spec = chain_spec
|
||||
self.rpc_provider = rpc_provider
|
||||
self.signer_provider = signer_provider
|
||||
|
||||
|
||||
def get_default(self):
|
||||
return self.get_by_label('default')
|
||||
|
||||
|
||||
def get_by_label(self, label):
|
||||
return RPCConnection.connect(self.chain_spec, label)
|
||||
|
||||
|
||||
@staticmethod
|
||||
def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
|
||||
RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
|
||||
if use_signer:
|
||||
|
||||
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
|
||||
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
|
||||
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
|
||||
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label)
|
||||
rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
||||
logg.info('set up rpc: {}'.format(rpc))
|
||||
return rpc
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
|
||||
|
||||
|
||||
# TOOD: re-implement file backend option from omittec code:
|
||||
#broker = config.get('CELERY_BROKER_URL')
|
||||
#if broker[:4] == 'file':
|
||||
# bq = tempfile.mkdtemp()
|
||||
# bp = tempfile.mkdtemp()
|
||||
# conf_update = {
|
||||
# 'broker_url': broker,
|
||||
# 'broker_transport_options': {
|
||||
# 'data_folder_in': bq,
|
||||
# 'data_folder_out': bq,
|
||||
# 'data_folder_processed': bp,
|
||||
# },
|
||||
# }
|
||||
# if config.true('CELERY_DEBUG'):
|
||||
# conf_update['result_extended'] = True
|
||||
# current_app.conf.update(conf_update)
|
||||
# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
|
||||
#else:
|
||||
# conf_update = {
|
||||
# 'broker_url': broker,
|
||||
# }
|
||||
# if config.true('CELERY_DEBUG'):
|
||||
# conf_update['result_extended'] = True
|
||||
# current_app.conf.update(conf_update)
|
||||
#
|
||||
#result = config.get('CELERY_RESULT_URL')
|
||||
#if result[:4] == 'file':
|
||||
# rq = tempfile.mkdtemp()
|
||||
# current_app.conf.update({
|
||||
# 'result_backend': 'file://{}'.format(rq),
|
||||
# })
|
||||
# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
|
||||
#else:
|
||||
# current_app.conf.update({
|
||||
# 'result_backend': result,
|
||||
# })
|
||||
#
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[celery]
|
||||
broker_url =
|
||||
result_url =
|
||||
queue = cic-eth
|
||||
debug = 0
|
||||
worker_pool = prefork
|
||||
worker_count = 0
|
||||
debug_log = 0
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
[cic]
|
||||
registry_address =
|
||||
trust_address =
|
||||
default_token_symbol =
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
|
||||
run_dir = /run
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[database]
|
||||
engine =
|
||||
driver =
|
||||
host =
|
||||
port =
|
||||
name =
|
||||
user =
|
||||
password =
|
||||
debug = 0
|
||||
pool_size = 0
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
[eth]
|
||||
gas_holder_minimum_units = 180000
|
||||
gas_holder_refill_units = 15
|
||||
gas_holder_refill_threshold = 3
|
||||
gas_gifter_refill_buffer = 3
|
||||
gas_gift_min_price_buffer = 20
|
||||
min_fee_price = 1
|
||||
max_fee_units = 8000000
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[redis]
|
||||
host = localhost
|
||||
port = 6379
|
||||
db = 0
|
||||
timeout = 20.0
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
[server]
|
||||
port=5000
|
||||
host="0.0.0.0"
|
||||
workers=1
|
||||
config=
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
[signer]
|
||||
provider =
|
||||
|
|
@ -2,7 +2,6 @@
|
|||
import logging
|
||||
|
||||
# external imports
|
||||
import cic_eth.cli
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.address import is_checksum_address
|
||||
from cic_eth.registry import (
|
||||
|
|
@ -14,6 +13,7 @@ from cic_eth_registry import CICRegistry
|
|||
from cic_eth_registry.error import UnknownContractError
|
||||
|
||||
# legacy imports
|
||||
import cic_base.cli
|
||||
from cic_base.legacy.db import SessionBase
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
|
@ -30,12 +30,12 @@ class CICSettings:
|
|||
def process_common(self, config):
|
||||
self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
|
||||
rpc = cic_eth.cli.RPC.from_config(config)
|
||||
rpc = cic_base.cli.RPC.from_config(config)
|
||||
self.o['RPC'] = rpc.get_default()
|
||||
|
||||
|
||||
def process_celery(self, config):
|
||||
cic_eth.cli.CeleryApp.from_config(config)
|
||||
cic_base.cli.CeleryApp.from_config(config)
|
||||
self.o['CELERY_QUEUE'] = config.get('CELERY_QUEUE')
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,2 +1,4 @@
|
|||
cic-eth-registry~=0.6.9
|
||||
celery~=4.4.7
|
||||
chainlib-eth~=0.1.0
|
||||
urlybird~=0.0.2
|
||||
|
|
|
|||
Loading…
Reference in New Issue