Import cic-eth cli sources

This commit is contained in:
lash 2022-04-24 15:25:58 +00:00
parent b2421cce8a
commit c55716cb9b
Signed by: lash
GPG Key ID: 21D2E7BB88C2A746
16 changed files with 398 additions and 3 deletions

10
cic_base/cli/__init__.py Normal file
View File

@ -0,0 +1,10 @@
# local imports
from .base import *
from .chain import (
EthChainInterface,
chain_interface,
)
from .rpc import RPC
from .arg import ArgumentParser
from .config import Config
from .celery import CeleryApp

39
cic_base/cli/arg.py Normal file
View File

@ -0,0 +1,39 @@
# external imports
from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
# local imports
from .base import CICFlag, Flag
class ArgumentParser(BaseArgumentParser):
def process_local_flags(self, local_arg_flags):
if local_arg_flags & CICFlag.REDIS:
self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
if local_arg_flags & CICFlag.REDIS_CALLBACK:
self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
if local_arg_flags & CICFlag.CELERY:
self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
if local_arg_flags & CICFlag.SERVER:
self.add_argument('--server-port', type=int, default=5000, help='Server port')
self.add_argument('--server-host', type=str, default="0.0.0.0", help='Server host')
self.add_argument('--server-workers', type=int, default=1, help='The number of worker processes for handling requests')
self.add_argument('--server-config', type=str, default=None, help='Gunicorn config file, or python module. It will override all other server args. (see https://docs.gunicorn.org/en/19.2.1/settings.html#config-file)')
if local_arg_flags & CICFlag.SYNCER:
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
if local_arg_flags & CICFlag.CHAIN:
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

35
cic_base/cli/base.py Normal file
View File

@ -0,0 +1,35 @@
# standard imports
import enum
# external imports
from chainlib.eth.cli import (
argflag_std_read,
argflag_std_write,
argflag_std_base,
Flag,
)
class CICFlag(enum.IntEnum):
# celery - nibble 1
CELERY = 1
# redis - nibble 2
REDIS = 16
REDIS_CALLBACK = 32
# chain - nibble 3
CHAIN = 256
# sync - nibble 4
SYNCER = 4096
# server - nibble 5
SERVER=65536
argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
argflag_local_task = CICFlag.CELERY
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
argflag_local_chain = CICFlag.CHAIN
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
argflag_local_server = CICFlag.SERVER | CICFlag.REDIS | CICFlag.REDIS_CALLBACK | CICFlag.CELERY | Flag.CHAIN_SPEC

24
cic_base/cli/celery.py Normal file
View File

@ -0,0 +1,24 @@
# standard imports
import logging
# external imports
import celery
logg = logging.getLogger(__name__)
class CeleryApp:
@classmethod
def from_config(cls, config):
backend_url = config.get('CELERY_RESULT_URL')
broker_url = config.get('CELERY_BROKER_URL')
celery_app = None
if backend_url != None:
celery_app = celery.Celery(broker=broker_url, backend=backend_url)
logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
else:
celery_app = celery.Celery(broker=broker_url)
logg.info('creating celery app without results backend on {}'.format(broker_url))
return celery_app

21
cic_base/cli/chain.py Normal file
View File

@ -0,0 +1,21 @@
# external imports
from chainlib.eth.block import (
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
Tx,
)
from chainlib.interface import ChainInterface
class EthChainInterface(ChainInterface):
def __init__(self):
self._tx_receipt = receipt
self._block_by_number = block_by_number
self._block_from_src = Block.from_src
self._src_normalize = Tx.src_normalize
chain_interface = EthChainInterface()

130
cic_base/cli/config.py Normal file
View File

@ -0,0 +1,130 @@
# standard imports
import os
import logging
import urllib.parse
import copy
# external imports
from chainlib.eth.cli import (
Config as BaseConfig,
Flag,
)
from urlybird.merge import (
urlhostmerge,
urlmerge,
)
# local imports
from .base import CICFlag
script_dir = os.path.dirname(os.path.realpath(__file__))
logg = logging.getLogger(__name__)
class Config(BaseConfig):
local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
@classmethod
def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
expanded_base_config_dir = [cls.local_base_config_dir]
if base_config_dir != None:
if isinstance(base_config_dir, str):
base_config_dir = [base_config_dir]
for d in base_config_dir:
expanded_base_config_dir.append(d)
config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
local_args_override = {}
if local_arg_flags & CICFlag.REDIS:
local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
if local_arg_flags & CICFlag.CHAIN:
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
if local_arg_flags & CICFlag.SERVER:
local_args_override['SERVER_PORT'] = getattr(args, 'server_port')
local_args_override['SERVER_HOST'] = getattr(args, 'server_host')
local_args_override['SERVER_WORKERS'] = getattr(args, 'server_workers')
local_args_override['SERVER_CONFIG'] = getattr(args, 'server_config')
if local_arg_flags & CICFlag.CELERY:
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
if local_arg_flags & CICFlag.SYNCER:
local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
config.dict_override(local_args_override, 'local cli args')
local_celery_args_override = {}
if local_arg_flags & CICFlag.CELERY:
hostport = urlhostmerge(
None,
config.get('REDIS_HOST'),
config.get('REDIS_PORT'),
)
db = getattr(args, 'redis_db', None)
if db != None:
db = str(db)
redis_url = (
'redis',
hostport,
db,
)
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
hostport = urlhostmerge(
celery_config_url[1],
getattr(args, 'celery_host', None),
getattr(args, 'celery_port', None),
)
db = getattr(args, 'redis_db', None)
if db != None:
db = str(db)
celery_arg_url = (
getattr(args, 'celery_scheme', None),
hostport,
db,
)
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
celery_url_string = urllib.parse.urlunsplit(celery_url)
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
if not getattr(args, 'celery_no_result'):
local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
if local_celery_args_override['CELERY_RESULT_URL'] == None:
local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
hostport = urlhostmerge(
celery_config_url[1],
getattr(args, 'celery_result_host', None),
getattr(args, 'celery_result_port', None),
)
celery_arg_url = (
getattr(args, 'celery_result_scheme', None),
hostport,
getattr(args, 'celery_result_db', None),
)
celery_url = urlmerge(celery_config_url, celery_arg_url)
logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
celery_url_string = urllib.parse.urlunsplit(celery_url)
local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
config.dict_override(local_celery_args_override, 'local celery cli args')
if local_arg_flags & CICFlag.REDIS_CALLBACK:
redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
logg.debug('config loaded:\n{}'.format(config))
return config

90
cic_base/cli/rpc.py Normal file
View File

@ -0,0 +1,90 @@
# standard imports
import logging
# external imports
from chainlib.connection import (
RPCConnection,
ConnType,
)
from chainlib.eth.connection import (
EthUnixSignerConnection,
EthHTTPSignerConnection,
)
from chainlib.chain import ChainSpec
logg = logging.getLogger(__name__)
class RPC:
def __init__(self, chain_spec, rpc_provider, signer_provider=None):
self.chain_spec = chain_spec
self.rpc_provider = rpc_provider
self.signer_provider = signer_provider
def get_default(self):
return self.get_by_label('default')
def get_by_label(self, label):
return RPCConnection.connect(self.chain_spec, label)
@staticmethod
def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
if use_signer:
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label)
rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
logg.info('set up rpc: {}'.format(rpc))
return rpc
def __str__(self):
return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
# TOOD: re-implement file backend option from omittec code:
#broker = config.get('CELERY_BROKER_URL')
#if broker[:4] == 'file':
# bq = tempfile.mkdtemp()
# bp = tempfile.mkdtemp()
# conf_update = {
# 'broker_url': broker,
# 'broker_transport_options': {
# 'data_folder_in': bq,
# 'data_folder_out': bq,
# 'data_folder_processed': bp,
# },
# }
# if config.true('CELERY_DEBUG'):
# conf_update['result_extended'] = True
# current_app.conf.update(conf_update)
# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
#else:
# conf_update = {
# 'broker_url': broker,
# }
# if config.true('CELERY_DEBUG'):
# conf_update['result_extended'] = True
# current_app.conf.update(conf_update)
#
#result = config.get('CELERY_RESULT_URL')
#if result[:4] == 'file':
# rq = tempfile.mkdtemp()
# current_app.conf.update({
# 'result_backend': 'file://{}'.format(rq),
# })
# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
#else:
# current_app.conf.update({
# 'result_backend': result,
# })
#

View File

@ -0,0 +1,8 @@
[celery]
broker_url =
result_url =
queue = cic-eth
debug = 0
worker_pool = prefork
worker_count = 0
debug_log = 0

View File

@ -0,0 +1,6 @@
[cic]
registry_address =
trust_address =
default_token_symbol =
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
run_dir = /run

View File

@ -0,0 +1,10 @@
[database]
engine =
driver =
host =
port =
name =
user =
password =
debug = 0
pool_size = 0

View File

@ -0,0 +1,8 @@
[eth]
gas_holder_minimum_units = 180000
gas_holder_refill_units = 15
gas_holder_refill_threshold = 3
gas_gifter_refill_buffer = 3
gas_gift_min_price_buffer = 20
min_fee_price = 1
max_fee_units = 8000000

View File

@ -0,0 +1,5 @@
[redis]
host = localhost
port = 6379
db = 0
timeout = 20.0

View File

@ -0,0 +1,5 @@
[server]
port=5000
host="0.0.0.0"
workers=1
config=

View File

@ -0,0 +1,2 @@
[signer]
provider =

View File

@ -2,7 +2,6 @@
import logging import logging
# external imports # external imports
import cic_eth.cli
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainlib.eth.address import is_checksum_address from chainlib.eth.address import is_checksum_address
from cic_eth.registry import ( from cic_eth.registry import (
@ -14,6 +13,7 @@ from cic_eth_registry import CICRegistry
from cic_eth_registry.error import UnknownContractError from cic_eth_registry.error import UnknownContractError
# legacy imports # legacy imports
import cic_base.cli
from cic_base.legacy.db import SessionBase from cic_base.legacy.db import SessionBase
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)
@ -30,12 +30,12 @@ class CICSettings:
def process_common(self, config): def process_common(self, config):
self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) self.o['CHAIN_SPEC'] = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
rpc = cic_eth.cli.RPC.from_config(config) rpc = cic_base.cli.RPC.from_config(config)
self.o['RPC'] = rpc.get_default() self.o['RPC'] = rpc.get_default()
def process_celery(self, config): def process_celery(self, config):
cic_eth.cli.CeleryApp.from_config(config) cic_base.cli.CeleryApp.from_config(config)
self.o['CELERY_QUEUE'] = config.get('CELERY_QUEUE') self.o['CELERY_QUEUE'] = config.get('CELERY_QUEUE')

View File

@ -1,2 +1,4 @@
cic-eth-registry~=0.6.9 cic-eth-registry~=0.6.9
celery~=4.4.7 celery~=4.4.7
chainlib-eth~=0.1.0
urlybird~=0.0.2