grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

migrations: Enable deployment and data seeding to Bloxberg

Browse Source
This commit is contained in:
Louis Holbrook
2021-12-22 18:24:05 +00:00
committed by Philip Wafula
parent b15cfee1c9
commit d7c4cb71eb
65 changed files with 940 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py
View File

@@ -3,3 +3,4 @@ from .tx import TxFilter
from .gas import GasFilter
from .register import RegistrationFilter
from .transferauth import TransferAuthFilter
from .token import TokenFilter

63
apps/cic-eth/cic_eth/runnable/daemons/filters/token.py Normal file
View File

@@ -0,0 +1,63 @@
# standard imports
import logging
# external imports
from eth_erc20 import ERC20
from chainlib.eth.contract import (
ABIContractEncoder,
ABIContractType,
)
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.address import is_same_address
from chainlib.eth.error import RequestMismatchException
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from eth_token_index import TokenUniqueSymbolIndex
import celery
# local imports
from .base import SyncFilter
logg = logging.getLogger(__name__)
class TokenFilter(SyncFilter):
def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
self.queue = queue
self.chain_spec = chain_spec
self.caller_address = call_address
def filter(self, conn, block, tx, db_session=None):
if not tx.payload:
return (None, None)
try:
r = ERC20.parse_transfer_request(tx.payload)
except RequestMismatchException:
return (None, None)
token_address = tx.inputs[0]
token = ERC20Token(self.chain_spec, conn, token_address)
registry = CICRegistry(self.chain_spec, conn)
r = registry.by_name(token.symbol, sender_address=self.caller_address)
if is_same_address(r, ZERO_ADDRESS):
return None
enc = ABIContractEncoder()
enc.method('transfer')
method = enc.get()
s = celery.signature(
'cic_eth.eth.gas.apply_gas_value_cache',
[
token_address,
method,
tx.gas_used,
tx.hash,
],
queue=self.queue,
)
return s.apply_async()

18
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@@ -67,7 +67,10 @@ from cic_eth.registry import (
connect_declarator,
connect_token_registry,
)
from cic_eth.task import BaseTask
from cic_eth.task import (
BaseTask,
CriticalWeb3Task,
)
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@@ -76,18 +79,18 @@ arg_flags = cic_eth.cli.argflag_std_read
local_arg_flags = cic_eth.cli.argflag_local_task
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
#argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei')
argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
args = argparser.parse_args()
# process config
extra_args = {
# 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
'aux_all': None,
'aux': None,
'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
'min_fee_price': 'ETH_MIN_FEE_PRICE',
}
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
@@ -215,6 +218,7 @@ def main():
argv.append('-n')
argv.append(config.get('CELERY_QUEUE'))
# TODO: More elegant way of setting queue-wide settings
BaseTask.default_token_symbol = default_token_symbol
BaseTask.default_token_address = default_token_address
default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
@@ -222,6 +226,14 @@ def main():
BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name
BaseTask.trusted_addresses = trusted_addresses
CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
if config.get('ETH_MIN_FEE_PRICE'):
BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

10
apps/cic-eth/cic_eth/runnable/daemons/tracker.py
View File

@@ -36,6 +36,7 @@ from cic_eth.runnable.daemons.filters import (
TxFilter,
RegistrationFilter,
TransferAuthFilter,
TokenFilter,
)
from cic_eth.stat import init_chain_stat
from cic_eth.registry import (
@@ -99,10 +100,10 @@ def main():
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
if len(syncer_backends) == 0:
initial_block_start = config.get('SYNCER_OFFSET')
initial_block_offset = block_offset
initial_block_start = int(config.get('SYNCER_OFFSET'))
initial_block_offset = int(block_offset)
if config.true('SYNCER_NO_HISTORY'):
initial_block_start = block_offset
initial_block_start = initial_block_offset
initial_block_offset += 1
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
@@ -154,6 +155,8 @@ def main():
gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE'))
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
i = 0
@@ -163,6 +166,7 @@ def main():
syncer.add_filter(registration_filter)
# TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
syncer.add_filter(tx_filter)
syncer.add_filter(token_gas_cache_filter)
#syncer.add_filter(transfer_auth_filter)
for cf in callback_filters:
syncer.add_filter(cf)

25
apps/cic-eth/cic_eth/runnable/tag.py
View File

@@ -8,6 +8,7 @@ import re
# external imports
import cic_eth.cli
from chainlib.chain import ChainSpec
from chainlib.eth.address import is_address
from xdg.BaseDirectory import xdg_config_home
# local imports
@@ -21,12 +22,18 @@ logg = logging.getLogger()
arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_positional('tag', type=str, help='address tag')
argparser.add_positional('address', type=str, help='address')
argparser.add_argument('--set', action='store_true', help='sets the given tag')
argparser.add_argument('--tag', type=str, help='operate on the given tag')
argparser.add_positional('address', required=False, type=str, help='address associated with tag')
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
extra_args = {
'set': None,
'tag': None,
'address': None,
}
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
@@ -39,7 +46,17 @@ api = AdminApi(None)
def main():
admin_api.tag_account(args.tag, args.address, chain_spec)
if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')):
sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS')))
sys.exit(1)
if config.get('_SET'):
admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS'))
else:
t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS'))
r = t.get()
for v in r:
sys.stdout.write('{}\t{}\n'.format(v[1], v[0]))
if __name__ == '__main__':

2
apps/cic-eth/cic_eth/runnable/transfer.py
View File

@@ -18,7 +18,7 @@ from cic_eth.api import Api
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger('create_account_script')
arg_flags = cic_eth.cli.argflag_std_base
arg_flags = cic_eth.cli.argflag_local_base
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')

97
apps/cic-eth/cic_eth/runnable/view.py
View File

@@ -16,9 +16,14 @@ import confini
import celery
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
from hexathon import add_0x
from hexathon import (
add_0x,
strip_0x,
uniform as hex_uniform,
)
# local imports
import cic_eth.cli
from cic_eth.api.admin import AdminApi
from cic_eth.db.enum import (
StatusEnum,
@@ -31,59 +36,35 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
default_format = 'terminal'
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
arg_flags = cic_eth.cli.argflag_std_base
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', help='be more verbose', action='store_true')
argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"')
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
if args.v == True:
logging.getLogger().setLevel(logging.INFO)
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config.process()
args_override = {
'ETH_PROVIDER': getattr(args, 'p'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
extra_args = {
'f': '_FORMAT',
'query': '_QUERY',
}
# override args
config.dict_override(args_override, 'cli args')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
try:
config.add(add_0x(args.query), '_QUERY', True)
except:
config.add(args.query, '_QUERY', True)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
queue = config.get('CELERY_QUEUE')
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
queue = args.q
# connect to celery
celery_app = cic_eth.cli.CeleryApp.from_config(config)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
# set up rpc
rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True)
conn = rpc.get_default()
rpc = EthHTTPConnection(args.p)
#registry_address = config.get('CIC_REGISTRY_ADDRESS')
admin_api = AdminApi(rpc)
admin_api = AdminApi(conn)
t = admin_api.registry()
registry_address = t.get()
@@ -113,7 +94,7 @@ def render_tx(o, **kwargs):
for v in o.get('status_log', []):
d = datetime.datetime.fromisoformat(v[0])
e = status_str(v[1], args.status_raw)
e = status_str(v[1], config.get('_RAW'))
content += '{}: {}\n'.format(d, e)
return content
@@ -154,20 +135,24 @@ def render_lock(o, **kwargs):
def main():
txs = []
renderer = render_tx
if len(config.get('_QUERY')) > 66:
#registry = connect_registry(rpc, chain_spec, registry_address)
#admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
elif len(config.get('_QUERY')) > 42:
#registry = connect_registry(rpc, chain_spec, registry_address)
#admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
elif len(config.get('_QUERY')) == 42:
#registry = connect_registry(rpc, chain_spec, registry_address)
txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
query = config.get('_QUERY')
try:
query = hex_uniform(strip_0x(query))
except TypeError:
pass
except ValueError:
pass
if len(query) > 64:
admin_api.tx(chain_spec, tx_raw=query, renderer=renderer)
elif len(query) > 40:
admin_api.tx(chain_spec, tx_hash=query, renderer=renderer)
elif len(query) == 40:
txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account)
renderer = render_account
elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
elif len(query) >= 4 and query[:4] == 'lock':
t = admin_api.get_lock()
txs = t.get()
renderer = render_lock
@@ -175,7 +160,7 @@ def main():
r = renderer(txs)
sys.stdout.write(r + '\n')
else:
raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
raise ValueError('cannot parse argument {}'.format(query))
if __name__ == '__main__':