From d7c4cb71eb1164efe561c64415dab26170516ae1 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Wed, 22 Dec 2021 18:24:05 +0000 Subject: [PATCH] migrations: Enable deployment and data seeding to Bloxberg --- apps/cic-cache/cic_cache/cli/arg.py | 2 +- .../cic_cache/runnable/daemons/tracker.py | 6 +- apps/cic-cache/docker/db.sh | 2 +- apps/cic-eth/cic_eth/api/admin.py | 32 ++++- apps/cic-eth/cic_eth/check/gas.py | 26 ++-- apps/cic-eth/cic_eth/check/start.py | 18 +++ apps/cic-eth/cic_eth/cli/arg.py | 18 ++- apps/cic-eth/cic_eth/cli/base.py | 4 +- apps/cic-eth/cic_eth/cli/config.py | 65 ++++++++- apps/cic-eth/cic_eth/data/config/celery.ini | 2 +- apps/cic-eth/cic_eth/data/config/cic.ini | 2 +- apps/cic-eth/cic_eth/data/config/eth.ini | 6 +- .../default/versions/75d4767b3031_lock.py | 2 +- .../versions/c91cafc3e0c1_add_gas_cache.py | 31 +++++ apps/cic-eth/cic_eth/db/models/gas_cache.py | 27 ++++ apps/cic-eth/cic_eth/db/models/nonce.py | 4 +- apps/cic-eth/cic_eth/db/models/role.py | 18 ++- apps/cic-eth/cic_eth/enum.py | 6 +- apps/cic-eth/cic_eth/error.py | 4 +- apps/cic-eth/cic_eth/eth/account.py | 39 +++++- apps/cic-eth/cic_eth/eth/erc20.py | 27 +++- apps/cic-eth/cic_eth/eth/gas.py | 55 +++++++- apps/cic-eth/cic_eth/eth/util.py | 54 ++++++++ .../cic-eth/cic_eth/pytest/fixtures_config.py | 7 +- .../runnable/daemons/filters/__init__.py | 1 + .../cic_eth/runnable/daemons/filters/token.py | 63 +++++++++ .../cic_eth/runnable/daemons/tasker.py | 18 ++- .../cic_eth/runnable/daemons/tracker.py | 10 +- apps/cic-eth/cic_eth/runnable/tag.py | 25 +++- apps/cic-eth/cic_eth/runnable/transfer.py | 2 +- apps/cic-eth/cic_eth/runnable/view.py | 97 ++++++------- apps/cic-eth/cic_eth/task.py | 37 +++-- apps/cic-eth/config/test/accounts.ini | 2 - apps/cic-eth/config/test/bancor.ini | 2 - apps/cic-eth/config/test/celery.ini | 2 - apps/cic-eth/config/test/chain.ini | 2 - apps/cic-eth/config/test/cic.ini | 4 - apps/cic-eth/config/test/dispatcher.ini | 2 - apps/cic-eth/config/test/eth.ini | 8 -- apps/cic-eth/config/test/signer.ini | 5 +- apps/cic-eth/config/test/ssl.ini | 6 - apps/cic-eth/config/test/syncer.ini | 2 - apps/cic-eth/requirements.txt | 4 +- apps/cic-eth/setup.cfg | 2 +- .../tests/filters/test_token_filter.py | 97 +++++++++++++ apps/cic-eth/tests/task/api/test_admin.py | 6 +- apps/cic-eth/tests/task/test_task_account.py | 50 ++++++- .../cic_signer/data/config/database.ini | 10 ++ .../cic_signer/data/config/signer.ini | 3 + apps/cic-signer/requirements.txt | 1 + apps/cic-signer/scripts/sweep.py | 128 ++++++++++++++++++ apps/contract-migration/3_deploy_token.sh | 6 + apps/contract-migration/4_init_custodial.sh | 5 +- apps/contract-migration/config.sh | 3 + apps/contract-migration/config/config.ini | 3 + apps/contract-migration/requirements.txt | 4 +- apps/data-seeding/cic_eth/import_users.py | 7 +- apps/data-seeding/cic_ussd/import_balance.py | 5 +- apps/data-seeding/config/token.ini | 2 + apps/data-seeding/config/traffic.ini | 4 + apps/data-seeding/docker/Dockerfile | 22 ++- apps/data-seeding/import_ussd.sh | 27 ++-- apps/data-seeding/requirements.txt | 4 +- apps/data-seeding/verify.py | 2 +- docker-compose.yml | 24 ++-- 65 files changed, 940 insertions(+), 224 deletions(-) create mode 100644 apps/cic-eth/cic_eth/check/start.py create mode 100644 apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py create mode 100644 apps/cic-eth/cic_eth/db/models/gas_cache.py create mode 100644 apps/cic-eth/cic_eth/eth/util.py create mode 100644 apps/cic-eth/cic_eth/runnable/daemons/filters/token.py delete mode 100644 apps/cic-eth/config/test/accounts.ini delete mode 100644 apps/cic-eth/config/test/bancor.ini delete mode 100644 apps/cic-eth/config/test/chain.ini delete mode 100644 apps/cic-eth/config/test/cic.ini delete mode 100644 apps/cic-eth/config/test/dispatcher.ini delete mode 100644 apps/cic-eth/config/test/eth.ini delete mode 100644 apps/cic-eth/config/test/ssl.ini delete mode 100644 apps/cic-eth/config/test/syncer.ini create mode 100644 apps/cic-eth/tests/filters/test_token_filter.py create mode 100644 apps/cic-signer/cic_signer/data/config/database.ini create mode 100644 apps/cic-signer/cic_signer/data/config/signer.ini create mode 100644 apps/cic-signer/scripts/sweep.py create mode 100644 apps/data-seeding/config/token.ini create mode 100644 apps/data-seeding/config/traffic.ini diff --git a/apps/cic-cache/cic_cache/cli/arg.py b/apps/cic-cache/cic_cache/cli/arg.py index 2d4e6e8..43126eb 100644 --- a/apps/cic-cache/cic_cache/cli/arg.py +++ b/apps/cic-cache/cic_cache/cli/arg.py @@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser): if local_arg_flags & CICFlag.CELERY: self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue') if local_arg_flags & CICFlag.SYNCER: - self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--offset', type=int, help='Start block height for initial history sync') self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') if local_arg_flags & CICFlag.CHAIN: self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index 16ed1f4..d5646ea 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -95,10 +95,10 @@ def main(): syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_OFFSET') - initial_block_offset = block_offset + initial_block_start = int(config.get('SYNCER_OFFSET')) + initial_block_offset = int(block_offset) if config.get('SYNCER_NO_HISTORY'): - initial_block_start = block_offset + initial_block_start = initial_block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset)) diff --git a/apps/cic-cache/docker/db.sh b/apps/cic-cache/docker/db.sh index 002cea1..1b38945 100644 --- a/apps/cic-cache/docker/db.sh +++ b/apps/cic-cache/docker/db.sh @@ -2,5 +2,5 @@ set -e >&2 echo executing database migration -python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv +python scripts/migrate.py --migrations-dir /usr/local/share/cic-cache/alembic -vv set +e diff --git a/apps/cic-eth/cic_eth/api/admin.py b/apps/cic-eth/cic_eth/api/admin.py index 33bc207..0edaa9e 100644 --- a/apps/cic-eth/cic_eth/api/admin.py +++ b/apps/cic-eth/cic_eth/api/admin.py @@ -123,7 +123,7 @@ class AdminApi: return s_lock.apply_async() - def tag_account(self, tag, address_hex, chain_spec): + def tag_account(self, chain_spec, tag, address): """Persistently associate an address with a plaintext tag. Some tags are known by the system and is used to resolve addresses to use for certain transactions. @@ -138,7 +138,7 @@ class AdminApi: 'cic_eth.eth.account.set_role', [ tag, - address_hex, + address, chain_spec.asdict(), ], queue=self.queue, @@ -146,6 +146,30 @@ class AdminApi: return s_tag.apply_async() + def get_tag_account(self, chain_spec, tag=None, address=None): + if address != None: + s_tag = celery.signature( + 'cic_eth.eth.account.role', + [ + address, + chain_spec.asdict(), + ], + queue=self.queue, + ) + + else: + s_tag = celery.signature( + 'cic_eth.eth.account.role_account', + [ + tag, + chain_spec.asdict(), + ], + queue=self.queue, + ) + + return s_tag.apply_async() + + def have_account(self, address_hex, chain_spec): s_have = celery.signature( 'cic_eth.eth.account.have', @@ -503,7 +527,7 @@ class AdminApi: queue=self.queue, ) t = s.apply_async() - role = t.get() + role = t.get()[0][1] if role != None: tx['sender_description'] = role @@ -556,7 +580,7 @@ class AdminApi: queue=self.queue, ) t = s.apply_async() - role = t.get() + role = t.get()[0][1] if role != None: tx['recipient_description'] = role diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py index 4add981..56fe23e 100644 --- a/apps/cic-eth/cic_eth/check/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase from cic_eth.db.enum import LockEnum from cic_eth.error import LockedError from cic_eth.admin.ctrl import check_lock +from cic_eth.eth.gas import have_gas_minimum -logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger(__name__) def health(*args, **kwargs): @@ -31,18 +32,15 @@ def health(*args, **kwargs): return True gas_provider = AccountRole.get_address('GAS_GIFTER', session=session) + min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER')) + if config.get('ETH_MIN_FEE_PRICE'): + min_gas *= int(config.get('ETH_MIN_FEE_PRICE')) + + r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session) + session.close() + + if not r: + logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas)) - rpc = RPCConnection.connect(chain_spec, 'default') - o = balance(gas_provider) - r = rpc.do(o) - try: - r = int(r, 16) - except TypeError: - r = int(r) - gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE')) - if r < gas_min: - logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min)) - return False - - return True + return r diff --git a/apps/cic-eth/cic_eth/check/start.py b/apps/cic-eth/cic_eth/check/start.py new file mode 100644 index 0000000..720c80e --- /dev/null +++ b/apps/cic-eth/cic_eth/check/start.py @@ -0,0 +1,18 @@ +# external imports +from chainlib.chain import ChainSpec + +# local imports +from cic_eth.admin.ctrl import check_lock +from cic_eth.enum import LockEnum +from cic_eth.error import LockedError + + +def health(*args, **kwargs): + config = kwargs['config'] + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + try: + check_lock(None, chain_spec.asdict(), LockEnum.START) + except LockedError as e: + return False + return True diff --git a/apps/cic-eth/cic_eth/cli/arg.py b/apps/cic-eth/cic_eth/cli/arg.py index 91fcc07..acdbc63 100644 --- a/apps/cic-eth/cic_eth/cli/arg.py +++ b/apps/cic-eth/cic_eth/cli/arg.py @@ -16,16 +16,22 @@ class ArgumentParser(BaseArgumentParser): self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') if local_arg_flags & CICFlag.REDIS_CALLBACK: - self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') - self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)') + self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)') self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') if local_arg_flags & CICFlag.CELERY: + self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")') + self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)') + self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)') + self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)') + self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)') + self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)') + self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)') + self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)') + self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend') self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') if local_arg_flags & CICFlag.SYNCER: - self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--offset', type=int, help='Start block height for initial history sync') self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') if local_arg_flags & CICFlag.CHAIN: self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') - - - diff --git a/apps/cic-eth/cic_eth/cli/base.py b/apps/cic-eth/cic_eth/cli/base.py index b8b3ef2..71108b2 100644 --- a/apps/cic-eth/cic_eth/cli/base.py +++ b/apps/cic-eth/cic_eth/cli/base.py @@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum): # sync - nibble 4 SYNCER = 4096 - -argflag_local_task = CICFlag.CELERY +argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC +argflag_local_task = CICFlag.CELERY argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK argflag_local_chain = CICFlag.CHAIN argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-eth/cic_eth/cli/config.py b/apps/cic-eth/cic_eth/cli/config.py index 8bfadc6..24ffcf1 100644 --- a/apps/cic-eth/cic_eth/cli/config.py +++ b/apps/cic-eth/cic_eth/cli/config.py @@ -1,12 +1,18 @@ # standard imports import os import logging +import urllib.parse +import copy # external imports from chainlib.eth.cli import ( Config as BaseConfig, Flag, ) +from urlybird.merge import ( + urlhostmerge, + urlmerge, + ) # local imports from .base import CICFlag @@ -40,6 +46,7 @@ class Config(BaseConfig): if local_arg_flags & CICFlag.CHAIN: local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + if local_arg_flags & CICFlag.CELERY: local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') @@ -49,15 +56,61 @@ class Config(BaseConfig): config.dict_override(local_args_override, 'local cli args') - if local_arg_flags & CICFlag.REDIS_CALLBACK: - config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') - config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') - + local_celery_args_override = {} if local_arg_flags & CICFlag.CELERY: + hostport = urlhostmerge( + None, + config.get('REDIS_HOST'), + config.get('REDIS_PORT'), + ) + redis_url = ( + 'redis', + hostport, + getattr(args, 'redis_db', None), + ) + celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL')) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_host', None), + getattr(args, 'celery_port', None), + ) + celery_arg_url = ( + getattr(args, 'celery_scheme', None), + hostport, + getattr(args, 'celery_db', None), + ) + celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string + if not getattr(args, 'celery_no_result'): + local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL') + if local_celery_args_override['CELERY_RESULT_URL'] == None: + local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL'] + celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL']) + hostport = urlhostmerge( + celery_config_url[1], + getattr(args, 'celery_result_host', None), + getattr(args, 'celery_result_port', None), + ) + celery_arg_url = ( + getattr(args, 'celery_result_scheme', None), + hostport, + getattr(args, 'celery_result_db', None), + ) + celery_url = urlmerge(celery_config_url, celery_arg_url) + logg.debug('celery url {} {}'.format(celery_config_url, celery_url)) + celery_url_string = urllib.parse.urlunsplit(celery_url) + local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + config.dict_override(local_celery_args_override, 'local celery cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST')) + redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT')) + config.add(redis_host_callback, '_REDIS_HOST_CALLBACK') + config.add(redis_port_callback, '_REDIS_PORT_CALLBACK') + logg.debug('config loaded:\n{}'.format(config)) return config - - diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini index f2ad10a..5b68ee9 100644 --- a/apps/cic-eth/cic_eth/data/config/celery.ini +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -1,5 +1,5 @@ [celery] -broker_url = redis://localhost:6379 +broker_url = result_url = queue = cic-eth debug = 0 diff --git a/apps/cic-eth/cic_eth/data/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini index 103566f..7d36d45 100644 --- a/apps/cic-eth/cic_eth/data/config/cic.ini +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -2,5 +2,5 @@ registry_address = trust_address = default_token_symbol = -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start run_dir = /run diff --git a/apps/cic-eth/cic_eth/data/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini index fd58e91..2de2d1b 100644 --- a/apps/cic-eth/cic_eth/data/config/eth.ini +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -1,2 +1,6 @@ [eth] -gas_gifter_minimum_balance = 10000000000000000000000 +gas_holder_minimum_units = 180000 +gas_holder_refill_units = 15 +gas_holder_refill_threshold = 3 +gas_gifter_refill_buffer = 3 +min_fee_price = 1 diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py index dec5cf5..5c2936d 100644 --- a/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py @@ -23,7 +23,7 @@ def upgrade(): op.create_table( 'lock', sa.Column('id', sa.Integer, primary_key=True), - sa.Column("address", sa.String(42), nullable=True), + sa.Column("address", sa.String, nullable=True), sa.Column('blockchain', sa.String), sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow), diff --git a/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py new file mode 100644 index 0000000..cc1b0e0 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py @@ -0,0 +1,31 @@ +"""Add gas cache + +Revision ID: c91cafc3e0c1 +Revises: aee12aeb47ec +Create Date: 2021-10-28 20:45:34.239865 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c91cafc3e0c1' +down_revision = 'aee12aeb47ec' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'gas_cache', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column("address", sa.String, nullable=False), + sa.Column("tx_hash", sa.String, nullable=True), + sa.Column("method", sa.String, nullable=True), + sa.Column("value", sa.BIGINT(), nullable=False), + ) + + +def downgrade(): + op.drop_table('gas_cache') diff --git a/apps/cic-eth/cic_eth/db/models/gas_cache.py b/apps/cic-eth/cic_eth/db/models/gas_cache.py new file mode 100644 index 0000000..af514e1 --- /dev/null +++ b/apps/cic-eth/cic_eth/db/models/gas_cache.py @@ -0,0 +1,27 @@ +# standard imports +import logging + +# external imports +from sqlalchemy import Column, String, NUMERIC + +# local imports +from .base import SessionBase + +logg = logging.getLogger(__name__) + + +class GasCache(SessionBase): + """Provides gas budget cache for token operations + """ + __tablename__ = 'gas_cache' + + address = Column(String()) + tx_hash = Column(String()) + method = Column(String()) + value = Column(NUMERIC()) + + def __init__(self, address, method, value, tx_hash): + self.address = address + self.tx_hash = tx_hash + self.method = method + self.value = value diff --git a/apps/cic-eth/cic_eth/db/models/nonce.py b/apps/cic-eth/cic_eth/db/models/nonce.py index 280e94c..71f404f 100644 --- a/apps/cic-eth/cic_eth/db/models/nonce.py +++ b/apps/cic-eth/cic_eth/db/models/nonce.py @@ -12,7 +12,7 @@ from cic_eth.error import ( IntegrityError, ) -logg = logging.getLogger() +logg = logging.getLogger(__name__) class Nonce(SessionBase): @@ -21,7 +21,7 @@ class Nonce(SessionBase): __tablename__ = 'nonce' nonce = Column(Integer) - address_hex = Column(String(42)) + address_hex = Column(String(40)) @staticmethod diff --git a/apps/cic-eth/cic_eth/db/models/role.py b/apps/cic-eth/cic_eth/db/models/role.py index 9343f8d..a9a12a6 100644 --- a/apps/cic-eth/cic_eth/db/models/role.py +++ b/apps/cic-eth/cic_eth/db/models/role.py @@ -24,8 +24,22 @@ class AccountRole(SessionBase): tag = Column(Text) address_hex = Column(String(42)) - - # TODO: + + @staticmethod + def all(session=None): + session = SessionBase.bind_session(session) + + pairs = [] + + q = session.query(AccountRole.tag, AccountRole.address_hex) + for r in q.all(): + pairs.append((r[1], r[0]),) + + SessionBase.release_session(session) + + return pairs + + @staticmethod def get_address(tag, session): """Get Ethereum address matching the given tag diff --git a/apps/cic-eth/cic_eth/enum.py b/apps/cic-eth/cic_eth/enum.py index 2ce3ecc..3cc85ff 100644 --- a/apps/cic-eth/cic_eth/enum.py +++ b/apps/cic-eth/cic_eth/enum.py @@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum): class LockEnum(enum.IntEnum): """ STICKY: When set, reset is not possible + INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance) + START: When set, startup is not possible, regardless of state CREATE: Disable creation of accounts SEND: Disable sending to network QUEUE: Disable queueing new or modified transactions + QUERY: Disable all queue state and transaction queries """ STICKY=1 INIT=2 @@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum): SEND=8 QUEUE=16 QUERY=32 - ALL=int(0xfffffffffffffffe) + START=int(0x80000000) + ALL=int(0x7ffffffe) def status_str(v, bits_only=False): diff --git a/apps/cic-eth/cic_eth/error.py b/apps/cic-eth/cic_eth/error.py index 7e8dbfa..9c0689a 100644 --- a/apps/cic-eth/cic_eth/error.py +++ b/apps/cic-eth/cic_eth/error.py @@ -64,8 +64,10 @@ class LockedError(Exception): class SeppukuError(Exception): """Exception base class for all errors that should cause system shutdown - """ + def __init__(self, message, lockdown=False): + self.message = message + self.lockdown = lockdown class SignerError(SeppukuError): diff --git a/apps/cic-eth/cic_eth/eth/account.py b/apps/cic-eth/cic_eth/eth/account.py index 9f4916c..5b83421 100644 --- a/apps/cic-eth/cic_eth/eth/account.py +++ b/apps/cic-eth/cic_eth/eth/account.py @@ -136,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None): # Generate and sign transaction rpc_signer = RPCConnection.connect(chain_spec, 'signer') nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce) - gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas) + gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas) account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED) rpc_signer.disconnect() @@ -192,7 +192,7 @@ def gift(self, account_address, chain_spec_dict): # Generate and sign transaction rpc_signer = RPCConnection.connect(chain_spec, 'signer') nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce) - gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas) + gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas) faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED) rpc_signer.disconnect() @@ -266,19 +266,46 @@ def set_role(self, tag, address, chain_spec_dict): @celery_app.task(bind=True, base=BaseTask) def role(self, address, chain_spec_dict): - """Return account role for address + """Return account role for address and/or role :param account: Account to check :type account: str, 0x-hex - :param chain_str: Chain spec string representation - :type chain_str: str + :param chain_spec_dict: Chain spec dict representation + :type chain_spec_dict: dict :returns: Account, or None if not exists :rtype: Varies """ session = self.create_session() role_tag = AccountRole.role_for(address, session=session) session.close() - return role_tag + return [(address, role_tag,)] + + +@celery_app.task(bind=True, base=BaseTask) +def role_account(self, role_tag, chain_spec_dict): + """Return address for role. + + If the role parameter is None, will return addresses for all roles. + + :param role_tag: Role to match + :type role_tag: str + :param chain_spec_dict: Chain spec dict representation + :type chain_spec_dict: dict + :returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags + :rtype: list + """ + session = self.create_session() + + pairs = None + if role_tag != None: + addr = AccountRole.get_address(role_tag, session=session) + pairs = [(addr, role_tag,)] + else: + pairs = AccountRole.all(session=session) + + session.close() + + return pairs @celery_app.task(bind=True, base=CriticalSQLAlchemyTask) diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 4fbe2e5..bf34acf 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -10,6 +10,9 @@ from chainlib.eth.tx import ( TxFormat, unpack, ) +from chainlib.eth.contract import ( + ABIContractEncoder, + ) from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token from hexathon import ( @@ -31,10 +34,8 @@ from cic_eth.error import ( YouAreBrokeError, ) from cic_eth.queue.tx import register_tx -from cic_eth.eth.gas import ( - create_check_gas_task, - MaxGasOracle, - ) +from cic_eth.eth.gas import create_check_gas_task +from cic_eth.eth.util import CacheGasOracle from cic_eth.ext.address import translate_address from cic_eth.task import ( CriticalSQLAlchemyTask, @@ -154,8 +155,12 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s rpc_signer = RPCConnection.connect(chain_spec, 'signer') session = self.create_session() + nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) - gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) + enc = ABIContractEncoder() + enc.method('transferFrom') + method = enc.get() + gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) try: (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED) @@ -225,8 +230,12 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d rpc_signer = RPCConnection.connect(chain_spec, 'signer') session = self.create_session() + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price) nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) - gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) try: (tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED) @@ -294,8 +303,12 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic rpc_signer = RPCConnection.connect(chain_spec, 'signer') session = self.create_session() + nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session) - gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas) + enc = ABIContractEncoder() + enc.method('approve') + method = enc.get() + gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session) c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle) try: (tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED) diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 435949f..95e7ab5 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache from chainqueue.db.models.otx import Otx # local imports +from cic_eth.db.models.gas_cache import GasCache from cic_eth.db.models.role import AccountRole from cic_eth.db.models.base import SessionBase from cic_eth.error import ( @@ -65,17 +66,56 @@ from cic_eth.encode import ( ZERO_ADDRESS_NORMAL, unpack_normal, ) +from cic_eth.error import SeppukuError +from cic_eth.eth.util import MAXIMUM_FEE_UNITS celery_app = celery.current_app logg = logging.getLogger() -MAXIMUM_FEE_UNITS = 8000000 -class MaxGasOracle: +@celery_app.task(base=CriticalSQLAlchemyTask) +def apply_gas_value_cache(address, method, value, tx_hash): + return apply_gas_value_cache_local(address, method, value, tx_hash) - def gas(code=None): - return MAXIMUM_FEE_UNITS + +def apply_gas_value_cache_local(address, method, value, tx_hash, session=None): + address = tx_normalize.executable_address(address) + tx_hash = tx_normalize.tx_hash(tx_hash) + value = int(value) + + session = SessionBase.bind_session(session) + q = session.query(GasCache) + q = q.filter(GasCache.address==address) + q = q.filter(GasCache.method==method) + o = q.first() + + if o == None: + o = GasCache(address, method, value, tx_hash) + elif tx.gas_used > o.value: + o.value = value + o.tx_hash = strip_0x(tx_hash) + + session.add(o) + session.commit() + + SessionBase.release_session(session) + + +def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None): + if rpc == None: + rpc = RPCConnection.connect(chain_spec, 'default') + o = balance(add_0x(address)) + r = rpc.do(o) + try: + r = int(r) + except ValueError: + r = strip_0x(r) + r = int(r, 16) + logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas)) + if r < min_gas: + return False + return True def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): @@ -357,6 +397,13 @@ def refill_gas(self, recipient_address, chain_spec_dict): # set up evm RPC connection rpc = RPCConnection.connect(chain_spec, 'default') + # check the gas balance of the gifter + if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount): + raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider)) + + if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance): + logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider)) + # set up transaction builder nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session) gas_oracle = self.create_gas_oracle(rpc) diff --git a/apps/cic-eth/cic_eth/eth/util.py b/apps/cic-eth/cic_eth/eth/util.py new file mode 100644 index 0000000..6720c66 --- /dev/null +++ b/apps/cic-eth/cic_eth/eth/util.py @@ -0,0 +1,54 @@ +# standard imports +import logging + +# external imports +from chainlib.eth.gas import RPCGasOracle +from hexathon import strip_0x + +# local imports +from cic_eth.db.models.gas_cache import GasCache +from cic_eth.encode import tx_normalize +from cic_eth.db.models.base import SessionBase + +MAXIMUM_FEE_UNITS = 8000000 + +logg = logging.getLogger(__name__) + + +class MaxGasOracle(RPCGasOracle): + + def get_fee_units(self, code=None): + return MAXIMUM_FEE_UNITS + + +class CacheGasOracle(MaxGasOracle): + """Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units. + + :todo: instead of max units, connect a pluggable gas heuristics engine. + """ + + def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None): + super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator) + self.value = None + self.address = address + self.method = method + + address = tx_normalize.executable_address(address) + session = SessionBase.bind_session(session) + q = session.query(GasCache) + q = q.filter(GasCache.address==address) + if method != None: + method = strip_0x(method) + q = q.filter(GasCache.method==method) + o = q.first() + if o != None: + self.value = int(o.value) + + SessionBase.release_session(session) + + + def get_fee_units(self, code=None): + if self.value != None: + logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method)) + return self.value + return super(CacheGasOracle, self).get_fee_units(code=code) diff --git a/apps/cic-eth/cic_eth/pytest/fixtures_config.py b/apps/cic-eth/cic_eth/pytest/fixtures_config.py index 27d7c9c..e44078f 100644 --- a/apps/cic-eth/cic_eth/pytest/fixtures_config.py +++ b/apps/cic-eth/cic_eth/pytest/fixtures_config.py @@ -8,15 +8,14 @@ import confini script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.dirname(os.path.dirname(script_dir)) +config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config') logg = logging.getLogger(__name__) @pytest.fixture(scope='session') def load_config(): - config_dir = os.environ.get('CONFINI_DIR') - if config_dir == None: - config_dir = os.path.join(root_dir, 'config/test') - conf = confini.Config(config_dir, 'CICTEST') + override_config_dir = os.path.join(root_dir, 'config', 'test') + conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir]) conf.process() logg.debug('config {}'.format(conf)) return conf diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py index d2a9823..8b63297 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py @@ -3,3 +3,4 @@ from .tx import TxFilter from .gas import GasFilter from .register import RegistrationFilter from .transferauth import TransferAuthFilter +from .token import TokenFilter diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py new file mode 100644 index 0000000..e1cb54f --- /dev/null +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/token.py @@ -0,0 +1,63 @@ +# standard imports +import logging + +# external imports +from eth_erc20 import ERC20 +from chainlib.eth.contract import ( + ABIContractEncoder, + ABIContractType, + ) +from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.address import is_same_address +from chainlib.eth.error import RequestMismatchException +from cic_eth_registry import CICRegistry +from cic_eth_registry.erc20 import ERC20Token +from eth_token_index import TokenUniqueSymbolIndex +import celery + +# local imports +from .base import SyncFilter + +logg = logging.getLogger(__name__) + + +class TokenFilter(SyncFilter): + + def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS): + self.queue = queue + self.chain_spec = chain_spec + self.caller_address = call_address + + + def filter(self, conn, block, tx, db_session=None): + if not tx.payload: + return (None, None) + + try: + r = ERC20.parse_transfer_request(tx.payload) + except RequestMismatchException: + return (None, None) + + token_address = tx.inputs[0] + token = ERC20Token(self.chain_spec, conn, token_address) + + registry = CICRegistry(self.chain_spec, conn) + r = registry.by_name(token.symbol, sender_address=self.caller_address) + if is_same_address(r, ZERO_ADDRESS): + return None + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + s = celery.signature( + 'cic_eth.eth.gas.apply_gas_value_cache', + [ + token_address, + method, + tx.gas_used, + tx.hash, + ], + queue=self.queue, + ) + return s.apply_async() diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 7e3a22a..d2751e4 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -67,7 +67,10 @@ from cic_eth.registry import ( connect_declarator, connect_token_registry, ) -from cic_eth.task import BaseTask +from cic_eth.task import ( + BaseTask, + CriticalWeb3Task, + ) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -76,18 +79,18 @@ arg_flags = cic_eth.cli.argflag_std_read local_arg_flags = cic_eth.cli.argflag_local_task argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.process_local_flags(local_arg_flags) -#argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use') argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage') argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path') +argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei') argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path') args = argparser.parse_args() # process config extra_args = { -# 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL', 'aux_all': None, 'aux': None, 'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS', + 'min_fee_price': 'ETH_MIN_FEE_PRICE', } config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) @@ -215,6 +218,7 @@ def main(): argv.append('-n') argv.append(config.get('CELERY_QUEUE')) + # TODO: More elegant way of setting queue-wide settings BaseTask.default_token_symbol = default_token_symbol BaseTask.default_token_address = default_token_address default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address)) @@ -222,6 +226,14 @@ def main(): BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name BaseTask.trusted_addresses = trusted_addresses + CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS')) + CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD')) + CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER')) + if config.get('ETH_MIN_FEE_PRICE'): + BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE')) + CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price + CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price + CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price BaseTask.run_dir = config.get('CIC_RUN_DIR') logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 5117537..680eb79 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -36,6 +36,7 @@ from cic_eth.runnable.daemons.filters import ( TxFilter, RegistrationFilter, TransferAuthFilter, + TokenFilter, ) from cic_eth.stat import init_chain_stat from cic_eth.registry import ( @@ -99,10 +100,10 @@ def main(): syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_OFFSET') - initial_block_offset = block_offset + initial_block_start = int(config.get('SYNCER_OFFSET')) + initial_block_offset = int(block_offset) if config.true('SYNCER_NO_HISTORY'): - initial_block_start = block_offset + initial_block_start = initial_block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset)) @@ -154,6 +155,8 @@ def main(): gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE')) + token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE')) + #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) i = 0 @@ -163,6 +166,7 @@ def main(): syncer.add_filter(registration_filter) # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break syncer.add_filter(tx_filter) + syncer.add_filter(token_gas_cache_filter) #syncer.add_filter(transfer_auth_filter) for cf in callback_filters: syncer.add_filter(cf) diff --git a/apps/cic-eth/cic_eth/runnable/tag.py b/apps/cic-eth/cic_eth/runnable/tag.py index b4bdd83..7231c7a 100644 --- a/apps/cic-eth/cic_eth/runnable/tag.py +++ b/apps/cic-eth/cic_eth/runnable/tag.py @@ -8,6 +8,7 @@ import re # external imports import cic_eth.cli from chainlib.chain import ChainSpec +from chainlib.eth.address import is_address from xdg.BaseDirectory import xdg_config_home # local imports @@ -21,12 +22,18 @@ logg = logging.getLogger() arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC local_arg_flags = cic_eth.cli.argflag_local_taskcallback argparser = cic_eth.cli.ArgumentParser(arg_flags) -argparser.add_positional('tag', type=str, help='address tag') -argparser.add_positional('address', type=str, help='address') +argparser.add_argument('--set', action='store_true', help='sets the given tag') +argparser.add_argument('--tag', type=str, help='operate on the given tag') +argparser.add_positional('address', required=False, type=str, help='address associated with tag') argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) +extra_args = { + 'set': None, + 'tag': None, + 'address': None, + } +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) celery_app = cic_eth.cli.CeleryApp.from_config(config) @@ -39,7 +46,17 @@ api = AdminApi(None) def main(): - admin_api.tag_account(args.tag, args.address, chain_spec) + if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')): + sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS'))) + sys.exit(1) + + if config.get('_SET'): + admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS')) + else: + t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS')) + r = t.get() + for v in r: + sys.stdout.write('{}\t{}\n'.format(v[1], v[0])) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/transfer.py b/apps/cic-eth/cic_eth/runnable/transfer.py index 4bbaa30..67c8314 100644 --- a/apps/cic-eth/cic_eth/runnable/transfer.py +++ b/apps/cic-eth/cic_eth/runnable/transfer.py @@ -18,7 +18,7 @@ from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger('create_account_script') -arg_flags = cic_eth.cli.argflag_std_base +arg_flags = cic_eth.cli.argflag_local_base local_arg_flags = cic_eth.cli.argflag_local_taskcallback argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol') diff --git a/apps/cic-eth/cic_eth/runnable/view.py b/apps/cic-eth/cic_eth/runnable/view.py index 5bd9c0a..bdc40c3 100644 --- a/apps/cic-eth/cic_eth/runnable/view.py +++ b/apps/cic-eth/cic_eth/runnable/view.py @@ -16,9 +16,14 @@ import confini import celery from chainlib.chain import ChainSpec from chainlib.eth.connection import EthHTTPConnection -from hexathon import add_0x +from hexathon import ( + add_0x, + strip_0x, + uniform as hex_uniform, + ) # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi from cic_eth.db.enum import ( StatusEnum, @@ -31,59 +36,35 @@ logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format') -argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), +extra_args = { + 'f': '_FORMAT', + 'query': '_QUERY', } -# override args -config.dict_override(args_override, 'cli args') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) -try: - config.add(add_0x(args.query), '_QUERY', True) -except: - config.add(args.query, '_QUERY', True) +celery_app = cic_eth.cli.CeleryApp.from_config(config) +queue = config.get('CELERY_QUEUE') -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -queue = args.q +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True) +conn = rpc.get_default() -rpc = EthHTTPConnection(args.p) - -#registry_address = config.get('CIC_REGISTRY_ADDRESS') - -admin_api = AdminApi(rpc) +admin_api = AdminApi(conn) t = admin_api.registry() registry_address = t.get() @@ -113,7 +94,7 @@ def render_tx(o, **kwargs): for v in o.get('status_log', []): d = datetime.datetime.fromisoformat(v[0]) - e = status_str(v[1], args.status_raw) + e = status_str(v[1], config.get('_RAW')) content += '{}: {}\n'.format(d, e) return content @@ -154,20 +135,24 @@ def render_lock(o, **kwargs): def main(): txs = [] renderer = render_tx - if len(config.get('_QUERY')) > 66: - #registry = connect_registry(rpc, chain_spec, registry_address) - #admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer) - admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer) - elif len(config.get('_QUERY')) > 42: - #registry = connect_registry(rpc, chain_spec, registry_address) - #admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer) - admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer) - elif len(config.get('_QUERY')) == 42: - #registry = connect_registry(rpc, chain_spec, registry_address) - txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account) + query = config.get('_QUERY') + try: + query = hex_uniform(strip_0x(query)) + except TypeError: + pass + except ValueError: + pass + + if len(query) > 64: + admin_api.tx(chain_spec, tx_raw=query, renderer=renderer) + elif len(query) > 40: + admin_api.tx(chain_spec, tx_hash=query, renderer=renderer) + + elif len(query) == 40: + txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account) renderer = render_account - elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock': + elif len(query) >= 4 and query[:4] == 'lock': t = admin_api.get_lock() txs = t.get() renderer = render_lock @@ -175,7 +160,7 @@ def main(): r = renderer(txs) sys.stdout.write(r + '\n') else: - raise ValueError('cannot parse argument {}'.format(config.get('_QUERY'))) + raise ValueError('cannot parse argument {}'.format(query)) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/task.py b/apps/cic-eth/cic_eth/task.py index 149e180..8a484ac 100644 --- a/apps/cic-eth/cic_eth/task.py +++ b/apps/cic-eth/cic_eth/task.py @@ -17,6 +17,7 @@ from cic_eth_registry.error import UnknownContractError # local imports from cic_eth.error import SeppukuError from cic_eth.db.models.base import SessionBase +from cic_eth.eth.util import CacheGasOracle #logg = logging.getLogger().getChild(__name__) logg = logging.getLogger() @@ -29,14 +30,32 @@ class BaseTask(celery.Task): session_func = SessionBase.create_session call_address = ZERO_ADDRESS trusted_addresses = [] - create_nonce_oracle = RPCNonceOracle - create_gas_oracle = RPCGasOracle + min_fee_price = 1 default_token_address = None default_token_symbol = None default_token_name = None default_token_decimals = None run_dir = '/run' + + def create_gas_oracle(self, conn, address=None, *args, **kwargs): + if address == None: + return RPCGasOracle( + conn, + code_callback=kwargs.get('code_callback'), + min_price=self.min_fee_price, + id_generator=kwargs.get('id_generator'), + ) + + return CacheGasOracle( + conn, + address, + method=kwargs.get('method'), + min_price=self.min_fee_price, + id_generator=kwargs.get('id_generator'), + ) + + def create_session(self): return BaseTask.session_func() @@ -78,19 +97,18 @@ class CriticalWeb3Task(CriticalTask): autoretry_for = ( ConnectionError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 + safe_gas_threshold_amount = 60000 * 3 safe_gas_refill_amount = safe_gas_threshold_amount * 5 + safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100 -class CriticalSQLAlchemyAndWeb3Task(CriticalTask): +class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task): autoretry_for = ( sqlalchemy.exc.DatabaseError, sqlalchemy.exc.TimeoutError, ConnectionError, sqlalchemy.exc.ResourceClosedError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 - safe_gas_refill_amount = safe_gas_threshold_amount * 5 class CriticalSQLAlchemyAndSignerTask(CriticalTask): @@ -100,14 +118,11 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask): sqlalchemy.exc.ResourceClosedError, ) -class CriticalWeb3AndSignerTask(CriticalTask): +class CriticalWeb3AndSignerTask(CriticalWeb3Task): autoretry_for = ( ConnectionError, ) - safe_gas_threshold_amount = 2000000000 * 60000 * 3 - safe_gas_refill_amount = safe_gas_threshold_amount * 5 - - + @celery_app.task() def check_health(self): pass diff --git a/apps/cic-eth/config/test/accounts.ini b/apps/cic-eth/config/test/accounts.ini deleted file mode 100644 index 819a83b..0000000 --- a/apps/cic-eth/config/test/accounts.ini +++ /dev/null @@ -1,2 +0,0 @@ -[accounts] -writer_address = diff --git a/apps/cic-eth/config/test/bancor.ini b/apps/cic-eth/config/test/bancor.ini deleted file mode 100644 index 961c0e6..0000000 --- a/apps/cic-eth/config/test/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = tests/testdata/bancor diff --git a/apps/cic-eth/config/test/celery.ini b/apps/cic-eth/config/test/celery.ini index 1f12b77..30a519c 100644 --- a/apps/cic-eth/config/test/celery.ini +++ b/apps/cic-eth/config/test/celery.ini @@ -1,5 +1,3 @@ [celery] broker_url = filesystem:// result_url = filesystem:// -#broker_url = redis:// -#result_url = redis:// diff --git a/apps/cic-eth/config/test/chain.ini b/apps/cic-eth/config/test/chain.ini deleted file mode 100644 index 9fda098..0000000 --- a/apps/cic-eth/config/test/chain.ini +++ /dev/null @@ -1,2 +0,0 @@ -[chain] -spec = diff --git a/apps/cic-eth/config/test/cic.ini b/apps/cic-eth/config/test/cic.ini deleted file mode 100644 index d985ae3..0000000 --- a/apps/cic-eth/config/test/cic.ini +++ /dev/null @@ -1,4 +0,0 @@ -[cic] -registry_address = -chain_spec = -trust_address = diff --git a/apps/cic-eth/config/test/dispatcher.ini b/apps/cic-eth/config/test/dispatcher.ini deleted file mode 100644 index f7b270a..0000000 --- a/apps/cic-eth/config/test/dispatcher.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.1 diff --git a/apps/cic-eth/config/test/eth.ini b/apps/cic-eth/config/test/eth.ini deleted file mode 100644 index 4c9edda..0000000 --- a/apps/cic-eth/config/test/eth.ini +++ /dev/null @@ -1,8 +0,0 @@ -[eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 -provider = http://localhost:8545 -gas_provider_address = -#chain_id = -abi_dir = -faucet_giver_address = diff --git a/apps/cic-eth/config/test/signer.ini b/apps/cic-eth/config/test/signer.ini index 0fa1f01..2e1cbc0 100644 --- a/apps/cic-eth/config/test/signer.ini +++ b/apps/cic-eth/config/test/signer.ini @@ -1,5 +1,2 @@ [signer] -socket_path = /run/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = +provider = /run/crypto-dev-signer/jsonrpc.ipc diff --git a/apps/cic-eth/config/test/ssl.ini b/apps/cic-eth/config/test/ssl.ini deleted file mode 100644 index bd49472..0000000 --- a/apps/cic-eth/config/test/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/test/syncer.ini b/apps/cic-eth/config/test/syncer.ini deleted file mode 100644 index 9723674..0000000 --- a/apps/cic-eth/config/test/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[SYNCER] -loop_interval = 1 diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 7a996c4..b1097b2 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ celery==4.4.7 -chainlib-eth>=0.0.10a16,<0.1.0 +chainlib-eth>=0.0.10a20,<0.1.0 semver==2.13.0 -crypto-dev-signer>=0.4.15rc2,<0.5.0 +urlybird~=0.0.1a2 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index e67b303..de73a58 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -1,7 +1,7 @@ [metadata] name = cic-eth #version = attr: cic_eth.version.__version_string__ -version = 0.12.4a13 +version = 0.12.5a2 description = CIC Network Ethereum interaction author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-eth/tests/filters/test_token_filter.py b/apps/cic-eth/tests/filters/test_token_filter.py new file mode 100644 index 0000000..df2bfdb --- /dev/null +++ b/apps/cic-eth/tests/filters/test_token_filter.py @@ -0,0 +1,97 @@ +# external imports +from eth_erc20 import ERC20 +from chainlib.connection import RPCConnection +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import ( + Gas, + OverrideGasOracle, + ) +from chainlib.eth.tx import ( + TxFormat, + receipt, + raw, + unpack, + Tx, + ) +from chainlib.eth.block import ( + Block, + block_latest, + block_by_number, + ) +from chainlib.eth.address import is_same_address +from chainlib.eth.contract import ABIContractEncoder +from hexathon import strip_0x +from eth_token_index import TokenUniqueSymbolIndex + +# local imports +from cic_eth.runnable.daemons.filters.token import TokenFilter +from cic_eth.db.models.gas_cache import GasCache +from cic_eth.db.models.base import SessionBase + + +def test_filter_gas( + default_chain_spec, + init_database, + eth_rpc, + eth_signer, + contract_roles, + agent_roles, + token_roles, + foo_token, + token_registry, + register_lookups, + celery_session_worker, + cic_registry, + ): + + rpc = RPCConnection.connect(default_chain_spec, 'default') + nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc) + gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000) + c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle) + (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED) + o = raw(tx_signed_raw_hex) + eth_rpc.do(o) + o = receipt(tx_hash_hex) + rcpt = eth_rpc.do(o) + assert rcpt['status'] == 1 + + fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE']) + + o = block_latest() + r = eth_rpc.do(o) + o = block_by_number(r, include_tx=False) + r = eth_rpc.do(o) + block = Block(r) + block.txs = [tx_hash_hex] + + tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) + tx_src = unpack(tx_signed_raw_bytes, default_chain_spec) + tx = Tx(tx_src, block=block) + tx.apply_receipt(rcpt) + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + assert t == None + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc) + c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle) + (tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token) + eth_rpc.do(o) + o = receipt(tx_hash_hex) + r = eth_rpc.do(o) + assert r['status'] == 1 + + t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + r = t.get_leaf() + assert t.successful() + + q = init_database.query(GasCache) + q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex)) + o = q.first() + + assert is_same_address(o.address, strip_0x(foo_token)) + assert o.value > 0 + + enc = ABIContractEncoder() + enc.method('transfer') + method = enc.get() + + assert o.method == method diff --git a/apps/cic-eth/tests/task/api/test_admin.py b/apps/cic-eth/tests/task/api/test_admin.py index 363fb8d..d732fb9 100644 --- a/apps/cic-eth/tests/task/api/test_admin.py +++ b/apps/cic-eth/tests/task/api/test_admin.py @@ -103,11 +103,11 @@ def test_tag_account( api = AdminApi(eth_rpc, queue=None) - t = api.tag_account('foo', agent_roles['ALICE'], default_chain_spec) + t = api.tag_account(default_chain_spec, 'foo', agent_roles['ALICE']) t.get() - t = api.tag_account('bar', agent_roles['BOB'], default_chain_spec) + t = api.tag_account(default_chain_spec, 'bar', agent_roles['BOB']) t.get() - t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec) + t = api.tag_account(default_chain_spec, 'bar', agent_roles['CAROL']) t.get() assert AccountRole.get_address('foo', init_database) == tx_normalize.wallet_address(agent_roles['ALICE']) diff --git a/apps/cic-eth/tests/task/test_task_account.py b/apps/cic-eth/tests/task/test_task_account.py index d1c1c3d..58f1b49 100644 --- a/apps/cic-eth/tests/task/test_task_account.py +++ b/apps/cic-eth/tests/task/test_task_account.py @@ -141,9 +141,57 @@ def test_role_task( ) t = s.apply_async() r = t.get() - assert r == 'foo' + assert r[0][0] == address + assert r[0][1] == 'foo' +def test_get_role_task( + init_database, + celery_session_worker, + default_chain_spec, + ): + address_foo = '0x' + os.urandom(20).hex() + role_foo = AccountRole.set('foo', address_foo) + init_database.add(role_foo) + + address_bar = '0x' + os.urandom(20).hex() + role_bar = AccountRole.set('bar', address_bar) + init_database.add(role_bar) + + init_database.commit() + + s = celery.signature( + 'cic_eth.eth.account.role_account', + [ + 'bar', + default_chain_spec.asdict(), + ], + queue=None, + ) + t = s.apply_async() + r = t.get() + assert r[0][0] == address_bar + assert r[0][1] == 'bar' + + s = celery.signature( + 'cic_eth.eth.account.role_account', + [ + None, + default_chain_spec.asdict(), + ], + queue=None, + ) + t = s.apply_async() + r = t.get() + x_tags = ['foo', 'bar'] + x_addrs = [address_foo, address_bar] + + for v in r: + x_addrs.remove(v[0]) + x_tags.remove(v[1]) + + assert len(x_tags) == 0 + assert len(x_addrs) == 0 def test_gift( init_database, diff --git a/apps/cic-signer/cic_signer/data/config/database.ini b/apps/cic-signer/cic_signer/data/config/database.ini new file mode 100644 index 0000000..2416dda --- /dev/null +++ b/apps/cic-signer/cic_signer/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = postgres +driver = psycopg2 +host = localhost +port = 5432 +name = cic_signer +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-signer/cic_signer/data/config/signer.ini b/apps/cic-signer/cic_signer/data/config/signer.ini new file mode 100644 index 0000000..8b0c82f --- /dev/null +++ b/apps/cic-signer/cic_signer/data/config/signer.ini @@ -0,0 +1,3 @@ +[signer] +provider = +secret = diff --git a/apps/cic-signer/requirements.txt b/apps/cic-signer/requirements.txt index 4a2e9bf..f940881 100644 --- a/apps/cic-signer/requirements.txt +++ b/apps/cic-signer/requirements.txt @@ -1 +1,2 @@ funga-eth[sql]>=0.5.1a1,<0.6.0 +chainlib-eth>=0.0.10a18 diff --git a/apps/cic-signer/scripts/sweep.py b/apps/cic-signer/scripts/sweep.py new file mode 100644 index 0000000..5b4f151 --- /dev/null +++ b/apps/cic-signer/scripts/sweep.py @@ -0,0 +1,128 @@ +# standard imports +import os +import logging +import uuid +import random +import sys + +# external imports +from chainlib.chain import ChainSpec +from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.gas import ( + balance, + Gas, + ) +from hexathon import ( + add_0x, + strip_0x, + ) +from chainlib.eth.connection import EthHTTPSignerConnection +from funga.eth.signer import EIP155Signer +from funga.eth.keystore.sql import SQLKeystore +from chainlib.cli.wallet import Wallet +from chainlib.eth.address import AddressChecksum +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.gas import OverrideGasOracle +from chainlib.eth.address import ( + is_checksum_address, + to_checksum_address, + ) +from chainlib.eth.tx import ( + TxFormat, + ) +import chainlib.eth.cli + + +script_dir = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(script_dir, '..', 'cic_signer', 'data', 'config') + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + +arg_flags = chainlib.eth.cli.argflag_std_write | chainlib.eth.cli.Flag.WALLET +argparser = chainlib.eth.cli.ArgumentParser(arg_flags) +args = argparser.parse_args() + +config = chainlib.eth.cli.Config.from_args(args, arg_flags, base_config_dir=config_dir) + +# set up rpc +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + +# connect to database +dsn = 'postgresql://{}:{}@{}:{}/{}'.format( + config.get('DATABASE_USER'), + config.get('DATABASE_PASSWORD'), + config.get('DATABASE_HOST'), + config.get('DATABASE_PORT'), + config.get('DATABASE_NAME'), + ) +logg.info('using dsn {}'.format(dsn)) + +keystore = SQLKeystore(dsn, symmetric_key=bytes.fromhex(config.get('SIGNER_SECRET'))) +wallet = Wallet(EIP155Signer, keystore=keystore, checksummer=AddressChecksum) + +rpc = chainlib.eth.cli.Rpc(wallet=wallet) +conn = rpc.connect_by_config(config) + +wallet.init() + +def main(): + if config.get('_RECIPIENT') == None: + sys.stderr.write('Missing sink address\n') + sys.exit(1) + + sink_address = config.get('_RECIPIENT') + if config.get('_UNSAFE'): + sink_address = to_checksum_address(sink_address) + if not is_checksum_address(sink_address): + sys.stderr.write('Invalid sink address {}\n'.format(sink_address)) + sys.exit(1) + + if (config.get('_RPC_SEND')): + verify_string = random.randbytes(4).hex() + verify_string_check = input("\033[;31m*** WARNING! WARNING! WARNING! ***\033[;39m\nThis action will transfer all remaining gas from all accounts in custodial care to account {}. To confirm, please enter the string: {}\n".format(config.get('_RECIPIENT'), verify_string)) + if verify_string != verify_string_check: + sys.stderr.write('Verify string mismatch. Aborting!\n') + sys.exit(1) + + signer = rpc.get_signer() + + gas_oracle = rpc.get_gas_oracle() + gas_pair = gas_oracle.get_fee() + gas_price = gas_pair[0] + gas_limit = 21000 + gas_cost = gas_price * gas_limit + gas_oracle = OverrideGasOracle(price=gas_price, limit=gas_limit) + logg.info('using gas price {}'.format(gas_price)) + + for r in keystore.list(): + account = r[0] + + o = balance(add_0x(account)) + r = conn.do(o) + account_balance = 0 + try: + r = strip_0x(r) + account_balance = int(r, 16) + except ValueError: + account_balance = int(r) + + transfer_amount = account_balance - gas_cost + if transfer_amount <= 0: + logg.warning('address {} has balance {} which is less than gas cost {}, skipping'.format(account, account_balance, gas_cost)) + continue + + nonce_oracle = RPCNonceOracle(account, conn) + c = Gas(chain_spec, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, signer=signer) + tx_hash_hex = None + if (config.get('_RPC_SEND')): + (tx_hash_hex, o) = c.create(account, config.get('_RECIPIENT'), transfer_amount) + r = conn.do(o) + else: + (tx_hash_hex, o) = c.create(account, config.get('_RECIPIENT'), transfer_amount, tx_format=TxFormat.RLP_SIGNED) + + logg.info('address {} balance {} net transfer {} tx {}'.format(account, account_balance, transfer_amount, tx_hash_hex)) + +if __name__ == '__main__': + main() diff --git a/apps/contract-migration/3_deploy_token.sh b/apps/contract-migration/3_deploy_token.sh index 9f1e227..6ef3004 100644 --- a/apps/contract-migration/3_deploy_token.sh +++ b/apps/contract-migration/3_deploy_token.sh @@ -152,6 +152,12 @@ else deploy_minter_${TOKEN_MINTER_MODE} $TOKEN_ADDRESS fi +>&2 echo -e "\033[;96mTransfer a single token to self to poke the gas cacher\033[;39m" +advance_nonce +debug_rpc +r=`erc20-transfer $DEV_WAIT_FLAG --nonce $nonce $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC -u $DEV_DEBUG_FLAG -s -e $TOKEN_ADDRESS -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER 1` +add_pending_tx_hash $r + check_wait 3 >&2 echo -e "\033[;96mWriting token metadata and proofs\033[;39m" diff --git a/apps/contract-migration/4_init_custodial.sh b/apps/contract-migration/4_init_custodial.sh index 5dc17d7..6997e91 100644 --- a/apps/contract-migration/4_init_custodial.sh +++ b/apps/contract-migration/4_init_custodial.sh @@ -18,6 +18,7 @@ fi must_address "$CIC_REGISTRY_ADDRESS" "registry" must_eth_rpc + # get required addresses from registries token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry` accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry` @@ -28,11 +29,11 @@ REDIS_HOST_CALLBACK=${REDIS_HOST_CALLBACK:-$REDIS_HOST} REDIS_PORT_CALLBACK=${REDIS_PORT_CALLBACK:-$REDIS_PORT} >&2 echo -e "\033[;96mcreate account for gas gifter\033[;39m" gas_gifter=`cic-eth-create --redis-timeout 120 $DEV_DEBUG_FLAG --redis-host-callback $REDIS_HOST_CALLBACK --redis-port-callback $REDIS_PORT_CALLBACK --no-register` -cic-eth-tag -i $CHAIN_SPEC GAS_GIFTER $gas_gifter +cic-eth-tag -i $CHAIN_SPEC --set --tag GAS_GIFTER $gas_gifter >&2 echo -e "\033[;96mcreate account for accounts index writer\033[;39m" accounts_index_writer=`cic-eth-create --redis-timeout 120 $DEV_DEBUG_FLAG --redis-host-callback $REDIS_HOST_CALLBACK --redis-port-callback $REDIS_PORT_CALLBACK --no-register` -cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer +cic-eth-tag -i $CHAIN_SPEC --set --tag ACCOUNT_REGISTRY_WRITER $accounts_index_writer # Assign system writer for accounts index diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index ea69a52..48097f3 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -1,6 +1,7 @@ #!/bin/bash set -a +set -e if [ -z $DEV_DATA_DIR ]; then export DEV_DATA_DIR=`mktemp -d` @@ -33,6 +34,7 @@ else fi rm $bash_debug_flag -f ${DEV_DATA_DIR}/env_reset rm $bash_debug_flag -f $noncefile + export SYNCER_OFFSET=`eth-info --raw block` confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset fi @@ -55,4 +57,5 @@ fi # Migration variable processing confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset +set +e set +a diff --git a/apps/contract-migration/config/config.ini b/apps/contract-migration/config/config.ini index 78edef7..2d8e93a 100644 --- a/apps/contract-migration/config/config.ini +++ b/apps/contract-migration/config/config.ini @@ -25,3 +25,6 @@ port = [cic] registry_address = trust_address = + +[syncer] +offset = diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 36ec884..5819190 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,5 +1,5 @@ -cic-eth[tools]==0.12.4a13 -chainlib-eth>=0.0.10a15,<0.1.0 +cic-eth[tools]==0.12.5a2 +chainlib-eth>=0.0.10a17,<0.1.0 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 eth-address-index>=0.2.4a1,<0.3.0 diff --git a/apps/data-seeding/cic_eth/import_users.py b/apps/data-seeding/cic_eth/import_users.py index 1322c63..bf66c38 100644 --- a/apps/data-seeding/cic_eth/import_users.py +++ b/apps/data-seeding/cic_eth/import_users.py @@ -40,7 +40,7 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, help='config override directory') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-f', action='store_true', help='force clear previous state') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec') argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') @@ -227,9 +227,10 @@ if __name__ == '__main__': ) os.makedirs(os.path.dirname(filepath), exist_ok=True) - sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) + sub_old_chain_str = '{}:{}'.format(old_chain_spec.network_id(), old_chain_spec.common_name()) + logg.debug('u id {}'.format(u.identities)) f = open(filepath, 'w') - k = u.identities['evm'][sub_old_chain_str][0] + k = u.identities['evm'][old_chain_spec.fork()][sub_old_chain_str][0] tag_data = {'tags': user_tags[strip_0x(k)]} f.write(json.dumps(tag_data)) f.close() diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 6d8b9a2..bff5ed8 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -65,7 +65,8 @@ args_override = { 'REDIS_DB': getattr(args, 'redis_db'), 'META_HOST': getattr(args, 'meta_host'), 'META_PORT': getattr(args, 'meta_port'), - 'WALLET_KEY_FILE': getattr(args, 'y') + 'WALLET_KEY_FILE': getattr(args, 'y'), + 'TOKEN_SYMBOL': getattr(args, 'token_symbol'), } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -110,7 +111,7 @@ def main(): config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer) - ImportTask.balance_processor.init(args.token_symbol) + ImportTask.balance_processor.init(config.get('TOKEN_SYMBOL')) balances = {} accuracy = 10 ** 6 count = 0 diff --git a/apps/data-seeding/config/token.ini b/apps/data-seeding/config/token.ini new file mode 100644 index 0000000..b14d3a0 --- /dev/null +++ b/apps/data-seeding/config/token.ini @@ -0,0 +1,2 @@ +[token] +symbol = diff --git a/apps/data-seeding/config/traffic.ini b/apps/data-seeding/config/traffic.ini new file mode 100644 index 0000000..df89d1a --- /dev/null +++ b/apps/data-seeding/config/traffic.ini @@ -0,0 +1,4 @@ +[traffic] +#local.noop_traffic = 2 +#local.account = 2 +local.transfer = 2 diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index 9afaa5e..f47ca65 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -1,16 +1,25 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45 +ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" + +FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 WORKDIR /root RUN mkdir -vp /usr/local/etc/cic -COPY package.json \ - package-lock.json \ - ./ +ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org} +RUN npm config set snyk=false +#RUN npm config set registry={NPM_REPOSITORY} +RUN npm config set registry=${NPM_REPOSITORY} + +# copy the dependencies +COPY package.json package-lock.json ./ +RUN --mount=type=cache,mode=0755,target=/root/.npm \ + npm set cache /root/.npm && \ + npm cache verify && \ + npm ci --verbose -RUN npm ci --production +#RUN npm ci --production --verbose #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install COPY common/ cic_ussd/common/ @@ -22,6 +31,7 @@ ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL=https://pypi.org/simple RUN pip install --index-url $PIP_INDEX_URL \ + --pre \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh index 4b9e955..8dd4c3e 100755 --- a/apps/data-seeding/import_ussd.sh +++ b/apps/data-seeding/import_ussd.sh @@ -1,5 +1,11 @@ #!/bin/bash +. /tmp/cic/config/env_reset + +OUT_DIR=out +CONFIG_DIR=config +confini-dump --schema-dir $CONFIG_DIR + if [[ -d "$OUT_DIR" ]] then echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m" @@ -19,14 +25,14 @@ then fi echo -e "\033[;96mCreating seed data...\033[;96m" -python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" +python create_import_users.py -vv -c "$CONFIG_DIR" --dir "$OUT_DIR" "$NUMBER_OF_USERS" wait $! echo -e "\033[;96mCheck for running celery workers ...\033[;96m" if [ -f ./cic-ussd-import.pid ]; then echo -e "\033[;96mFound a running worker. Killing ...\033[;96m" - kill -9 $( nohup.out 2> nohup.err < /dev/null & + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" > nohup.out 2> nohup.err < /dev/null & else echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m" TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) - nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" & + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG_DIR" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$WALLET_KEY_FILE" "$OUT_DIR" & fi echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}" @@ -53,12 +59,12 @@ done IMPORT_BALANCE_JOB=$(=0.5.1a1,<=0.5.15 faker==4.17.1 chainsyncer~=0.0.7a3 -chainlib-eth~=0.0.10a10 +chainlib-eth~=0.0.10a18 eth-address-index~=0.2.4a1 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index fd59df4..b01a5e8 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -60,7 +60,7 @@ eth_tests = [ ] phone_tests = [ - 'ussd', +# 'ussd', 'ussd_pins' ] diff --git a/docker-compose.yml b/docker-compose.yml index c6cc85e..1fec4a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -153,6 +153,7 @@ services: SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} + ETH_MIN_FEE_PRICE: $ETH_MIN_FEE_PRICE restart: unless-stopped depends_on: - evm @@ -495,6 +496,9 @@ services: context: apps/cic-ussd dockerfile: docker/Dockerfile args: + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} environment: DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -519,8 +523,8 @@ services: - postgres - redis #- cic-meta-server - - cic-eth-tasker - - cic-cache-tasker + #- cic-eth-tasker + #- cic-cache-tasker volumes: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" @@ -633,6 +637,7 @@ services: context: apps/data-seeding dockerfile: docker/Dockerfile args: + NPM_REPOSITORY: ${DEV_NPM_REPOSITORY:-https://registry.npmjs.org} DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net} @@ -651,27 +656,28 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: config + #OUT_DIR: out + NUMBER_OF_USERS: ${NUMBER_OF_USERS:-10} + #CONFIG_DIR: config CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} - TOKEN_SYMBOL: GFT + TOKEN_SYMBOL: $TOKEN_SYMBOL #KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c} USSD_HOST: cic-user-ussd-server USSD_PORT: 9000 INCLUDE_BALANCES: y - USSD_SSL: n - NOTIFY_DATABASE_NAME: cic_notify + GIFT_THRESHOLD: ${GIFT_THRESHOLD:-0} + USSD_SSL: $USSD_SSL + DATABASE_NAME_NOTIFY: cic_notify REDIS_HOST: redis REDIS_PORT: 6379 REDIS_DB: 0 META_HOST: meta META_PORT: 8000 META_URL: http://meta:8000 + # TODO: this should be generated from host/port/ssl USSD_PROVIDER: http://cic-user-ussd-server:9000 CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd command: - /bin/bash - -c