diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index 0746df7f..ce69b14b 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -23,15 +23,20 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.connection import RPCConnection from chainlib.eth.block import ( block_latest, + block_by_number, + Block, ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface from hexathon import ( strip_0x, ) from chainsyncer.backend.sql import SQLBackend -from chainsyncer.driver import ( - HeadSyncer, - HistorySyncer, - ) +from chainsyncer.driver.head import HeadSyncer +from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports @@ -83,6 +88,16 @@ def register_filter_tags(filters, session): session.rollback() logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1])) +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() + def main(): # Connect to blockchain with chainlib @@ -113,10 +128,10 @@ def main(): logg.info('resuming sync session {}'.format(syncer_backend)) for syncer_backend in syncer_backends: - syncers.append(HistorySyncer(syncer_backend)) + syncers.append(HistorySyncer(syncer_backend, chain_interface)) syncer_backend = SQLBackend.live(chain_spec, block_offset+1) - syncers.append(HeadSyncer(syncer_backend)) + syncers.append(HeadSyncer(syncer_backend, chain_interface)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 865db82e..705e4211 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,4 +1,4 @@ -cic-base~=0.1.3a1 +cic-base~=0.1.3a2 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 @@ -9,5 +9,5 @@ semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.3a1 +chainsyncer[sql]~=0.0.3a3 erc20-faucet~=0.2.2a1 diff --git a/apps/cic-cache/scripts/migrate.py b/apps/cic-cache/scripts/migrate.py index 6db1ed64..b6925144 100644 --- a/apps/cic-cache/scripts/migrate.py +++ b/apps/cic-cache/scripts/migrate.py @@ -2,6 +2,7 @@ import os import argparse import logging +import re import alembic from alembic.config import Config as AlembicConfig @@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') +argparser.add_argument('-f', action='store_true', help='force action') argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() @@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini')) ac.set_main_option('sqlalchemy.url', dsn) ac.set_main_option('script_location', migrations_dir) +if args.reset: + if not args.f: + if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')): + logg.error('user chickened out on requested reset, bailing') + sys.exit(1) + alembic.command.downgrade(ac, 'base') alembic.command.upgrade(ac, 'head') diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 58d33eca..c44cce0b 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -21,7 +21,14 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.connection import RPCConnection from chainlib.eth.block import ( block_latest, + block_by_number, + Block, ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface from hexathon import ( strip_0x, ) @@ -78,6 +85,17 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() + + def main(): # connect to celery celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) @@ -119,11 +137,11 @@ def main(): for syncer_backend in syncer_backends: try: - syncers.append(HistorySyncer(syncer_backend)) + syncers.append(HistorySyncer(syncer_backend, chain_interface)) logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend)) except AttributeError: logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) - syncers.append(HeadSyncer(syncer_backend)) + syncers.append(HeadSyncer(syncer_backend, chain_interface)) connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 1187eeff..66e49c2a 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ -cic-base~=0.1.3a1 +cic-base~=0.1.3a2 celery==4.4.7 crypto-dev-signer~=0.4.14b6 confini~=0.3.6rc3 @@ -16,7 +16,7 @@ moolb~=0.1.1b2 eth-address-index~=0.1.2a1 chainlib-eth~=0.0.5a1 hexathon~=0.0.1a7 -chainsyncer[sql]~=0.0.3a1 +chainsyncer[sql]~=0.0.3a3 chainqueue~=0.0.2b5 sarafu-faucet~=0.0.4a1 erc20-faucet~=0.2.2a1 diff --git a/apps/cic-eth/scripts/migrate.py b/apps/cic-eth/scripts/migrate.py index 4a2c0139..d3602989 100644 --- a/apps/cic-eth/scripts/migrate.py +++ b/apps/cic-eth/scripts/migrate.py @@ -2,6 +2,8 @@ import os import argparse import logging +import re +import sys import alembic from alembic.config import Config as AlembicConfig @@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, default=config_dir, help='config file') argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') +argparser.add_argument('-f', action='store_true', help='force action') argparser.add_argument('-v', action='store_true', help='be verbose') argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() @@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini')) ac.set_main_option('sqlalchemy.url', dsn) ac.set_main_option('script_location', migrations_dir) +if args.reset: + if not args.f: + if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')): + logg.error('user chickened out on requested reset, bailing') + sys.exit(1) + alembic.command.downgrade(ac, 'base') alembic.command.upgrade(ac, 'head') diff --git a/apps/data-seeding/cic_eth/import_balance.py b/apps/data-seeding/cic_eth/import_balance.py index 7a8aa92c..cb0a48e6 100644 --- a/apps/data-seeding/cic_eth/import_balance.py +++ b/apps/data-seeding/cic_eth/import_balance.py @@ -18,13 +18,18 @@ from hexathon import ( add_0x, ) from chainsyncer.backend.memory import MemBackend -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( block_latest, block_by_number, Block, ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface from chainlib.hash import keccak256_string_to_hex from chainlib.eth.address import to_checksum_address from chainlib.eth.gas import OverrideGasOracle @@ -70,13 +75,14 @@ elif args.vv == True: config_dir = os.path.join(args.c) os.makedirs(config_dir, 0o777, True) config = confini.Config(config_dir, args.env_prefix) -config.process() # override args +config.process() +logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'KEYSTORE_FILE_PATH': getattr(args, 'key-file') + 'KEYSTORE_FILE_PATH': getattr(args, 'y'), } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -109,6 +115,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py token_symbol = args.token_symbol +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() + class Handler: @@ -308,7 +324,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/data-seeding/config/cic.ini b/apps/data-seeding/config/cic.ini index f58bf250..5064df15 100644 --- a/apps/data-seeding/config/cic.ini +++ b/apps/data-seeding/config/cic.ini @@ -7,4 +7,5 @@ approval_escrow_address = chain_spec = evm:bloxberg:8996 tx_retry_delay = trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -user_ussd_svc_service_port= \ No newline at end of file +user_ussd_svc_service_port = + diff --git a/apps/data-seeding/config/eth.ini b/apps/data-seeding/config/eth.ini index 54eed6ab..321384f8 100644 --- a/apps/data-seeding/config/eth.ini +++ b/apps/data-seeding/config/eth.ini @@ -1,8 +1,2 @@ [eth] -#ws_provider = ws://localhost:8546 -#ttp_provider = http://localhost:8545 provider = http://localhost:63545 -gas_provider_address = -#chain_id = -abi_dir = /usr/local/share/cic/solidity/abi -account_accounts_index_writer = diff --git a/apps/data-seeding/create_import_users.py b/apps/data-seeding/create_import_users.py index 0e0b51ad..dc252fce 100644 --- a/apps/data-seeding/create_import_users.py +++ b/apps/data-seeding/create_import_users.py @@ -204,9 +204,9 @@ def gen(): ])) if random.randint(0, 1): # fake.local_latitude() - p.location['latitude'] = (random.random() + 180) - 90 + p.location['latitude'] = (random.random() * 180) - 90 # fake.local_latitude() - p.location['longitude'] = (random.random() + 360) - 180 + p.location['longitude'] = (random.random() * 360) - 179 return (old_blockchain_checksum_address, phone, p) diff --git a/apps/data-seeding/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py index 2e977d93..afabc16c 100644 --- a/apps/data-seeding/eth/import_balance.py +++ b/apps/data-seeding/eth/import_balance.py @@ -18,13 +18,18 @@ from hexathon import ( add_0x, ) from chainsyncer.backend.memory import MemBackend -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( block_latest, block_by_number, Block, ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface from chainlib.hash import keccak256_string_to_hex from chainlib.eth.address import to_checksum_address from chainlib.eth.gas import OverrideGasOracle @@ -75,7 +80,7 @@ args_override = { 'CIC_CHAIN_SPEC': getattr(args, 'i'), 'ETH_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'KEYSTORE_FILE_PATH': getattr(args, 'key-file') + 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli flag') config.censor('PASSWORD', 'DATABASE') @@ -108,6 +113,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py token_symbol = args.token_symbol +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() + class Handler: @@ -303,7 +318,7 @@ def main(): f.close() syncer_backend.set(block_offset, 0) - syncer = HeadSyncer(syncer_backend, block_callback=progress_callback) + syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback) handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle) syncer.add_filter(handler) syncer.loop(1, conn) diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index 8ef9fd84..be04b2f2 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -59,7 +59,7 @@ config.process() args_override = { 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'KEYSTORE_FILE_PATH': getattr(args, 'key-file') + 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli') config.add(args.user_dir, '_USERDIR', True) diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 8e014acb..b09bb5d6 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,4 +1,4 @@ -cic-base[full_graph]==0.1.3a1 +cic-base[full_graph]==0.1.3a2 sarafu-faucet==0.0.4a1 cic-eth==0.11.1a1 cic-types==0.1.0a13