Rehabilitate imports

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -23,15 +23,20 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.connection import RPCConnection
 from chainlib.eth.block import (
         block_latest,
+        block_by_number,
+        Block,
         )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
 from hexathon import (
         strip_0x,
         )
 from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver import (
+from chainsyncer.driver.head import HeadSyncer
-        HeadSyncer,
+from chainsyncer.driver.history import HistorySyncer
-        HistorySyncer,
-        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
@@ -83,6 +88,16 @@ def register_filter_tags(filters, session):
             session.rollback()
             logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
 
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()
+
 
 def main():
     # Connect to blockchain with chainlib
@@ -113,10 +128,10 @@ def main():
             logg.info('resuming sync session {}'.format(syncer_backend))
 
     for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend))
+        syncers.append(HistorySyncer(syncer_backend, chain_interface))
 
     syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend))
+    syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-cache/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.3a1
+cic-base~=0.1.3a2
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
@@ -9,5 +9,5 @@ semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.3a1
+chainsyncer[sql]~=0.0.3a3
 erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -2,6 +2,7 @@
 import os
 import argparse
 import logging
+import re
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -21,7 +21,14 @@ from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.connection import RPCConnection
 from chainlib.eth.block import (
         block_latest,
+        block_by_number,
+        Block,
         )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
 from hexathon import (
         strip_0x,
         )
@@ -78,6 +85,17 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()
+
+
 def main():
     # connect to celery
     celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@@ -119,11 +137,11 @@ def main():
 
     for syncer_backend in syncer_backends:
         try:
-            syncers.append(HistorySyncer(syncer_backend))
+            syncers.append(HistorySyncer(syncer_backend, chain_interface))
             logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
         except AttributeError:
             logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
-            syncers.append(HeadSyncer(syncer_backend))
+            syncers.append(HeadSyncer(syncer_backend, chain_interface))
 
     connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
 

apps/cic-eth/requirements.txt

@@ -1,4 +1,4 @@
-cic-base~=0.1.3a1
+cic-base~=0.1.3a2
 celery==4.4.7
 crypto-dev-signer~=0.4.14b6
 confini~=0.3.6rc3
@@ -16,7 +16,7 @@ moolb~=0.1.1b2
 eth-address-index~=0.1.2a1
 chainlib-eth~=0.0.5a1
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.3a1
+chainsyncer[sql]~=0.0.3a3
 chainqueue~=0.0.2b5
 sarafu-faucet~=0.0.4a1
 erc20-faucet~=0.2.2a1

apps/cic-eth/scripts/migrate.py

@@ -2,6 +2,8 @@
 import os
 import argparse
 import logging
+import re
+import sys
 
 import alembic
 from alembic.config import Config as AlembicConfig
@@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('-c', type=str, default=config_dir, help='config file')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
+argparser.add_argument('-f', action='store_true', help='force action')
 argparser.add_argument('-v', action='store_true', help='be verbose')
 argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
@@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
 ac.set_main_option('sqlalchemy.url', dsn)
 ac.set_main_option('script_location', migrations_dir)
 
+if args.reset:
+    if not args.f:
+        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
+            logg.error('user chickened out on requested reset, bailing')
+            sys.exit(1)
+    alembic.command.downgrade(ac, 'base')
 alembic.command.upgrade(ac, 'head')

apps/data-seeding/cic_eth/import_balance.py

@@ -18,13 +18,18 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
         block_by_number,
         Block,
         )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
@@ -70,13 +75,14 @@ elif args.vv == True:
 config_dir = os.path.join(args.c)
 os.makedirs(config_dir, 0o777, True)
 config = confini.Config(config_dir, args.env_prefix)
-config.process()
 # override args
+config.process()
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
 args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y'),
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -109,6 +115,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py
 
 token_symbol = args.token_symbol
 
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()
+
 
 class Handler:
 
@@ -308,7 +324,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/config/cic.ini

@@ -7,4 +7,5 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
-user_ussd_svc_service_port=
+user_ussd_svc_service_port =
+

apps/data-seeding/config/eth.ini

@@ -1,8 +1,2 @@
 [eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
 provider = http://localhost:63545
-gas_provider_address = 
-#chain_id = 
-abi_dir = /usr/local/share/cic/solidity/abi
-account_accounts_index_writer = 

apps/data-seeding/create_import_users.py

@@ -204,9 +204,9 @@ def gen():
          ]))
     if random.randint(0, 1):
         # fake.local_latitude()
-        p.location['latitude'] = (random.random() + 180) - 90
+        p.location['latitude'] = (random.random() * 180) - 90
         # fake.local_latitude()
-        p.location['longitude'] = (random.random() + 360) - 180
+        p.location['longitude'] = (random.random() * 360) - 179
 
     return (old_blockchain_checksum_address, phone, p)
 

apps/data-seeding/eth/import_balance.py

@@ -18,13 +18,18 @@ from hexathon import (
         add_0x,
         )
 from chainsyncer.backend.memory import MemBackend
-from chainsyncer.driver import HeadSyncer
+from chainsyncer.driver.head import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
         block_latest,
         block_by_number,
         Block,
         )
+from chainlib.eth.tx import (
+        receipt,
+        Tx,
+        )
+from chainlib.interface import ChainInterface
 from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.gas import OverrideGasOracle
@@ -75,7 +80,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -108,6 +113,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py
 
 token_symbol = args.token_symbol
 
+class EthChainInterface(ChainInterface):
+    
+    def __init__(self):
+        self._tx_receipt = receipt
+        self._block_by_number = block_by_number
+        self._block_from_src = Block.from_src
+        self._src_normalize = Tx.src_normalize
+
+chain_interface = EthChainInterface()
+
 
 class Handler:
 
@@ -303,7 +318,7 @@ def main():
     f.close()
 
     syncer_backend.set(block_offset, 0)
-    syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
+    syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
     handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
     syncer.add_filter(handler)
     syncer.loop(1, conn)

apps/data-seeding/eth/import_users.py

@@ -59,7 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
+        'KEYSTORE_FILE_PATH': getattr(args, 'y')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/requirements.txt

@@ -1,4 +1,4 @@
-cic-base[full_graph]==0.1.3a1
+cic-base[full_graph]==0.1.3a2
 sarafu-faucet==0.0.4a1
 cic-eth==0.11.1a1
 cic-types==0.1.0a13