add some more utils

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,6 +2,7 @@
 import logging
 
 # external imports
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -12,7 +13,6 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
-from eth_erc20 import ERC20
 
 # local imports
 from .base import TagSyncFilter

apps/cic-cache/requirements.txt

@@ -1,12 +1,12 @@
-cic-base~=0.1.2b8
+cic-base~=0.1.2b6
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.5a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.2a2

apps/cic-cache/test_requirements.txt

@@ -8,4 +8,6 @@ eth_tester==0.5.0b3
 py-evm==0.3.0a20
 web3==5.12.2
 cic-eth-registry~=0.5.5a3
-cic-base[full]==0.1.2b8
+giftable-erc20-token~=0.0.8a10
+eth-address-index~=0.1.1a10
+sarafu-faucet~=0.0.3a1

apps/cic-eth/cic_eth/eth/account.py

@@ -3,7 +3,7 @@ import logging
 
 # external imports
 import celery
-from erc20_faucet import Faucet
+from erc20_single_shot_faucet import SingleShotFaucet as Faucet
 from hexathon import (
         strip_0x,
         )
@@ -20,8 +20,8 @@ from chainlib.eth.tx import (
         )
 from chainlib.chain import ChainSpec
 from chainlib.error import JSONRPCException
-from eth_accounts_index.registry import AccountRegistry # TODO, use interface module instead (needs gas limit method)
-from sarafu_faucet import MinterFaucet
+from eth_accounts_index import AccountRegistry
+from sarafu_faucet import MinterFaucet as Faucet
 from chainqueue.db.models.tx import TxCache
 
 # local import
@@ -127,13 +127,13 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
     if writer_address == ZERO_ADDRESS:
         session.close()
         raise RoleMissingError('call address for resgistering {}'.format(account_address))
-    account_registry_address = registry.by_name('AccountsIndex', sender_address=call_address)
+    account_registry_address = registry.by_name('AccountRegistry', sender_address=call_address)
    
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, AccountsIndex.gas)
-    account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
+    gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
+    account_registry = AccountRegistry(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
 
@@ -185,7 +185,7 @@ def gift(self, account_address, chain_spec_dict):
     # Generate and sign transaction
     rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
-    gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
+    gas_oracle = self.create_gas_oracle(rpc, Faucet.gas)
     faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
     (tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
     rpc_signer.disconnect()
@@ -338,7 +338,7 @@ def cache_account_data(
     chain_spec = ChainSpec.from_dict(chain_spec_dict)
     tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
     tx = unpack(tx_signed_raw_bytes, chain_spec)
-    tx_data = AccountsIndex.parse_add_request(tx['data'])
+    tx_data = AccountRegistry.parse_add_request(tx['data'])
 
     session = SessionBase.create_session()
     tx_cache = TxCache(

apps/cic-eth/cic_eth/eth/erc20.py

@@ -6,6 +6,7 @@ import celery
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         TxFormat,
         unpack,
@@ -15,7 +16,6 @@ from cic_eth_registry.erc20 import ERC20Token
 from hexathon import strip_0x
 from chainqueue.db.models.tx import TxCache
 from chainqueue.error import NotLocalTxError
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.db.models.base import SessionBase

apps/cic-eth/cic_eth/ext/address.py

@@ -7,7 +7,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from cic_eth_registry import CICRegistry
-from eth_address_declarator import Declarator
+from eth_address_declarator import AddressDeclarator
 
 # local imports
 from cic_eth.task import BaseTask
@@ -23,12 +23,12 @@ def translate_address(address, trusted_addresses, chain_spec, sender_address=ZER
     registry = CICRegistry(chain_spec, rpc)
     
     declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    c = Declarator(chain_spec)
+    c = AddressDeclarator(chain_spec)
 
     for trusted_address in trusted_addresses:
         o = c.declaration(declarator_address, trusted_address, address, sender_address=sender_address)
         r = rpc.do(o)
-        declaration_hex = Declarator.parse_declaration(r)
+        declaration_hex = AddressDeclarator.parse_declaration(r)
         declaration_hex = declaration_hex[0].rstrip('0')
         declaration_bytes = bytes.fromhex(declaration_hex)
         declaration = None

apps/cic-eth/cic_eth/ext/tx.py

@@ -14,13 +14,13 @@ from chainlib.eth.tx import (
         )
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
+from chainlib.eth.erc20 import ERC20
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.query import get_tx_cache
-from eth_erc20 import ERC20
 
 # local imports
 from cic_eth.queue.time import tx_times

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -13,7 +13,9 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from erc20_faucet import Faucet
+# TODO: use sarafu_Faucet for both when inheritance has been implemented
+from erc20_single_shot_faucet import SingleShotFaucet
+from sarafu_faucet import MinterFaucet as Faucet
 
 # local imports
 from .base import SyncFilter
@@ -69,13 +71,14 @@ class CallbackFilter(SyncFilter):
         #transfer_data['token_address'] = tx.inputs[0]
         faucet_contract = tx.inputs[0]
 
-        o = Faucet.token(faucet_contract, sender_address=self.caller_address)
+        c = SingleShotFaucet(self.chain_spec)
+        o = c.token(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
         transfer_data['token_address'] = add_0x(c.parse_token(r))
 
-        o = c.token_amount(faucet_contract, sender_address=self.caller_address)
+        o = c.amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['value'] = c.parse_token_amount(r)
+        transfer_data['value'] = c.parse_amount(r)
 
         return ('tokengift', transfer_data)
 

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         11,
         0,
-        'beta.12',
+        'beta.11',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b8
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./

apps/cic-eth/requirements.txt

@@ -1,24 +1,25 @@
-cic-base==0.1.2b8
+cic-base==0.1.2b5
 celery==4.4.7
 crypto-dev-signer~=0.4.14b3
 confini~=0.3.6rc3
-cic-eth-registry~=0.5.5a4
+cic-eth-registry~=0.5.4a16
+#cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
-eth_accounts_index~=0.0.11a11
-erc20-transfer-authorization~=0.3.1a6
+eth_accounts_index~=0.0.11a9
+erc20-transfer-authorization~=0.3.1a5
 uWSGI==2.0.19.1
 semver==2.13.0
 websocket-client==0.57.0
 moolb~=0.1.1b2
-eth-address-index~=0.1.1a11
-chainlib~=0.0.3a1
+eth-address-index~=0.1.1a9
+chainlib~=0.0.2a20
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a4
+chainsyncer[sql]~=0.0.2a2
 chainqueue~=0.0.2a2
-sarafu-faucet==0.0.3a1
+pysha3==1.0.2
 coincurve==15.0.0
-sarafu-faucet==0.0.3a2
+sarafu-faucet==0.0.2a28
 potaahto~=0.0.1a1

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -120,7 +120,7 @@ class MetadataRequestsHandler(Metadata):
         data = json.loads(response_data.decode('utf-8'))
         if result.status_code == 200 and self.cic_type == ':cic.person':
             person = Person()
-            deserialized_person = person.deserialize(person_data=data)
+            deserialized_person = person.deserialize(person_data=json.loads(data))
             data = json.dumps(deserialized_person.serialize())
             cache_data(self.metadata_pointer, data=data)
             logg.debug(f'caching: {data} with key: {self.metadata_pointer}')

apps/cic-ussd/cic_ussd/operations.py

@@ -325,14 +325,6 @@ def process_menu_interaction_requests(chain_str: str,
         # get user
         user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
 
-        # retrieve and cache user's metadata
-        blockchain_address = user.blockchain_address
-        s_query_person_metadata = celery.signature(
-            'cic_ussd.tasks.metadata.query_person_metadata',
-            [blockchain_address]
-        )
-        s_query_person_metadata.apply_async(queue='cic-ussd')
-
         # find any existing ussd session
         existing_ussd_session = UssdSession.session.query(UssdSession).filter_by(
             external_session_id=external_session_id).first()

apps/cic-ussd/cic_ussd/processor.py

@@ -371,6 +371,13 @@ def process_start_menu(display_key: str, user: Account):
     # get operational balance
     operational_balance = compute_operational_balance(balances=balances_data)
 
+    # retrieve and cache account's metadata
+    s_query_person_metadata = celery.signature(
+        'cic_ussd.tasks.metadata.query_person_metadata',
+        [blockchain_address]
+    )
+    s_query_person_metadata.apply_async(queue='cic-ussd')
+
     # retrieve and cache account's statement
     retrieve_account_statement(blockchain_address=blockchain_address)
 

apps/contract-migration/docker/Dockerfile

@@ -51,19 +51,17 @@ ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
 # WORKDIR /home/grassroots
 # USER grassroots
 
-ARG pip_extra_args=""
-ARG pip_index_url=https://pypi.org/simple
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2b8
-ARG cic_eth_version=0.11.0b12
-ARG sarafu_token_version=0.0.1a8
-ARG sarafu_faucet_version=0.0.3a2
-RUN pip install --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url \
-	cic-base[full_graph]==$cic_base_version \
+ARG cic_base_version=0.1.2a79
+ARG cic_eth_version=0.11.0b8+build.c2286e5c
+ARG sarafu_faucet_version=0.0.2a28
+ARG sarafu_token_version=0.0.1a6
+ARG cic_contracts_version=0.0.2a2
+RUN pip install  --index-url https://pypi.org/simple --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \
+	cic-contracts==$cic_contracts_version \
 	sarafu-faucet==$sarafu_faucet_version \
-	sarafu-token==$sarafu_token_version \
-	cic-eth==$cic_eth_version 
+	sarafu-token==$sarafu_token_version
 
 # -------------- begin runtime container ---------------- 
 FROM python:3.8.6-slim-buster as runtime-image

apps/contract-migration/scripts/README.md

@@ -104,8 +104,8 @@ If importing using `cic_eth` or `cic_ussd` also run:
 * cic-eth-retrier
 
 If importing using `cic_ussd` also run:
-* cic-user-tasker
-* cic-user-ussd-server
+* cic-ussd-tasker
+* cic-ussd-server
 * cic-notify-tasker
 
 If metadata is to be imported, also run:
@@ -169,24 +169,6 @@ In second terminal:
 
 `python cic_ussd/import_users.py -v -c config out`
 
-Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
-
-In one terminal run:
-
-`python create_import_pins.py -c config -v --userdir <path to the users export dir tree> pinsdir <path to pin export dir tree>`
-
-This script will recursively walk through all the directories defining user data in the users export directory and generate a csv file containing phone numbers and password hashes generated using fernet in a manner reflecting the nature of said hashes in the current sempo system.
-This csv file will be stored in the pins export dir defined as the positional argument.
-
-Once the creation of the pins file is complete, proceed to import the pins and ussd data as follows:
-
-- To import the pins:
-
-`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
-
-- To import ussd data:
-`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
-
 The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.

apps/contract-migration/scripts/cic_ussd/import_pins.py

@@ -1,70 +0,0 @@
-# standard import
-import argparse
-import csv
-import logging
-import os
-
-# third-party imports
-import celery
-import confini
-
-# local imports
-from import_task import *
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_config_dir = './config'
-
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
-arg_parser.add_argument('--env-prefix',
-                        default=os.environ.get('CONFINI_ENV_PREFIX'),
-                        dest='env_prefix',
-                        type=str,
-                        help='environment prefix for variables to overwrite configuration')
-arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit transaction tasks to')
-arg_parser.add_argument('-v', help='be verbose', action='store_true')
-arg_parser.add_argument('-vv', help='be more verbose', action='store_true')
-arg_parser.add_argument('pins_dir', default='out', type=str, help='user export directory')
-args = arg_parser.parse_args()
-
-# set log levels
-if args.v:
-    logg.setLevel(logging.INFO)
-elif args.vv:
-    logg.setLevel(logging.DEBUG)
-
-# process configs
-config_dir = args.c
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-db_configs = {
-    'database': config.get('DATABASE_NAME'),
-    'host': config.get('DATABASE_HOST'),
-    'port': config.get('DATABASE_PORT'),
-    'user': config.get('DATABASE_USER'),
-    'password': config.get('DATABASE_PASSWORD')
-}
-
-
-def main():
-    with open(f'{args.pins_dir}/pins.csv') as pins_file:
-        phone_to_pins = [tuple(row) for row in csv.reader(pins_file)]
-
-    s_import_pins = celery.signature(
-        'import_task.set_pins',
-        (db_configs, phone_to_pins),
-        queue=args.q
-    )
-    s_import_pins.apply_async()
-
-    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
-    celery_app.worker_main(argv)
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/scripts/cic_ussd/import_task.py

@@ -8,8 +8,6 @@ import json
 
 # external imports
 import celery
-import psycopg2
-from psycopg2 import extras
 from hexathon import (
         strip_0x,
         add_0x,
@@ -55,7 +53,7 @@ class MetadataTask(ImportTask):
     def meta_url(self):
         scheme = 'http'
         if self.meta_ssl:
-            scheme += 's'
+            scheme += s
         url = urllib.parse.urlparse('{}://{}:{}/{}'.format(scheme, self.meta_host, self.meta_port, self.meta_path))
         return urllib.parse.urlunparse(url)
 
@@ -93,6 +91,7 @@ def resolve_phone(self, phone):
 def generate_metadata(self, address, phone):
     old_address = old_address_from_phone(self.import_dir, phone)
 
+    logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
     logg.debug('address {}'.format(address))
     old_address_upper = strip_0x(old_address).upper()
     metadata_path = '{}/old/{}/{}/{}.json'.format(
@@ -217,60 +216,3 @@ def send_txs(self, nonce):
 
 
     return nonce
-
-
-@celery_app.task
-def set_pins(config: dict, phone_to_pins: list):
-    # define db connection
-    db_conn = psycopg2.connect(
-        database=config.get('database'),
-        host=config.get('host'),
-        port=config.get('port'),
-        user=config.get('user'),
-        password=config.get('password')
-    )
-    db_cursor = db_conn.cursor()
-
-    # update db
-    for element in phone_to_pins:
-        sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
-        db_cursor.execute(sql, (element[1], element[0]))
-        logg.debug(f'Updating: {element[0]} with: {element[1]}')
-
-    # commit changes
-    db_conn.commit()
-
-    # close connections
-    db_cursor.close()
-    db_conn.close()
-
-
-@celery_app.task
-def set_ussd_data(config: dict, ussd_data: dict):
-    # define db connection
-    db_conn = psycopg2.connect(
-        database=config.get('database'),
-        host=config.get('host'),
-        port=config.get('port'),
-        user=config.get('user'),
-        password=config.get('password')
-    )
-    db_cursor = db_conn.cursor()
-
-    # process ussd_data
-    account_status = 1
-    if ussd_data['is_activated'] == 1:
-        account_status = 2
-    preferred_language = ussd_data['preferred_language']
-    phone_number = ussd_data['phone']
-
-    sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s'
-    db_cursor.execute(sql, (account_status, preferred_language, phone_number))
-
-    # commit changes
-    db_conn.commit()
-
-    # close connections
-    db_cursor.close()
-    db_conn.close()
-

apps/contract-migration/scripts/cic_ussd/import_users.py

@@ -87,13 +87,6 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
-db_configs = {
-    'database': config.get('DATABASE_NAME'),
-    'host': config.get('DATABASE_HOST'),
-    'port': config.get('DATABASE_PORT'),
-    'user': config.get('DATABASE_USER'),
-    'password': config.get('DATABASE_PASSWORD')
-}
   
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
@@ -142,60 +135,57 @@ if __name__ == '__main__':
         for y in x[2]:
             if y[len(y)-5:] != '.json':
                 continue
-            # handle json containing person object
-            filepath = None
-            if y != 'ussd_data.json':
-                filepath = os.path.join(x[0], y)
-                f = open(filepath, 'r')
-                try:
-                    o = json.load(f)
-                except json.decoder.JSONDecodeError as e:
-                    f.close()
-                    logg.error('load error for {}: {}'.format(y, e))
-                    continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
                 f.close()
-                u = Person.deserialize(o)
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person.deserialize(o)
 
-                new_address = register_ussd(i, u)
+            new_address = register_ussd(i, u)
 
-                phone_object = phonenumbers.parse(u.tel)
-                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+            phone_object = phonenumbers.parse(u.tel)
+            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-                s_phone = celery.signature(
-                        'import_task.resolve_phone',
-                        [
-                            phone,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_phone = celery.signature(
+                    'import_task.resolve_phone',
+                    [
+                        phone,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_meta = celery.signature(
-                        'import_task.generate_metadata',
-                        [
-                            phone,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_meta = celery.signature(
+                    'import_task.generate_metadata',
+                    [
+                        phone,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_balance = celery.signature(
-                        'import_task.opening_balance_tx',
-                        [
-                            phone,
-                            i,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_balance = celery.signature(
+                    'import_task.opening_balance_tx',
+                    [
+                        phone,
+                        i,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_meta.link(s_balance)
-                s_phone.link(s_meta)
-                # block time plus a bit of time for ussd processing
-                s_phone.apply_async(countdown=7)
+            s_meta.link(s_balance)
+            s_phone.link(s_meta)
+            s_phone.apply_async(countdown=7) # block time plus a bit of time for ussd processing
 
-                i += 1
-                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
-
-                j += 1
-                if j == batch_size:
-                    time.sleep(batch_delay)
-                    j = 0
+            i += 1
+            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+        
+            j += 1
+            if j == batch_size:
+                time.sleep(batch_delay)
+                j = 0
 
+    #fi.close()

apps/contract-migration/scripts/cic_ussd/import_ussd_data.py

@@ -1,70 +0,0 @@
-# standard imports
-import argparse
-import json
-import logging
-import os
-
-# external imports
-import celery
-from confini import Config
-
-# local imports
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_config_dir = '/usr/local/etc/cic'
-
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument('-c', type=str, default=default_config_dir, help='config file')
-arg_parser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
-arg_parser.add_argument('-v', action='store_true', help='Be verbose')
-arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
-arg_parser.add_argument('user_dir', type=str, help='path to users export dir tree')
-args = arg_parser.parse_args()
-
-if args.v:
-    logg.setLevel(logging.INFO)
-elif args.vv:
-    logg.setLevel(logging.DEBUG)
-
-config_dir = args.c
-config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-
-user_old_dir = os.path.join(args.user_dir, 'old')
-os.stat(user_old_dir)
-
-db_configs = {
-    'database': config.get('DATABASE_NAME'),
-    'host': config.get('DATABASE_HOST'),
-    'port': config.get('DATABASE_PORT'),
-    'user': config.get('DATABASE_USER'),
-    'password': config.get('DATABASE_PASSWORD')
-}
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-if __name__ == '__main__':
-    for x in os.walk(user_old_dir):
-        for y in x[2]:
-
-            if y[len(y) - 5:] != '.json':
-                continue
-
-            # handle ussd_data json object
-            if y == 'ussd_data.json':
-                filepath = os.path.join(x[0], y)
-                f = open(filepath, 'r')
-                try:
-                    ussd_data = json.load(f)
-                except json.decoder.JSONDecodeError as e:
-                    f.close()
-                    logg.error('load error for {}: {}'.format(y, e))
-                    continue
-                f.close()
-
-                s_set_ussd_data = celery.signature(
-                    'import_task.set_ussd_data',
-                    [db_configs, ussd_data]
-                )
-                s_set_ussd_data.apply_async(queue='cic-import-ussd')

apps/contract-migration/scripts/create_import_pins.py

@@ -1,90 +0,0 @@
-# standard imports
-import argparse
-import json
-import logging
-import os
-
-# third-party imports
-import bcrypt
-import celery
-import confini
-import phonenumbers
-import random
-from cic_types.models.person import Person
-from cryptography.fernet import Fernet
-
-# local imports
-
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-script_dir = os.path.realpath(os.path.dirname(__file__))
-default_config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
-
-arg_parser = argparse.ArgumentParser()
-arg_parser.add_argument('-c', type=str, default=default_config_dir, help='Config dir')
-arg_parser.add_argument('-v', action='store_true', help='Be verbose')
-arg_parser.add_argument('-vv', action='store_true', help='Be more verbose')
-arg_parser.add_argument('--userdir', type=str, help='path to users export dir tree')
-arg_parser.add_argument('pins_dir', type=str, help='path to pin export dir tree')
-
-
-args = arg_parser.parse_args()
-
-if args.v:
-    logg.setLevel(logging.INFO)
-elif args.vv:
-    logg.setLevel(logging.DEBUG)
-
-config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-logg.info('loaded config\n{}'.format(config))
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-user_dir = args.userdir
-pins_dir = args.pins_dir
-
-
-def generate_password_hash():
-    key = Fernet.generate_key()
-    fnt = Fernet(key)
-    pin = str(random.randint(1000, 9999))
-    return fnt.encrypt(bcrypt.hashpw(pin.encode('utf-8'), bcrypt.gensalt())).decode()
-
-
-user_old_dir = os.path.join(user_dir, 'old')
-logg.debug(f'reading user data from: {user_old_dir}')
-
-pins_file = open(f'{pins_dir}/pins.csv', 'w')
-
-if __name__ == '__main__':
-
-    for x in os.walk(user_old_dir):
-        for y in x[2]:
-            # skip non-json files
-            if y[len(y) - 5:] != '.json':
-                continue
-
-            # define file path for
-            filepath = None
-            if y != 'ussd_data.json':
-                filepath = os.path.join(x[0], y)
-                f = open(filepath, 'r')
-                try:
-                    o = json.load(f)
-                except json.decoder.JSONDecodeError as e:
-                    f.close()
-                    logg.error('load error for {}: {}'.format(y, e))
-                    continue
-                f.close()
-                u = Person.deserialize(o)
-
-                phone_object = phonenumbers.parse(u.tel)
-                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
-                password_hash = generate_password_hash()
-                pins_file.write(f'{phone},{password_hash}\n')
-                logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
-
-    pins_file.close()

apps/contract-migration/scripts/create_import_users.py

@@ -130,7 +130,6 @@ def genCats():
 def genAmount():
     return random.randint(0, gift_max) * gift_factor
 
-
 def genDob():
     dob_src = fake.date_of_birth(minimum_age=15)
     dob = {}
@@ -169,9 +168,8 @@ def gen():
             }
     p.location['area_name'] = city
     if random.randint(0, 1):
-        p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
-        p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
-
+        p.identities['latitude'] = (random.random() + 180) - 90 #fake.local_latitude()
+        p.identities['longitude'] = (random.random() + 360) - 180 #fake.local_latitude()
     
     return (old_blockchain_checksum_address, phone, p)
 
@@ -217,20 +215,6 @@ if __name__ == '__main__':
         json.dump(o.serialize(), f)
         f.close()
 
-        # create ussd data
-        ussd_data_dir = os.path.join(d, 'ussd_data')
-        os.makedirs(ussd_data_dir)
-        f = open('{}/{}/{}'.format(d, 'ussd_data', 'ussd_data.json'), 'w')
-        ussd_data = {
-            'phone': phone,
-            'is_activated': 1,
-            'preferred_language': random.sample(['en', 'sw'], 1)[0],
-            'is_disabled': False
-        }
-        json.dump(ussd_data, f)
-        f.close()
-
-
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2b8
-sarafu-faucet==0.0.3a2
-cic-eth==0.11.0b12
+cic-base[full_graph]==0.1.2b2
+sarafu-faucet==0.0.2a28
+cic-eth==0.11.0b10
 cic-types==0.1.0a10
 crypto-dev-signer==0.4.14b3

apps/contract-migration/scripts/verify.py

@@ -1,43 +1,52 @@
 # standard imports
-import argparse
-import copy
-import csv
-import hashlib
-import json
-import logging
 import os
 import sys
+import logging
+import time
+import argparse
+import sys
+import re
+import hashlib
+import csv
+import json
 import urllib
-import urllib.request
+import copy
 import uuid
-from collections import Counter
+import urllib.request
 
 # external imports
 import celery
-import confini
 import eth_abi
-import psycopg2
+import confini
+from hexathon import (
+        strip_0x,
+        add_0x,
+        )
+from chainsyncer.backend.memory import MemBackend
+from chainsyncer.driver import HeadSyncer
 from chainlib.chain import ChainSpec
-from chainlib.eth.address import to_checksum_address
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.constant import ZERO_ADDRESS
+from chainlib.eth.block import (
+        block_latest,
+        block_by_number,
+        Block,
+        )
+from chainlib.hash import keccak256_string_to_hex
+from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
-    OverrideGasOracle,
-    balance,
-)
+        OverrideGasOracle,
+        balance,
+        )
 from chainlib.eth.tx import TxFactory
-from chainlib.hash import keccak256_string_to_hex
 from chainlib.jsonrpc import jsonrpc_template
+from chainlib.eth.error import EthException
 from cic_types.models.person import (
-    Person,
-    generate_metadata_pointer,
-)
+        Person,
+        generate_metadata_pointer,
+        )
 from erc20_single_shot_faucet import SingleShotFaucet
-from hexathon import (
-    strip_0x,
-    add_0x,
-)
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -63,7 +72,6 @@ eth_tests = [
 
 phone_tests = [
         'ussd',
-        'ussd_pins'
         ]
 
 all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
@@ -163,39 +171,6 @@ if logg.isEnabledFor(logging.DEBUG):
     outfunc = logg.debug
 
 
-def send_ussd_request(address, data_dir):
-    upper_address = strip_0x(address).upper()
-    f = open(os.path.join(
-        data_dir,
-        'new',
-        upper_address[:2],
-        upper_address[2:4],
-        upper_address + '.json',
-    ), 'r'
-    )
-    o = json.load(f)
-    f.close()
-
-    p = Person.deserialize(o)
-    phone = p.tel
-
-    session = uuid.uuid4().hex
-    data = {
-        'sessionId': session,
-        'serviceCode': config.get('APP_SERVICE_CODE'),
-        'phoneNumber': phone,
-        'text': '',
-    }
-
-    req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-    data_str = json.dumps(data)
-    data_bytes = data_str.encode('utf-8')
-    req.add_header('Content-Type', 'application/json')
-    req.data = data_bytes
-    response = urllib.request.urlopen(req)
-    return response.read().decode('utf-8')
-
-
 class VerifierState:
 
     def __init__(self, item_keys, active_tests=None):
@@ -379,18 +354,42 @@ class Verifier:
 
 
     def verify_ussd(self, address, balance=None):
-        response_data = send_ussd_request(address, self.data_dir)
+        upper_address = strip_0x(address).upper()
+        f = open(os.path.join(
+            self.data_dir,
+            'new',
+            upper_address[:2],
+            upper_address[2:4],
+            upper_address + '.json',
+            ), 'r'
+            )
+        o = json.load(f)
+        f.close()
+
+        p = Person.deserialize(o)
+        phone = p.tel
+
+        session = uuid.uuid4().hex
+        data = {
+                'sessionId': session,
+                'serviceCode': config.get('APP_SERVICE_CODE'),
+                'phoneNumber': phone,
+                'text': config.get('APP_SERVICE_CODE'),
+            }
+
+        req = urllib.request.Request(config.get('_USSD_PROVIDER'))
+        data_str = json.dumps(data)
+        data_bytes = data_str.encode('utf-8')
+        req.add_header('Content-Type', 'application/json')
+        req.data = data_bytes
+        response = urllib.request.urlopen(req)
+        response_data = response.read().decode('utf-8')
         state = response_data[:3]
         out = response_data[4:]
         m = '{} {}'.format(state, out[:7])
         if m != 'CON Welcome':
             raise VerifierError(response_data, 'ussd')
 
-    def verify_ussd_pins(self, address, balance):
-        response_data = send_ussd_request(address, self.data_dir)
-        if response_data[:11] != 'CON Balance':
-            raise VerifierError(response_data, 'pins')
-
 
     def verify(self, address, balance, debug_stem=None):
   

apps/contract-migration/seed_cic_eth.sh

@@ -99,7 +99,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer sarafu tokens to token gifter address"
->&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 eth-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 

docker-compose.yml

@@ -76,9 +76,6 @@ services:
 
   contract-migration:
     build:
-      args:
-        pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
-        pip_extra_args: $PIP_EXTRA_ARGS
       context: apps/
       dockerfile: contract-migration/docker/Dockerfile
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest