Upgrade sarafu-faucet in cic-eth

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,6 +194,7 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
+    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/requirements.txt

@@ -18,8 +18,8 @@ chainlib~=0.0.3rc2
 hexathon~=0.0.1a7
 chainsyncer[sql]~=0.0.2a5
 chainqueue~=0.0.2b3
-sarafu-faucet==0.0.3a3
+sarafu-faucet~=0.0.3a3
-erc20-faucet==0.2.1a4
+erc20-faucet~=0.2.1a5
 coincurve==15.0.0
 potaahto~=0.0.1a2
 pycryptodome==3.10.1

apps/cic-ussd/cic_ussd/processor.py

@@ -423,6 +423,12 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
+    # retrieve metadata before any transition
+    key = generate_metadata_pointer(
+        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
+        cic_type=':cic.person'
+    )
+    person_metadata = get_cached_data(key=key)
 
     if ussd_session:
         if user_input == "0":
@@ -446,7 +452,7 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                     'exit_pin_mismatch',
                     'exit_invalid_request',
                     'exit_successful_transaction'
-                ]:
+                ] and person_metadata is not None:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -145,7 +145,7 @@ def application(env, start_response):
     if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
 
         if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
-            start_response('405 Urlencoded, please', errors_headers)
+            start_response('405 Play by the rules', errors_headers)
             return []
 
         post_data = env.get('wsgi.input').read()
@@ -213,9 +213,6 @@ def application(env, start_response):
         return [response_bytes]
 
     else:
-        logg.error('invalid query {}'.format(env))
-        for r in env:
-            logg.debug('{}: {}'.format(r, env))
         start_response('405 Play by the rules', errors_headers)
         return []
 

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b17
+cic_base[full_graph]~=0.1.2b15
-cic-eth~=0.11.0b17
+cic-eth~=0.11.0b16
 cic-notify~=0.4.0a5
 cic-types~=0.1.0a10

apps/contract-migration/reset.sh

@@ -76,6 +76,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	>&2 echo "add deployer address as account index writer"
 	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
+	>&2 echo "deploy contract registry contract"
 	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
@@ -87,7 +88,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
-	>&2 echo "deploy address declarator contract"
+	>&2 echo "deploy transfer auth contract"
 	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 

apps/data-seeding/README.md

@@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything:
 
 To import, run to _completion_:
 
-`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
+`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
 
 After the script completes, keystore files for all generated accouts will be found in `<datadir>/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one).
 
@@ -150,7 +150,7 @@ Then run:
 
 Run in sequence, in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
 
 In another terminal:
 
@@ -226,7 +226,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in
 
 ### Step 5 - Verify
 
-`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>` 
+`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>` 
 
 Included checks:
 * Private key is in cic-eth keystore
@@ -262,5 +262,3 @@ Should exit with code 0 if all input data is found in the respective services.
 - MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953
 
 - `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it.
-
-- A strict constraint is maintained insistin the use of postgresql-12.

apps/data-seeding/cic_ussd/import_balance.py

@@ -114,7 +114,7 @@ def main():
     conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
    
     ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer)
-    ImportTask.balance_processor.init(token_symbol)
+    ImportTask.balance_processor.init()
 
     # TODO get decimals from token
     balances = {}
@@ -139,7 +139,6 @@ def main():
 
     ImportTask.balances = balances
     ImportTask.count = i
-    ImportTask.import_dir = user_dir
 
     s = celery.signature(
             'import_task.send_txs',

apps/data-seeding/cic_ussd/import_pins.py

@@ -39,7 +39,6 @@ elif args.vv:
 config_dir = args.c
 config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
@@ -63,6 +62,9 @@ def main():
     )
     s_import_pins.apply_async()
 
+    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
+    celery_app.worker_main(argv)
+
 
 if __name__ == '__main__':
     main()

apps/data-seeding/cic_ussd/import_users.py

@@ -1,21 +1,29 @@
 # standard imports
-import argparse
-import json
-import logging
 import os
 import sys
+import json
+import logging
+import argparse
+import uuid
+import datetime
 import time
 import urllib.request
-import uuid
+from glob import glob
-from urllib.parse import urlencode
 
-# external imports
+# third-party imports
-import celery
-import confini
-import phonenumbers
 import redis
-from chainlib.chain import ChainSpec
+import confini
+import celery
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from chainlib.eth.address import to_checksum
 from cic_types.models.person import Person
+from cic_eth.api.api_task import Api
+from chainlib.chain import ChainSpec
+from cic_types.processor import generate_metadata_pointer
+import phonenumbers
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -79,13 +87,21 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
+db_configs = {
+    'database': config.get('DATABASE_NAME'),
+    'host': config.get('DATABASE_HOST'),
+    'port': config.get('DATABASE_PORT'),
+    'user': config.get('DATABASE_USER'),
+    'password': config.get('DATABASE_PASSWORD')
+}
+  
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
     url = 'http'
     if ssl:
         url += 's'
     url += '://{}:{}'.format(host, port)
-    url += '/?username={}&password={}'.format(username, password)
+    url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS'))
 
     logg.info('ussd service url {}'.format(url))
     logg.info('ussd phone {}'.format(phone))
@@ -98,10 +114,9 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl=
             'text': service_code,
         }
     req = urllib.request.Request(url)
-    req.method=('POST')
+    data_str = json.dumps(data)
-    data_str = urlencode(data)
     data_bytes = data_str.encode('utf-8')
-    req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+    req.add_header('Content-Type', 'application/json')
     req.data = data_bytes
 
     return req

apps/data-seeding/cic_ussd/import_ussd_data.py

@@ -31,7 +31,9 @@ elif args.vv:
 config_dir = args.c
 config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
-logg.debug('config loaded from {}:\n{}'.format(args.c, config))
+
+user_old_dir = os.path.join(args.user_dir, 'old')
+os.stat(user_old_dir)
 
 db_configs = {
     'database': config.get('DATABASE_NAME'),
@@ -43,15 +45,18 @@ db_configs = {
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 if __name__ == '__main__':
-    for x in os.walk(args.user_dir):
+    for x in os.walk(user_old_dir):
         for y in x[2]:
 
-            if y[len(y) - 5:] == '.json':
+            if y[len(y) - 5:] != '.json':
+                continue
+
+            # handle ussd_data json object
+            if y[:15] == '_ussd_data.json':
                 filepath = os.path.join(x[0], y)
                 f = open(filepath, 'r')
                 try:
                     ussd_data = json.load(f)
-                    logg.debug(f'LOADING USSD DATA: {ussd_data}')
                 except json.decoder.JSONDecodeError as e:
                     f.close()
                     logg.error('load error for {}: {}'.format(y, e))

apps/data-seeding/cic_ussd/import_util.py

@@ -6,7 +6,7 @@ from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import OverrideNonceOracle
-from eth_erc20 import ERC20
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.tx import (
         count,
         TxFormat,
@@ -37,7 +37,7 @@ class BalanceProcessor:
         self.value_multiplier = 1
     
 
-    def init(self, token_symbol):
+    def init(self):
         # Get Token registry address
         registry = Registry(self.chain_spec)
         o = registry.address_of(self.registry_address, 'TokenRegistry')
@@ -46,10 +46,10 @@ class BalanceProcessor:
         logg.info('found token index address {}'.format(self.token_index_address))
 
         token_registry = TokenUniqueSymbolIndex(self.chain_spec)
-        o = token_registry.address_of(self.token_index_address, token_symbol)
+        o = token_registry.address_of(self.token_index_address, 'SRF')
         r = self.conn.do(o)
         self.token_address = token_registry.parse_address_of(r)
-        logg.info('found {} token address {}'.format(token_symbol, self.token_address))
+        logg.info('found SRF token address {}'.format(self.token_address))
 
         tx_factory = ERC20(self.chain_spec)
         o = tx_factory.decimals(self.token_address)

apps/data-seeding/create_import_pins.py

@@ -3,7 +3,6 @@ import argparse
 import json
 import logging
 import os
-import uuid
 
 # third-party imports
 import bcrypt
@@ -84,7 +83,7 @@ if __name__ == '__main__':
 
                 phone_object = phonenumbers.parse(u.tel)
                 phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
-                password_hash = uuid.uuid4().hex
+                password_hash = generate_password_hash()
                 pins_file.write(f'{phone},{password_hash}\n')
                 logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
 

apps/data-seeding/verify.py

@@ -9,7 +9,6 @@ import sys
 import urllib
 import urllib.request
 import uuid
-import urllib.parse
 
 # external imports
 import celery
@@ -73,7 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
-argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@@ -186,9 +185,9 @@ def send_ussd_request(address, data_dir):
     }
 
     req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-    urlencoded_data = urllib.parse.urlencode(data)
+    data_str = json.dumps(data)
-    data_bytes = urlencoded_data.encode('utf-8')
+    data_bytes = data_str.encode('utf-8')
-    req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+    req.add_header('Content-Type', 'application/json')
     req.data = data_bytes
     response = urllib.request.urlopen(req)
     return response.read().decode('utf-8')
@@ -389,9 +388,10 @@ class Verifier:
 
     def verify_ussd_pins(self, address, balance):
         response_data = send_ussd_request(address, self.data_dir)
-        if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
+        if response_data[:11] != 'CON Balance':
             raise VerifierError(response_data, 'pins')
 
+
     def verify(self, address, balance, debug_stem=None):
   
         for k in active_tests: