Downgrade chainsyncer

add keystore

See merge request grassrootseconomics/cic-internal-integration!197

apps/cic-eth/requirements.txt

@@ -16,7 +16,7 @@ moolb~=0.1.1b2
 eth-address-index~=0.1.1a11
 chainlib~=0.0.3rc2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a5
+chainsyncer[sql]==0.0.2a5
 chainqueue~=0.0.2b3
 sarafu-faucet~=0.0.3a3
 erc20-faucet~=0.2.1a5

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -87,10 +87,18 @@ for key in config.store.keys():
         module = importlib.import_module(config.store[key])
         if key == 'TASKS_AFRICASTALKING':
             africastalking_notifier = module.AfricasTalkingNotifier
+
+            api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID')
+            logg.debug(f'SENDER ID VALUE IS: {api_sender_id}')
+
+            if not api_sender_id:
+                api_sender_id = None
+                logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}')
+
             africastalking_notifier.initialize(
                 config.get('AFRICASTALKING_API_USERNAME'),
                 config.get('AFRICASTALKING_API_KEY'),
-                config.get('AFRICASTALKING_API_SENDER_ID')
+                api_sender_id
             )
 
 

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -275,6 +275,18 @@
             "display_key": "ussd.kenya.new_pin_confirmation",
             "name": "new_pin_confirmation",
             "parent": "metadata_management"
+        },
+        "47": {
+            "description": "Year of birth entry menu.",
+            "display_key": "ussd.kenya.enter_date_of_birth",
+            "name": "enter_date_of_birth",
+            "parent": "metadata_management"
+        },
+        "48": {
+            "description": "Pin entry menu for changing year of birth data.",
+            "display_key": "ussd.kenya.dob_edit_pin_authorization",
+            "name": "dob_edit_pin_authorization",
+            "parent": "metadata_management"
         }
 
     }

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class CustomMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.custom', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PreferencesMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.preferences', identifier=identifier)

apps/cic-ussd/cic_ussd/phone_number.py

@@ -8,6 +8,10 @@ import phonenumbers
 from cic_ussd.db.models.account import Account
 
 
+class E164Format:
+    region = None
+
+
 def process_phone_number(phone_number: str, region: str):
     """This function parses any phone number for the provided region
     :param phone_number: A string with a phone number.
@@ -38,7 +42,7 @@ def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
     :rtype: Account|None
     """
     # consider adding region to user's metadata
-    phone_number = process_phone_number(phone_number=phone_number, region='KE')
+    phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region)
     user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
     return user
 

apps/cic-ussd/cic_ussd/processor.py

@@ -1,4 +1,5 @@
 # standard imports
+import datetime
 import logging
 import json
 import re
@@ -257,6 +258,10 @@ def process_display_user_metadata(user: Account, display_key: str):
         contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard'))
         logg.debug(f'{contact_data}')
         full_name = f'{contact_data.get("given")} {contact_data.get("family")}'
+        date_of_birth = user_metadata.get('date_of_birth')
+        year_of_birth = date_of_birth.get('year')
+        present_year = datetime.datetime.now().year
+        age = present_year - year_of_birth
         gender = user_metadata.get('gender')
         products = ', '.join(user_metadata.get('products'))
         location = user_metadata.get('location').get('area_name')
@@ -265,6 +270,7 @@ def process_display_user_metadata(user: Account, display_key: str):
             key=display_key,
             preferred_language=user.preferred_language,
             full_name=full_name,
+            age=age,
             gender=gender,
             location=location,
             products=products
@@ -289,7 +295,6 @@ def process_display_user_metadata(user: Account, display_key: str):
         )
 
 
-
 def process_account_statement(user: Account, display_key: str, ussd_session: dict):
     """
     :param user:

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -26,7 +26,7 @@ from cic_ussd.metadata.base import Metadata
 from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
-from cic_ussd.phone_number import process_phone_number, Support
+from cic_ussd.phone_number import process_phone_number, Support, E164Format
 from cic_ussd.processor import get_default_token_data
 from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
@@ -126,6 +126,7 @@ else:
 
 valid_service_codes = config.get('APP_SERVICE_CODE').split(",")
 
+E164Format.region = config.get('PHONE_NUMBER_REGION')
 Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
 
 
@@ -168,7 +169,7 @@ def application(env, start_response):
 
         # add validation for phone number
         if phone_number:
-            phone_number = process_phone_number(phone_number=phone_number, region=config.get('PHONE_NUMBER_REGION'))
+            phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region)
 
         # validate ip address
         if not check_ip(config=config, env=env):

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -66,6 +66,18 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     return user_input == '5'
 
 
+def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
+    """
+    This function checks that user input matches a string with value '6'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '6'
+    :rtype: bool
+    """
+    user_input, ussd_session, user = state_machine_data
+    return user_input == '6'
+
+
 def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '00'

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -11,7 +11,7 @@ from cic_ussd.balance import BalanceManager, compute_operational_balance
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
-from cic_ussd.phone_number import get_user_by_phone_number, process_phone_number
+from cic_ussd.phone_number import get_user_by_phone_number, process_phone_number, E164Format
 from cic_ussd.processor import retrieve_token_symbol
 from cic_ussd.redis import create_cached_data_key, get_cached_data
 from cic_ussd.transactions import OutgoingTransactionProcessor
@@ -29,8 +29,9 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :rtype: bool
     """
     user_input, ussd_session, user = state_machine_data
+    phone_number = process_phone_number(user_input, E164Format.region)
     recipient = get_user_by_phone_number(phone_number=user_input)
-    is_not_initiator = process_phone_number(user_input, 'KE') != user.phone_number
+    is_not_initiator = phone_number != user.phone_number
     has_active_account_status = user.get_account_status() == AccountStatus.ACTIVE.name
     return is_not_initiator and has_active_account_status and recipient is not None
 

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -29,6 +29,16 @@ def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account
     Account.session.add(user)
     Account.session.commit()
 
+    preferences_data = {
+        'preferred_language': 'en'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [user.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
+
 
 def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account]):
     """This function changes the user's preferred language to swahili.
@@ -40,6 +50,16 @@ def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account
     Account.session.add(user)
     Account.session.commit()
 
+    preferences_data = {
+        'preferred_language': 'sw'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [user.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
+
 
 def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account]):
     """This function sets user's account to active.
@@ -93,6 +113,9 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict,
     if 'given_name' in current_state:
         key = 'given_name'
 
+    if 'date_of_birth' in current_state:
+        key = 'date_of_birth'
+
     if 'family_name' in current_state:
         key = 'family_name'
 
@@ -130,6 +153,13 @@ def format_user_metadata(metadata: dict, user: Account):
     given_name = metadata.get('given_name')
     family_name = metadata.get('family_name')
 
+    if isinstance(metadata.get('date_of_birth'), dict):
+        date_of_birth = metadata.get('date_of_birth')
+    else:
+        date_of_birth = {
+            "year": int(metadata.get('date_of_birth')[:4])
+        }
+
     # check whether there's existing location data
     if isinstance(metadata.get('location'), dict):
         location = metadata.get('location')
@@ -154,6 +184,7 @@ def format_user_metadata(metadata: dict, user: Account):
     )
     return {
         "date_registered": date_registered,
+        "date_of_birth": date_of_birth,
         "gender": gender,
         "identities": identities,
         "location": location,
@@ -201,12 +232,12 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
 
     given_name = ussd_session.get('session_data').get('given_name')
     family_name = ussd_session.get('session_data').get('family_name')
+    date_of_birth = ussd_session.get('session_data').get('date_of_birth')
     gender = ussd_session.get('session_data').get('gender')
     location = ussd_session.get('session_data').get('location')
     products = ussd_session.get('session_data').get('products')
 
     # validate user metadata
-    person = Person()
     user_metadata = json.loads(user_metadata)
 
     # edit specific metadata attribute
@@ -214,6 +245,11 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
         user_metadata['given_name'] = given_name
     if family_name:
         user_metadata['family_name'] = family_name
+    if date_of_birth and len(date_of_birth) == 4:
+        year = int(date_of_birth[:4])
+        user_metadata['date_of_birth'] = {
+            'year': year
+        }
     if gender:
         user_metadata['gender'] = gender
     if location:

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -56,3 +56,15 @@ def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]):
         return True
     else:
         return False
+
+
+def is_valid_date(state_machine_data: Tuple[str, dict, Account]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, user = state_machine_data
+    # For MVP this value is defaulting to year
+    return len(user_input) == 4 and int(user_input) >= 1900

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -53,13 +53,25 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
             session.add(user)
             session.commit()
             session.close()
-    
+
             queue = self.request.delivery_info.get('routing_key')
-            s = celery.signature(
+
+            # add phone number metadata lookup
+            s_phone_pointer = celery.signature(
                     'cic_ussd.tasks.metadata.add_phone_pointer',
                     [result, phone_number]
                     )
-            s.apply_async(queue=queue)
+            s_phone_pointer.apply_async(queue=queue)
+
+            # add custom metadata tags
+            custom_metadata = {
+                "tags": ["ussd", "individual"]
+            }
+            s_custom_metadata = celery.signature(
+                'cic_ussd.tasks.metadata.add_custom_metadata',
+                [result, custom_metadata]
+            )
+            s_custom_metadata.apply_async(queue=queue)
 
             # expire cache
             cache.expire(task_id, timedelta(seconds=180))

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -7,8 +7,10 @@ from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
+from cic_ussd.metadata.custom import CustomMetadata
 from cic_ussd.metadata.person import PersonMetadata
 from cic_ussd.metadata.phone import PhonePointerMetadata
+from cic_ussd.metadata.preferences import PreferencesMetadata
 from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
@@ -44,7 +46,7 @@ def create_person_metadata(blockchain_address: str, data: dict):
 
 
 @celery_app.task
-def edit_person_metadata(blockchain_address: str, data: bytes):
+def edit_person_metadata(blockchain_address: str, data: dict):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.edit(data=data)
@@ -56,3 +58,17 @@ def add_phone_pointer(self, blockchain_address: str, phone_number: str):
     stripped_address = strip_0x(blockchain_address)
     phone_metadata_client = PhonePointerMetadata(identifier=identifier)
     phone_metadata_client.create(data=stripped_address)
+
+
+@celery_app.task()
+def add_custom_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = CustomMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)
+
+
+@celery_app.task()
+def add_preferences_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = PreferencesMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b17
-cic-eth~=0.11.0b17
+cic_base[full_graph]~=0.1.2b21
+cic-eth~=0.11.0b16
 cic-notify~=0.4.0a5
-cic-types~=0.1.0a10
+cic-types~=0.1.0a11

apps/cic-ussd/states/account_management_states.json

@@ -7,6 +7,7 @@
   "new_pin_confirmation",
   "display_user_metadata",
   "name_edit_pin_authorization",
+  "dob_edit_pin_authorization",
   "gender_edit_pin_authorization",
   "location_edit_pin_authorization",
   "products_edit_pin_authorization",

apps/cic-ussd/states/user_metadata_states.json

@@ -5,5 +5,6 @@
   "enter_age",
   "enter_location",
   "enter_products",
+  "enter_date_of_birth",
   "display_metadata_pin_authorization"
 ]

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -0,0 +1,39 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "dob_edit_pin_authorization",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "conditions": [
+      "cic_ussd.state_machine.logic.validator.has_cached_user_metadata",
+      "cic_ussd.state_machine.logic.validator.is_valid_date"
+    ]
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "enter_gender",
+    "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "exit_invalid_input",
+    "unless": "cic_ussd.state_machine.logic.validator.is_valid_date"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin",
+    "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  }
+]

apps/cic-ussd/transitions/name_setting_transitions.json

@@ -15,7 +15,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_family_name",
-    "dest": "enter_gender",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
   },

apps/cic-ussd/transitions/user_metadata_transitions.json

@@ -14,21 +14,27 @@
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_products",
+    "dest": "enter_location",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_four_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "display_metadata_pin_authorization",
+    "dest": "enter_products",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected"
   },
+  {
+    "trigger": "scan_data",
+    "source": "metadata_management",
+    "dest": "display_metadata_pin_authorization",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_six_selected"
+  },
   {
     "trigger": "scan_data",
     "source": "display_metadata_pin_authorization",

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -17,6 +17,9 @@ en:
     enter_family_name: |-
       CON Enter family name
       0. Back
+    enter_date_of_birth: |-
+      CON Enter year of birth
+      0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
@@ -52,14 +55,16 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit location
-      4. Edit products
-      5. View my profile
+      3. Edit Age
+      4. Edit location
+      5. Edit products
+      6. View my profile
       0. Back
     display_user_metadata: |-
       CON Your details are:
         Name: %{full_name}
         Gender: %{gender}
+        Age: %{age}
         Location: %{location}
         You sell: %{products}
       0. Back
@@ -117,6 +122,13 @@ en:
       retry: |-
         CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
+    dob_edit_pin_authorization:
+      first: |-
+        CON Please enter your PIN
+        0. Back
+      retry: |-
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
+        0. Back
     gender_edit_pin_authorization:
       first: |-
         CON Please enter your PIN

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -17,6 +17,9 @@ sw:
     enter_family_name: |-
       CON Weka jina lako la mwisho
       0. Nyuma
+    enter_date_of_birth: |-
+      CON Weka mwaka wa kuzaliwa
+      0. Nyuma
     enter_gender: |-
       CON Weka jinsia yako
       1. Mwanaume
@@ -52,14 +55,16 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3. Weka eneo
-      4. Weka bidhaa
-      5. Angalia wasifu wako
+      3  Weka umri
+      4. Weka eneo
+      5. Weka bidhaa
+      6. Angalia wasifu wako
       0. Nyuma
     display_user_metadata: |-
       CON Wasifu wako una maelezo yafuatayo:
         Jina: %{full_name}
         Jinsia: %{gender}
+        Umri: %{age}
         Eneo: %{location}
         Unauza: %{products}
       0. Nyuma
@@ -117,6 +122,13 @@ sw:
       retry: |-
         CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
         0. Nyuma
+    dob_edit_pin_authorization:
+      first: |-
+        CON Tafadhali weka PIN yako
+        0. Nyuma
+      retry: |-
+        CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
+        0. Nyuma
     gender_edit_pin_authorization:
       first: |-
         CON Tafadhali weka PIN yako

apps/data-seeding/cic_eth/import_balance.py

@@ -76,6 +76,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')

apps/data-seeding/cic_eth/import_users.py

@@ -194,8 +194,7 @@ if __name__ == '__main__':
             f.write(json.dumps(o))
             f.close()
 
-            #fi.write('{},{}\n'.format(new_address, old_address))
-            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
             meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
             os.symlink(os.path.realpath(filepath), meta_filepath)
 
@@ -221,7 +220,7 @@ if __name__ == '__main__':
 
 
             # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom')
             custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
 
             filepath = os.path.join(

apps/data-seeding/cic_meta/import_meta_preferences.js

@@ -0,0 +1,133 @@
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+
+const cic = require('@cicnet/cic-client-meta');
+const crdt = require('@cicnet/crdt-meta');
+
+//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
+
+const config = new crdt.Config('./config');
+config.process();
+console.log(config);
+
+
+function sendit(uid, envelope) {
+	const d = envelope.toJSON();
+
+	const contentLength = (new TextEncoder().encode(d)).length;
+	const opts = {
+		method: 'PUT',
+		headers: {
+			'Content-Type': 'application/json',
+			'Content-Length': contentLength,
+			'X-CIC-AUTOMERGE': 'client',
+
+		},
+	};
+	let url = config.get('META_URL');
+	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
+	console.log('posting to url: ' + url + uid);
+	const req = http.request(url + uid, opts, (res) => {
+		res.on('data', process.stdout.write);
+		res.on('end', () => {
+			console.log('result', res.statusCode, res.headers);
+		});
+	});
+	if (!req.write(d)) {
+		console.error('foo', d);
+		process.exit(1);
+	}
+	req.end();
+}
+
+function doOne(keystore, filePath, identifier) {
+	const signer = new crdt.PGPSigner(keystore);
+
+	const o = JSON.parse(fs.readFileSync(filePath).toString());
+
+	cic.Custom.toKey(identifier).then((uid) => {
+		const s = new crdt.Syncable(uid, o);
+		s.setSigner(signer);
+		s.onwrap = (env) => {
+			sendit(identifier, env);
+		};
+		s.sign();
+	});
+}
+
+const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+pk = fs.readFileSync(privateKeyPath);
+pubk = fs.readFileSync(publicKeyPath);
+
+new crdt.PGPKeyStore(
+	config.get('PGP_PASSPHRASE'),
+	pk,
+	pubk,
+	undefined,
+	undefined,
+	importMetaCustom,
+);
+
+const batchSize = 16;
+const batchDelay = 1000;
+const total = parseInt(process.argv[3]);
+const dataDir = process.argv[2];
+const workDir = path.join(dataDir, 'preferences/meta');
+const userDir = path.join(dataDir, 'preferences/new');
+let count = 0;
+let batchCount = 0;
+
+
+function importMetaCustom(keystore) {
+	let err;
+	let files;
+
+	try {
+		err, files = fs.readdirSync(workDir);
+	} catch {
+		console.error('source directory not yet ready', workDir);
+		setTimeout(importMetaCustom, batchDelay, keystore);
+		return;
+	}
+	let limit = batchSize;
+	if (files.length < limit) {
+		limit = files.length;
+	}
+	for (let i = 0; i < limit; i++) {
+		const file = files[i];
+		if (file.length < 3) {
+			console.debug('skipping file', file);
+			continue;
+		}
+		//const identifier = file.substr(0,file.length-5);
+		const identifier = file;
+		const filePath = path.join(workDir, file);
+		console.log(filePath);
+
+		//const address = fs.readFileSync(filePath).toString().substring(2).toUpperCase();
+		const custom = JSON.parse(fs.readFileSync(filePath).toString());
+		const customFilePath = path.join(
+			userDir,
+			identifier.substring(0, 2),
+			identifier.substring(2, 4),
+			identifier + '.json',
+		);
+
+		doOne(keystore, filePath, identifier);
+		fs.unlinkSync(filePath);
+		count++;
+		batchCount++;
+		if (batchCount == batchSize) {
+			console.debug('reached batch size, breathing');
+			batchCount=0;
+			setTimeout(importMetaCustom, batchDelay, keystore);
+			return;
+		}
+	}
+	if (count == total) {
+		return;
+	}
+	setTimeout(importMetaCustom, 100, keystore);
+}

apps/data-seeding/cic_ussd/import_balance.py

@@ -70,6 +70,7 @@ args_override = {
         'REDIS_DB': getattr(args, 'redis_db'),
         'META_HOST': getattr(args, 'meta_host'),
         'META_PORT': getattr(args, 'meta_port'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')

apps/data-seeding/cic_ussd/import_task.py

@@ -1,6 +1,7 @@
 # standard imports
 import os
 import logging
+import random
 import urllib.parse
 import urllib.error
 import urllib.request
@@ -136,6 +137,42 @@ def generate_metadata(self, address, phone):
             )
     os.symlink(os.path.realpath(filepath), meta_filepath)
 
+    # write ussd data
+    ussd_data = {
+        'phone': phone,
+        'is_activated': 1,
+        'preferred_language': random.sample(['en', 'sw'], 1)[0],
+        'is_disabled': False
+    }
+    ussd_data_dir = os.path.join(self.import_dir, 'ussd')
+    ussd_data_file_path = os.path.join(ussd_data_dir, f'{old_address}.json')
+    f = open(ussd_data_file_path, 'w')
+    f.write(json.dumps(ussd_data))
+    f.close()
+
+    # write preferences data
+    preferences_dir = os.path.join(self.import_dir, 'preferences')
+    preferences_data = {
+        'preferred_language': ussd_data['preferred_language']
+    }
+
+    preferences_key = generate_metadata_pointer(bytes.fromhex(new_address_clean[2:]), ':cic.preferences')
+    preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
+
+    filepath = os.path.join(
+        preferences_dir,
+        'new',
+        preferences_key[:2].upper(),
+        preferences_key[2:4].upper(),
+        preferences_key.upper() + '.json'
+    )
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
+    f = open(filepath, 'w')
+    f.write(json.dumps(preferences_data))
+    f.close()
+    os.symlink(os.path.realpath(filepath), preferences_filepath)
+
     logg.debug('found metadata {} for phone {}'.format(o, phone))
 
     return address

apps/data-seeding/cic_ussd/import_users.py

@@ -64,6 +64,12 @@ ps = r.pubsub()
 user_new_dir = os.path.join(args.user_dir, 'new')
 os.makedirs(user_new_dir)
 
+ussd_data_dir = os.path.join(args.user_dir, 'ussd')
+os.makedirs(ussd_data_dir)
+
+preferences_dir = os.path.join(args.user_dir, 'preferences')
+os.makedirs(os.path.join(preferences_dir, 'meta'))
+
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
@@ -111,7 +117,13 @@ def register_ussd(i, u):
     phone_object = phonenumbers.parse(u.tel)
     phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
     logg.debug('tel {} {}'.format(u.tel, phone))
-    req = build_ussd_request(phone, 'localhost', 63315, '*483*46#', '', '') 
+    req = build_ussd_request(
+        phone,
+        'localhost',
+        config.get('CIC_USER_USSD_SVC_SERVICE_PORT'),
+        config.get('APP_SERVICE_CODE'),
+        '',
+        '')
     response = urllib.request.urlopen(req)
     response_data = response.read().decode('utf-8')
     state = response_data[:3]
@@ -128,59 +140,57 @@ if __name__ == '__main__':
             if y[len(y)-5:] != '.json':
                 continue
             # handle json containing person object
-            filepath = None
-            if y[:15] != '_ussd_data.json':
-                filepath = os.path.join(x[0], y)
-                f = open(filepath, 'r')
-                try:
-                    o = json.load(f)
-                except json.decoder.JSONDecodeError as e:
-                    f.close()
-                    logg.error('load error for {}: {}'.format(y, e))
-                    continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
                 f.close()
-                u = Person.deserialize(o)
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person.deserialize(o)
 
-                new_address = register_ussd(i, u)
+            new_address = register_ussd(i, u)
 
-                phone_object = phonenumbers.parse(u.tel)
-                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+            phone_object = phonenumbers.parse(u.tel)
+            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-                s_phone = celery.signature(
-                        'import_task.resolve_phone',
-                        [
-                            phone,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_phone = celery.signature(
+                    'import_task.resolve_phone',
+                    [
+                        phone,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_meta = celery.signature(
-                        'import_task.generate_metadata',
-                        [
-                            phone,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_meta = celery.signature(
+                    'import_task.generate_metadata',
+                    [
+                        phone,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_balance = celery.signature(
-                        'import_task.opening_balance_tx',
-                        [
-                            phone,
-                            i,
-                            ],
-                        queue='cic-import-ussd',
-                        )
+            s_balance = celery.signature(
+                    'import_task.opening_balance_tx',
+                    [
+                        phone,
+                        i,
+                        ],
+                    queue='cic-import-ussd',
+                    )
 
-                s_meta.link(s_balance)
-                s_phone.link(s_meta)
-                # block time plus a bit of time for ussd processing
-                s_phone.apply_async(countdown=7)
+            s_meta.link(s_balance)
+            s_phone.link(s_meta)
+            # block time plus a bit of time for ussd processing
+            s_phone.apply_async(countdown=7)
 
-                i += 1
-                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+            i += 1
+            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
 
-                j += 1
-                if j == batch_size:
-                    time.sleep(batch_delay)
-                    j = 0
+            j += 1
+            if j == batch_size:
+                time.sleep(batch_delay)
+                j = 0
 

apps/data-seeding/config/app.ini

@@ -22,3 +22,6 @@ TRANSITIONS=/usr/src/cic-ussd/transitions/
 host =
 port =
 ssl =
+
+[keystore]
+file_path = keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c

apps/data-seeding/config/cic.ini

@@ -7,3 +7,4 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+user_ussd_svc_service_port=

apps/data-seeding/config/database.ini

@@ -1,5 +1,10 @@
 [database]
-name = sempo
-host = localhost
-port = 5432
-user = postgres
+NAME=sempo
+USER=postgres
+PASSWORD=
+HOST=localhost
+PORT=5432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0
+POOL_SIZE=1

apps/data-seeding/create_import_users.py

@@ -228,7 +228,6 @@ def prepareLocalFilePath(datadir, address):
 if __name__ == '__main__':
 
     base_dir = os.path.join(user_dir, 'old')
-    ussd_dir = os.path.join(user_dir, 'ussd')
     os.makedirs(base_dir, exist_ok=True)
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
@@ -248,23 +247,11 @@ if __name__ == '__main__':
 
         print(o)
 
-        ussd_data = {
-            'phone': phone,
-            'is_activated': 1,
-            'preferred_language': random.sample(['en', 'sw'], 1)[0],
-            'is_disabled': False
-        }
-
         d = prepareLocalFilePath(base_dir, uid)
         f = open('{}/{}'.format(d, uid + '.json'), 'w')
         json.dump(o.serialize(), f)
         f.close()
 
-        d = prepareLocalFilePath(ussd_dir, uid)
-        x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w')
-        json.dump(ussd_data, x)
-        x.close()
-
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/data-seeding/docker/Dockerfile

@@ -1,43 +1,19 @@
 # syntax = docker/dockerfile:1.2
 FROM python:3.8.6-slim-buster as compile-image
 
+WORKDIR /root
+
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
-WORKDIR /root
 RUN mkdir -vp /usr/local/etc/cic
 
 COPY data-seeding/requirements.txt . 
 
+COPY data-seeding/ . 
+
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
 
-# -------------- begin runtime container ---------------- 
-FROM python:3.8.6-slim-buster as runtime-image
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y jq bash iputils-ping socat telnet dnsutils
-
-COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
-COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
-COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-WORKDIR root/
-
-ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
-# RUN useradd -u 1001 --create-home grassroots
-# RUN adduser grassroots sudo && \
-#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-# WORKDIR /home/grassroots
-
-COPY data-seeding/ . 
-
-# we copied these from the root build container.
-# this is dumb though...I guess the compile image should have the same user
-# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
-
-# USER grassroots
 
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -75,6 +75,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')

apps/data-seeding/eth/import_users.py

@@ -59,6 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c

@@ -0,0 +1 @@
+{"address":"eb3907ecad74a0013c259d5874ae7f22dcbcc95c","crypto":{"cipher":"aes-128-ctr","ciphertext":"b0f70a8af4071faff2267374e2423cbc7a71012096fd2215866d8de7445cc215","cipherparams":{"iv":"9ac89383a7793226446dcb7e1b45cdf3"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"299f7b5df1d08a0a7b7f9c9eb44fe4798683b78da3513fcf9603fd913ab3336f"},"mac":"6f4ed36c11345a9a48353cd2f93f1f92958c96df15f3112a192bc994250e8d03"},"id":"61a9dd88-24a9-495c-9a51-152bd1bfaa5b","version":3}