Downgrade chainsyncer

add keystore

See merge request grassrootseconomics/cic-internal-integration!197

.gitignore

@@ -8,3 +8,8 @@ gmon.out
 *.egg-info
 dist/
 build/
+**/*sqlite
+**/.nyc_output
+**/coverage
+**/.venv
+.idea

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -194,6 +194,7 @@ def main():
     except UnknownContractError as e:
         logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
         sys.exit(1)
+    logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:

apps/cic-eth/requirements.txt

@@ -16,10 +16,10 @@ moolb~=0.1.1b2
 eth-address-index~=0.1.1a11
 chainlib~=0.0.3rc2
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a5
+chainsyncer[sql]==0.0.2a5
 chainqueue~=0.0.2b3
-sarafu-faucet==0.0.3a3
+sarafu-faucet~=0.0.3a3
-erc20-faucet==0.2.1a4
+erc20-faucet~=0.2.1a5
 coincurve==15.0.0
 potaahto~=0.0.1a2
 pycryptodome==3.10.1

apps/cic-meta/.gitignore

@@ -3,3 +3,5 @@ dist
 dist-web
 dist-server
 scratch
+coverage
+.nyc_output

apps/cic-meta/.gitlab-ci.yml

@@ -3,17 +3,38 @@
     variables:
         APP_NAME: cic-meta
         DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+        IMAGE_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:unittest-$CI_COMMIT_SHORT_SHA
 
 .cic_meta_changes_target:
     rules:
-        - changes:
+        - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-            - $CONTEXT/$APP_NAME/*
+#        - changes:
+#            - $CONTEXT/$APP_NAME/*
+        - when: always
 
-build-mr-cic-meta:
+cic-meta-build-mr:
+  stage: build
+  extends:
+    - .cic_meta_variables
+    - .cic_meta_changes_target
+  script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
+    # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
+    - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG 
+
+test-mr-cic-meta:
     extends:
-        - .cic_meta_changes_target
-        - .py_build_merge_request
         - .cic_meta_variables
+        - .cic_meta_changes_target
+    stage: test
+    image: $IMAGE_TAG 
+    script:
+        - cd /tmp/src/cic-meta
+        - npm install --dev 
+        - npm run test
+        - npm run test:coverage
+    needs: ["cic-meta-build-mr"]
 
 build-push-cic-meta:
     extends:

apps/cic-meta/docker/Dockerfile

@@ -4,29 +4,28 @@ WORKDIR /tmp/src/cic-meta
 
 RUN apk add --no-cache postgresql bash
 
-COPY cic-meta/package.json \
+# required to build the cic-client-meta module
-     ./
-
 COPY cic-meta/src/ src/
-COPY cic-meta/tests/ tests/
 COPY cic-meta/scripts/ scripts/
 
+# copy the dependencies
+COPY cic-meta/package.json .
+COPY cic-meta/tsconfig.json . 
+COPY cic-meta/webpack.config.js .
+     
 RUN npm install
 
-# see exports_dir gpg.ini
+COPY cic-meta/tests/ tests/
 COPY cic-meta/tests/*.asc /root/pgp/
-RUN alias tsc=node_modules/typescript/bin/tsc 
-
 
+# copy runtime configs
 COPY cic-meta/.config/ /usr/local/etc/cic-meta/
-# COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql
 
+# db migrations
 COPY cic-meta/docker/db.sh ./db.sh
 RUN chmod 755 ./db.sh
 
-#RUN alias ts-node=/tmp/src/cic-meta/node_modules/ts-node/dist/bin.js
+RUN alias tsc=node_modules/typescript/bin/tsc 
-#ENTRYPOINT [ "./node_modules/ts-node/dist/bin.js", "./scripts/server/server.ts" ]
-
 COPY cic-meta/docker/start_server.sh ./start_server.sh
 RUN chmod 755 ./start_server.sh
 ENTRYPOINT ["sh", "./start_server.sh"]

apps/cic-meta/package.json

@@ -11,6 +11,7 @@
 	"preferGlobal": true,
 	"scripts": {
 		"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
+		"test:coverage": "nyc mocha tests/*.ts --timeout 3000 --check-coverage=true",
 		"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
 		"build-server": "tsc -d --outDir dist-server scripts/server/*.ts",
 		"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
@@ -34,7 +35,9 @@
 	"devDependencies": {
 		"@types/mocha": "^8.0.3",
 		"mocha": "^8.2.0",
+		"nock": "^13.1.0",
 		"node-localstorage": "^2.1.6",
+		"nyc": "^15.1.0",
 		"ts-node": "^9.0.0",
 		"typescript": "^4.0.5",
 		"webpack": "^5.4.0",
@@ -50,5 +53,26 @@
 	"license": "GPL-3.0-or-later",
 	"engines": {
 		"node": ">=14.16.1"
+	},
+	"nyc": {
+		"include": [
+			"src/**/*.ts"
+		],
+		"extension": [
+			".ts"
+		],
+		"require": [
+			"ts-node/register"
+		],
+		"reporter": [
+			"text",
+			"html"
+		],
+		"sourceMap": true,
+		"instrument": true,
+		"branches": ">80",
+		"lines": ">80",
+		"functions": ">80",
+		"statements": ">80"
 	}
 }

apps/cic-meta/scripts/server/server.ts

@@ -204,7 +204,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty onctent', data);
+			console.error('empty content', data);
 			res.writeHead(400, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/cic-meta/src/custom.ts

@@ -9,7 +9,7 @@ class Custom extends Syncable implements Addressable {
         super('', v);
         Custom.toKey(name).then((cid) => {
             this.id = cid;
-            this.value = v;
+            this.name = name;
         });
     }
 

apps/cic-meta/src/meta.ts

@@ -100,13 +100,15 @@ class Meta {
             identifier = await User.toKey(name);
         } else if (type === 'phone') {
             identifier = await Phone.toKey(name);
-        } else {
+        } else if (type === 'custom') {
             identifier = await Custom.toKey(name);
+        } else {
+            identifier = await Custom.toKey(name, type);
         }
         return identifier;
     }
 
-    private wrap(syncable: Syncable): Promise<Envelope> {
+    wrap(syncable: Syncable): Promise<Envelope> {
         return new Promise<Envelope>(async (resolve, reject) => {
             syncable.setSigner(this.signer);
             syncable.onwrap = async (env) => {

apps/cic-meta/tests/custom.ts

@@ -0,0 +1,49 @@
+import * as assert from 'assert';
+import {Custom} from "../src";
+
+const testName = 'areas';
+const testObject = {
+    area: ['Nairobi', 'Mombasa', 'Kilifi']
+}
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+describe('custom', () => {
+
+    context('with predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName, testObject);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, testObject);
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    context('without predefined data', () => {
+        it('should create a custom object', () => {
+            const custom = new Custom(testName);
+            setTimeout(() => {
+                assert.strictEqual(custom.name, testName);
+                assert.deepStrictEqual(custom.m.data, {});
+                assert.strictEqual(custom.key(), testNameKey)
+            }, 0);
+        });
+    });
+
+    describe('#toKey()', () => {
+        context('without a custom identifier', () => {
+            it('should generate a key from the custom name', async () => {
+                assert.strictEqual(await Custom.toKey(testName), testNameKey);
+            });
+        });
+
+        context('with a custom identifier', () => {
+            it('should generate a key from the custom name with a custom identifier', async () => {
+                assert.strictEqual(await Custom.toKey(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/meta.ts

@@ -0,0 +1,176 @@
+import * as assert from 'assert';
+import * as fs from 'fs';
+const nock = require('nock');
+import {Meta} from "../src";
+import {getResponse, metaData, networkErrorResponse, notFoundResponse, putResponse} from "./response";
+import {Syncable} from "@cicnet/crdt-meta";
+
+const metaUrl = 'https://meta.dev.grassrootseconomics.net';
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+const testName = 'areas'
+const testNameKey = '8f3da0c90ba2b89ff217da96f6088cbaf987a1b58bc33c3a5e526e53cec7cfed';
+const testIdentifier = ':cic.area'
+const testIdentifierKey = 'da6194e6f33726546e82c328df4c120b844d6427859156518bd600765bf8b2b7';
+
+function readFile(filename) {
+    if(!fs.existsSync(filename)) {
+        console.error(`File ${filename} not found`);
+        return;
+    }
+    return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
+}
+
+const privateKey = readFile('./privatekeys.asc');
+
+describe('meta', () => {
+    beforeEach(() => {
+        nock(metaUrl)
+            .get(`/${testAddressKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testPhoneKey}`)
+            .reply(200, getResponse);
+
+        nock(metaUrl)
+            .get(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .get(`/${testIdentifier}`)
+            .replyWithError(networkErrorResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddressKey}`)
+            .reply(200, putResponse);
+
+        nock(metaUrl)
+            .put(`/${testAddress}`)
+            .reply(404);
+
+        nock(metaUrl)
+            .post('/post')
+            .reply(500);
+    });
+
+    describe('#get()', () => {
+        it('should fetch data from the meta service', async () => {
+            const account = await Meta.get(testAddressKey, metaUrl);
+            assert.strictEqual(account.toJSON(account), getResponse.payload);
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testAddress, metaUrl);
+                assert.strictEqual(account, `404: Not Found`);
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', async () => {
+                const account = await Meta.get(testIdentifier, metaUrl);
+                assert.strictEqual(account, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+            });
+        });
+    })
+
+    describe('#set()', () => {
+        context('object data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testAddressKey, metaData);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('string data', () => {
+            it('should set data to the meta server', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testPhoneKey, testAddress);
+                    assert.strictEqual(response, `${putResponse.status}: ${putResponse.statusText}`);
+                }
+            });
+        });
+
+        context('in case of network error', () => {
+            it('should respond with an error', () => {
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.set(testIdentifier, metaData);
+                    assert.strictEqual(response, `Request to ${metaUrl}/${testIdentifier} failed. Connection error.`);
+                }
+            });
+        });
+    });
+
+    describe('#updateMeta()', () => {
+        it('should update data in the meta server', async () => {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.updateMeta(syncable, testAddressKey);
+                assert.strictEqual(response, putResponse);
+            }
+        });
+
+        context('if item is not found', () => {
+            it('should respond with an error', () => {
+                const syncable = new Syncable(testAddress, metaData);
+                const meta = new Meta(metaUrl, privateKey);
+                meta.onload = async (status) => {
+                    const response = await meta.updateMeta(syncable, testAddress);
+                    assert.strictEqual(response, notFoundResponse);
+                }
+            });
+        });
+    });
+
+    describe('#wrap()', () => {
+        it('should sign a syncable object', function () {
+            const syncable = new Syncable(testAddressKey, metaData);
+            const meta = new Meta(metaUrl, privateKey);
+            meta.onload = async (status) => {
+                const response = await meta.wrap(syncable);
+                assert.strictEqual(response.toJSON(), getResponse);
+            }
+        });
+    })
+
+    describe('#getIdentifier()', () => {
+        context('without type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName), testNameKey);
+            });
+        });
+
+        context('with user type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testAddress, 'user'), testAddressKey);
+            });
+        });
+
+        context('with phone type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testPhone, 'phone'), testPhoneKey);
+            });
+        });
+
+        context('with custom type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, 'custom'), testNameKey);
+            });
+        });
+
+        context('with unrecognised type', () => {
+            it('should return an identifier', async () => {
+                assert.strictEqual(await Meta.getIdentifier(testName, testIdentifier), testIdentifierKey);
+            });
+        });
+    });
+});

apps/cic-meta/tests/phone.ts

@@ -0,0 +1,24 @@
+import * as assert from 'assert';
+import {Phone} from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testPhone = '+254123456789';
+const testPhoneKey = 'be3cc8212b7eb57c6217ddd42230bd8ccd2f01382bf8c1c77d3a683fa5a9bb16';
+
+describe('phone', () => {
+
+    it('should create a phone object', () => {
+        const phone = new Phone(testAddress, testPhone);
+        setTimeout(() => {
+            assert.strictEqual(phone.address, testAddress);
+            assert.strictEqual(phone.m.data.msisdn, testPhone);
+            assert.strictEqual(phone.key(), testPhoneKey)
+        }, 0);
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the phone number', async () => {
+            assert.strictEqual(await Phone.toKey(testPhone), testPhoneKey);
+        });
+    });
+});

apps/cic-meta/tests/user.ts

@@ -0,0 +1,54 @@
+import * as assert from 'assert';
+
+import { User } from "../src";
+
+const testAddress = '0xc1912fee45d61c87cc5ea59dae31190fffff232d';
+const testAddressKey = 'a51472cb4df63b199a4de01335b1b4d1bbee27ff4f03340aa1d592f26c6acfe2';
+const testUser = {
+    user: {
+        firstName: 'Test',
+        lastName: 'User'
+    }
+}
+
+describe('user', () => {
+
+    context('without predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, '');
+                assert.strictEqual(user.m.data.user.lastName, '');
+            }, 0);
+        });
+    });
+
+    context('with predefined data', () => {
+        it('should create a user object', () => {
+            const user = new User(testAddress, testUser);
+            setTimeout(() => {
+                assert.strictEqual(user.address, testAddress);
+                assert.strictEqual(user.key(), testAddressKey);
+                assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+                assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+            }, 0);
+        });
+    });
+
+    describe('#setName()', () => {
+        it('should set user\'s names to metadata', () => {
+            const user = new User(testAddress);
+            user.setName(testUser.user.firstName, testUser.user.lastName);
+            assert.strictEqual(user.m.data.user.firstName, testUser.user.firstName);
+            assert.strictEqual(user.m.data.user.lastName, testUser.user.lastName);
+        });
+    });
+
+    describe('#toKey()', () => {
+        it('should generate a key from the user\'s address', async () => {
+            assert.strictEqual(await User.toKey(testAddress), testAddressKey);
+        });
+    });
+});

apps/cic-meta/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "baseUrl": ".",
     "outDir": "./dist.browser",
-    "target": "es5",
+    "target": "es2015",
     "module": "commonjs",
     "moduleResolution": "node",
     "lib": ["es2016", "dom", "es5"],

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -87,10 +87,18 @@ for key in config.store.keys():
         module = importlib.import_module(config.store[key])
         if key == 'TASKS_AFRICASTALKING':
             africastalking_notifier = module.AfricasTalkingNotifier
+
+            api_sender_id = config.get('AFRICASTALKING_API_SENDER_ID')
+            logg.debug(f'SENDER ID VALUE IS: {api_sender_id}')
+
+            if not api_sender_id:
+                api_sender_id = None
+                logg.debug(f'SENDER ID RESOLVED TO NONE: {api_sender_id}')
+
             africastalking_notifier.initialize(
                 config.get('AFRICASTALKING_API_USERNAME'),
                 config.get('AFRICASTALKING_API_KEY'),
-                config.get('AFRICASTALKING_API_SENDER_ID')
+                api_sender_id
             )
 
 

apps/cic-ussd/.config/app.ini

@@ -5,6 +5,7 @@ LOCALE_PATH=/usr/src/cic-ussd/var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#,*483*061#,*384*96#
+SUPPORT_PHONE_NUMBER=0757628885
 
 [phone_number]
 REGION=KE

apps/cic-ussd/.config/test/app.ini

@@ -5,6 +5,7 @@ LOCALE_PATH=var/lib/locale/
 MAX_BODY_LENGTH=1024
 PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I=
 SERVICE_CODE=*483*46#
+SUPPORT_PHONE_NUMBER=0757628885
 
 [ussd]
 MENU_FILE=/usr/local/lib/python3.8/site-packages/cic_ussd/db/ussd_menu.json

apps/cic-ussd/cic_ussd/db/ussd_menu.json

@@ -275,6 +275,18 @@
             "display_key": "ussd.kenya.new_pin_confirmation",
             "name": "new_pin_confirmation",
             "parent": "metadata_management"
+        },
+        "47": {
+            "description": "Year of birth entry menu.",
+            "display_key": "ussd.kenya.enter_date_of_birth",
+            "name": "enter_date_of_birth",
+            "parent": "metadata_management"
+        },
+        "48": {
+            "description": "Pin entry menu for changing year of birth data.",
+            "display_key": "ussd.kenya.dob_edit_pin_authorization",
+            "name": "dob_edit_pin_authorization",
+            "parent": "metadata_management"
         }
 
     }

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class CustomMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.custom', identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -0,0 +1,12 @@
+# standard imports
+
+# external imports
+
+# local imports
+from .base import MetadataRequestsHandler
+
+
+class PreferencesMetadata(MetadataRequestsHandler):
+
+    def __init__(self, identifier: bytes):
+        super().__init__(cic_type=':cic.preferences', identifier=identifier)

apps/cic-ussd/cic_ussd/phone_number.py

@@ -8,6 +8,10 @@ import phonenumbers
 from cic_ussd.db.models.account import Account
 
 
+class E164Format:
+    region = None
+
+
 def process_phone_number(phone_number: str, region: str):
     """This function parses any phone number for the provided region
     :param phone_number: A string with a phone number.
@@ -38,6 +42,10 @@ def get_user_by_phone_number(phone_number: str) -> Optional[Account]:
     :rtype: Account|None
     """
     # consider adding region to user's metadata
-    phone_number = process_phone_number(phone_number=phone_number, region='KE')
+    phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region)
     user = Account.session.query(Account).filter_by(phone_number=phone_number).first()
     return user
+
+
+class Support:
+    phone_number = None

apps/cic-ussd/cic_ussd/processor.py

@@ -1,4 +1,5 @@
 # standard imports
+import datetime
 import logging
 import json
 import re
@@ -19,7 +20,7 @@ from cic_ussd.db.models.ussd_session import UssdSession
 from cic_ussd.error import MetadataNotFoundError, SeppukuError
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
-from cic_ussd.phone_number import get_user_by_phone_number
+from cic_ussd.phone_number import get_user_by_phone_number, Support
 from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data
 from cic_ussd.state_machine import UssdStateMachine
 from cic_ussd.conversions import to_wei, from_wei
@@ -257,6 +258,10 @@ def process_display_user_metadata(user: Account, display_key: str):
         contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard'))
         logg.debug(f'{contact_data}')
         full_name = f'{contact_data.get("given")} {contact_data.get("family")}'
+        date_of_birth = user_metadata.get('date_of_birth')
+        year_of_birth = date_of_birth.get('year')
+        present_year = datetime.datetime.now().year
+        age = present_year - year_of_birth
         gender = user_metadata.get('gender')
         products = ', '.join(user_metadata.get('products'))
         location = user_metadata.get('location').get('area_name')
@@ -265,12 +270,29 @@ def process_display_user_metadata(user: Account, display_key: str):
             key=display_key,
             preferred_language=user.preferred_language,
             full_name=full_name,
+            age=age,
             gender=gender,
             location=location,
             products=products
         )
     else:
-        raise MetadataNotFoundError(f'Expected person metadata but found none in cache for key: {key}')
+        # TODO [Philip]: All these translations could be moved to translation files.
+        logg.warning(f'Expected person metadata but found none in cache for key: {key}')
+
+        absent = ''
+        if user.preferred_language == 'en':
+            absent = 'Not provided'
+        elif user.preferred_language == 'sw':
+            absent = 'Haijawekwa'
+
+        return translation_for(
+            key=display_key,
+            preferred_language=user.preferred_language,
+            full_name=absent,
+            gender=absent,
+            location=absent,
+            products=absent
+        )
 
 
 def process_account_statement(user: Account, display_key: str, ussd_session: dict):
@@ -406,12 +428,6 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
     :return: A ussd menu's corresponding text value.
     :rtype: Document
     """
-    # retrieve metadata before any transition
-    key = generate_metadata_pointer(
-        identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
-        cic_type=':cic.person'
-    )
-    person_metadata = get_cached_data(key=key)
 
     if ussd_session:
         if user_input == "0":
@@ -435,7 +451,7 @@ def process_request(user_input: str, user: Account, ussd_session: Optional[dict]
                     'exit_pin_mismatch',
                     'exit_invalid_request',
                     'exit_successful_transaction'
-                ] and person_metadata is not None:
+                ]:
                     return UssdMenu.find_by_name(name='start')
                 else:
                     return UssdMenu.find_by_name(name=last_state)
@@ -467,6 +483,14 @@ def next_state(ussd_session: dict, user: Account, user_input: str) -> str:
     return new_state
 
 
+def process_exit_invalid_menu_option(display_key: str, preferred_language: str):
+    return translation_for(
+        key=display_key,
+        preferred_language=preferred_language,
+        support_phone=Support.phone_number
+    )
+
+
 def custom_display_text(
         display_key: str,
         menu_name: str,
@@ -503,5 +527,7 @@ def custom_display_text(
         return process_account_statement(display_key=display_key, user=user, ussd_session=ussd_session)
     elif menu_name == 'display_user_metadata':
         return process_display_user_metadata(display_key=display_key, user=user)
+    elif menu_name == 'exit_invalid_menu_option':
+        return process_exit_invalid_menu_option(display_key=display_key, preferred_language=user.preferred_language)
     else:
         return translation_for(key=display_key, preferred_language=user.preferred_language)

apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py

@@ -26,7 +26,7 @@ from cic_ussd.metadata.base import Metadata
 from cic_ussd.operations import (define_response_with_content,
                                  process_menu_interaction_requests,
                                  define_multilingual_responses)
-from cic_ussd.phone_number import process_phone_number
+from cic_ussd.phone_number import process_phone_number, Support, E164Format
 from cic_ussd.processor import get_default_token_data
 from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore
 from cic_ussd.requests import (get_request_endpoint,
@@ -126,6 +126,9 @@ else:
 
 valid_service_codes = config.get('APP_SERVICE_CODE').split(",")
 
+E164Format.region = config.get('PHONE_NUMBER_REGION')
+Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
+
 
 def application(env, start_response):
     """Loads python code for application to be accessible over web server
@@ -143,7 +146,7 @@ def application(env, start_response):
     if get_request_method(env=env) == 'POST' and get_request_endpoint(env=env) == '/':
 
         if env.get('CONTENT_TYPE') != 'application/x-www-form-urlencoded':
-            start_response('405 Play by the rules', errors_headers)
+            start_response('405 Urlencoded, please', errors_headers)
             return []
 
         post_data = env.get('wsgi.input').read()
@@ -166,7 +169,7 @@ def application(env, start_response):
 
         # add validation for phone number
         if phone_number:
-            phone_number = process_phone_number(phone_number=phone_number, region=config.get('PHONE_NUMBER_REGION'))
+            phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region)
 
         # validate ip address
         if not check_ip(config=config, env=env):
@@ -211,6 +214,9 @@ def application(env, start_response):
         return [response_bytes]
 
     else:
+        logg.error('invalid query {}'.format(env))
+        for r in env:
+            logg.debug('{}: {}'.format(r, env))
         start_response('405 Play by the rules', errors_headers)
         return []
 

apps/cic-ussd/cic_ussd/state_machine/logic/menu.py

@@ -66,6 +66,18 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     return user_input == '5'
 
 
+def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
+    """
+    This function checks that user input matches a string with value '6'
+    :param state_machine_data: A tuple containing user input, a ussd session and user object.
+    :type state_machine_data: tuple
+    :return: A user input's match with '6'
+    :rtype: bool
+    """
+    user_input, ussd_session, user = state_machine_data
+    return user_input == '6'
+
+
 def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool:
     """
     This function checks that user input matches a string with value '00'

apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py

@@ -11,7 +11,7 @@ from cic_ussd.balance import BalanceManager, compute_operational_balance
 from cic_ussd.chain import Chain
 from cic_ussd.db.models.account import AccountStatus, Account
 from cic_ussd.operations import save_to_in_memory_ussd_session_data
-from cic_ussd.phone_number import get_user_by_phone_number
+from cic_ussd.phone_number import get_user_by_phone_number, process_phone_number, E164Format
 from cic_ussd.processor import retrieve_token_symbol
 from cic_ussd.redis import create_cached_data_key, get_cached_data
 from cic_ussd.transactions import OutgoingTransactionProcessor
@@ -29,8 +29,9 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, Account]) -> bool:
     :rtype: bool
     """
     user_input, ussd_session, user = state_machine_data
+    phone_number = process_phone_number(user_input, E164Format.region)
     recipient = get_user_by_phone_number(phone_number=user_input)
-    is_not_initiator = user_input != user.phone_number
+    is_not_initiator = phone_number != user.phone_number
     has_active_account_status = user.get_account_status() == AccountStatus.ACTIVE.name
     return is_not_initiator and has_active_account_status and recipient is not None
 

apps/cic-ussd/cic_ussd/state_machine/logic/user.py

@@ -29,6 +29,16 @@ def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account
     Account.session.add(user)
     Account.session.commit()
 
+    preferences_data = {
+        'preferred_language': 'en'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [user.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
+
 
 def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account]):
     """This function changes the user's preferred language to swahili.
@@ -40,6 +50,16 @@ def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account
     Account.session.add(user)
     Account.session.commit()
 
+    preferences_data = {
+        'preferred_language': 'sw'
+    }
+
+    s = celery.signature(
+        'cic_ussd.tasks.metadata.add_preferences_metadata',
+        [user.blockchain_address, preferences_data]
+    )
+    s.apply_async(queue='cic-ussd')
+
 
 def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account]):
     """This function sets user's account to active.
@@ -93,6 +113,9 @@ def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict,
     if 'given_name' in current_state:
         key = 'given_name'
 
+    if 'date_of_birth' in current_state:
+        key = 'date_of_birth'
+
     if 'family_name' in current_state:
         key = 'family_name'
 
@@ -130,6 +153,13 @@ def format_user_metadata(metadata: dict, user: Account):
     given_name = metadata.get('given_name')
     family_name = metadata.get('family_name')
 
+    if isinstance(metadata.get('date_of_birth'), dict):
+        date_of_birth = metadata.get('date_of_birth')
+    else:
+        date_of_birth = {
+            "year": int(metadata.get('date_of_birth')[:4])
+        }
+
     # check whether there's existing location data
     if isinstance(metadata.get('location'), dict):
         location = metadata.get('location')
@@ -154,6 +184,7 @@ def format_user_metadata(metadata: dict, user: Account):
     )
     return {
         "date_registered": date_registered,
+        "date_of_birth": date_of_birth,
         "gender": gender,
         "identities": identities,
         "location": location,
@@ -201,12 +232,12 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
 
     given_name = ussd_session.get('session_data').get('given_name')
     family_name = ussd_session.get('session_data').get('family_name')
+    date_of_birth = ussd_session.get('session_data').get('date_of_birth')
     gender = ussd_session.get('session_data').get('gender')
     location = ussd_session.get('session_data').get('location')
     products = ussd_session.get('session_data').get('products')
 
     # validate user metadata
-    person = Person()
     user_metadata = json.loads(user_metadata)
 
     # edit specific metadata attribute
@@ -214,6 +245,11 @@ def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account]):
         user_metadata['given_name'] = given_name
     if family_name:
         user_metadata['family_name'] = family_name
+    if date_of_birth and len(date_of_birth) == 4:
+        year = int(date_of_birth[:4])
+        user_metadata['date_of_birth'] = {
+            'year': year
+        }
     if gender:
         user_metadata['gender'] = gender
     if location:

apps/cic-ussd/cic_ussd/state_machine/logic/validator.py

@@ -56,3 +56,15 @@ def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]):
         return True
     else:
         return False
+
+
+def is_valid_date(state_machine_data: Tuple[str, dict, Account]):
+    """
+    :param state_machine_data:
+    :type state_machine_data:
+    :return:
+    :rtype:
+    """
+    user_input, ussd_session, user = state_machine_data
+    # For MVP this value is defaulting to year
+    return len(user_input) == 4 and int(user_input) >= 1900

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -53,13 +53,25 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
             session.add(user)
             session.commit()
             session.close()
-    
+
             queue = self.request.delivery_info.get('routing_key')
-            s = celery.signature(
+
+            # add phone number metadata lookup
+            s_phone_pointer = celery.signature(
                     'cic_ussd.tasks.metadata.add_phone_pointer',
                     [result, phone_number]
                     )
-            s.apply_async(queue=queue)
+            s_phone_pointer.apply_async(queue=queue)
+
+            # add custom metadata tags
+            custom_metadata = {
+                "tags": ["ussd", "individual"]
+            }
+            s_custom_metadata = celery.signature(
+                'cic_ussd.tasks.metadata.add_custom_metadata',
+                [result, custom_metadata]
+            )
+            s_custom_metadata.apply_async(queue=queue)
 
             # expire cache
             cache.expire(task_id, timedelta(seconds=180))

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -7,8 +7,10 @@ from hexathon import strip_0x
 
 # local imports
 from cic_ussd.metadata import blockchain_address_to_metadata_pointer
+from cic_ussd.metadata.custom import CustomMetadata
 from cic_ussd.metadata.person import PersonMetadata
 from cic_ussd.metadata.phone import PhonePointerMetadata
+from cic_ussd.metadata.preferences import PreferencesMetadata
 from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
@@ -44,7 +46,7 @@ def create_person_metadata(blockchain_address: str, data: dict):
 
 
 @celery_app.task
-def edit_person_metadata(blockchain_address: str, data: bytes):
+def edit_person_metadata(blockchain_address: str, data: dict):
     identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
     person_metadata_client.edit(data=data)
@@ -56,3 +58,17 @@ def add_phone_pointer(self, blockchain_address: str, phone_number: str):
     stripped_address = strip_0x(blockchain_address)
     phone_metadata_client = PhonePointerMetadata(identifier=identifier)
     phone_metadata_client.create(data=stripped_address)
+
+
+@celery_app.task()
+def add_custom_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = CustomMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)
+
+
+@celery_app.task()
+def add_preferences_metadata(blockchain_address: str, data: dict):
+    identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
+    custom_metadata_client = PreferencesMetadata(identifier=identifier)
+    custom_metadata_client.create(data=data)

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
-cic_base[full_graph]~=0.1.2b15
+cic_base[full_graph]~=0.1.2b21
 cic-eth~=0.11.0b16
 cic-notify~=0.4.0a5
-cic-types~=0.1.0a10
+cic-types~=0.1.0a11

apps/cic-ussd/states/account_management_states.json

@@ -7,6 +7,7 @@
   "new_pin_confirmation",
   "display_user_metadata",
   "name_edit_pin_authorization",
+  "dob_edit_pin_authorization",
   "gender_edit_pin_authorization",
   "location_edit_pin_authorization",
   "products_edit_pin_authorization",

apps/cic-ussd/states/user_metadata_states.json

@@ -5,5 +5,6 @@
   "enter_age",
   "enter_location",
   "enter_products",
+  "enter_date_of_birth",
   "display_metadata_pin_authorization"
 ]

apps/cic-ussd/transitions/age_setting_transitions.json

@@ -0,0 +1,39 @@
+[
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "dob_edit_pin_authorization",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "conditions": [
+      "cic_ussd.state_machine.logic.validator.has_cached_user_metadata",
+      "cic_ussd.state_machine.logic.validator.is_valid_date"
+    ]
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "enter_gender",
+    "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date",
+    "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
+    "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "enter_date_of_birth",
+    "dest": "exit_invalid_input",
+    "unless": "cic_ussd.state_machine.logic.validator.is_valid_date"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin",
+    "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute"
+  },
+  {
+    "trigger": "scan_data",
+    "source": "dob_edit_pin_authorization",
+    "dest": "exit_pin_blocked",
+    "conditions": "cic_ussd.state_machine.logic.pin.is_locked_account"
+  }
+]

apps/cic-ussd/transitions/name_setting_transitions.json

@@ -15,7 +15,7 @@
   {
     "trigger": "scan_data",
     "source": "enter_family_name",
-    "dest": "enter_gender",
+    "dest": "enter_date_of_birth",
     "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
   },

apps/cic-ussd/transitions/signup_transitions.json

@@ -36,26 +36,12 @@
     "source": "initial_pin_entry",
     "dest": "exit_invalid_pin"
   },
-  {
-    "trigger": "scan_data",
-    "source": "initial_pin_confirmation",
-    "dest": "start",
-    "conditions": [
-      "cic_ussd.state_machine.logic.pin.pins_match",
-      "cic_ussd.state_machine.logic.validator.has_cached_user_metadata"
-    ],
-    "after": [
-      "cic_ussd.state_machine.logic.pin.complete_pin_change",
-      "cic_ussd.state_machine.logic.user.get_user_metadata",
-      "cic_ussd.state_machine.logic.user.update_account_status_to_active"
-    ]
-  },
   {
     "trigger": "scan_data",
     "source": "initial_pin_confirmation",
     "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata",
     "conditions": "cic_ussd.state_machine.logic.pin.pins_match",
-    "dest": "enter_given_name",
+    "dest": "start",
     "after": [
       "cic_ussd.state_machine.logic.pin.complete_pin_change",
       "cic_ussd.state_machine.logic.user.update_account_status_to_active"

apps/cic-ussd/transitions/user_metadata_transitions.json

@@ -14,21 +14,27 @@
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_location",
+    "dest": "enter_date_of_birth",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_three_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "enter_products",
+    "dest": "enter_location",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_four_selected"
   },
   {
     "trigger": "scan_data",
     "source": "metadata_management",
-    "dest": "display_metadata_pin_authorization",
+    "dest": "enter_products",
     "conditions": "cic_ussd.state_machine.logic.menu.menu_five_selected"
   },
+  {
+    "trigger": "scan_data",
+    "source": "metadata_management",
+    "dest": "display_metadata_pin_authorization",
+    "conditions": "cic_ussd.state_machine.logic.menu.menu_six_selected"
+  },
   {
     "trigger": "scan_data",
     "source": "display_metadata_pin_authorization",

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -17,6 +17,9 @@ en:
     enter_family_name: |-
       CON Enter family name
       0. Back
+    enter_date_of_birth: |-
+      CON Enter year of birth
+      0. Back
     enter_gender: |-
       CON Enter gender
       1. Male
@@ -52,14 +55,16 @@ en:
       CON My profile
       1. Edit name
       2. Edit gender
-      3. Edit location
+      3. Edit Age
-      4. Edit products
+      4. Edit location
-      5. View my profile
+      5. Edit products
+      6. View my profile
       0. Back
     display_user_metadata: |-
       CON Your details are:
         Name: %{full_name}
         Gender: %{gender}
+        Age: %{age}
         Location: %{location}
         You sell: %{products}
       0. Back
@@ -117,6 +122,13 @@ en:
       retry: |-
         CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
         0. Back
+    dob_edit_pin_authorization:
+      first: |-
+        CON Please enter your PIN
+        0. Back
+      retry: |-
+        CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
+        0. Back
     gender_edit_pin_authorization:
       first: |-
         CON Please enter your PIN

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -17,6 +17,9 @@ sw:
     enter_family_name: |-
       CON Weka jina lako la mwisho
       0. Nyuma
+    enter_date_of_birth: |-
+      CON Weka mwaka wa kuzaliwa
+      0. Nyuma
     enter_gender: |-
       CON Weka jinsia yako
       1. Mwanaume
@@ -52,14 +55,16 @@ sw:
       CON Wasifu wangu
       1. Weka jina
       2. Weka jinsia
-      3. Weka eneo
+      3  Weka umri
-      4. Weka bidhaa
+      4. Weka eneo
-      5. Angalia wasifu wako
+      5. Weka bidhaa
+      6. Angalia wasifu wako
       0. Nyuma
     display_user_metadata: |-
       CON Wasifu wako una maelezo yafuatayo:
         Jina: %{full_name}
         Jinsia: %{gender}
+        Umri: %{age}
         Eneo: %{location}
         Unauza: %{products}
       0. Nyuma
@@ -117,6 +122,13 @@ sw:
       retry: |-
         CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
         0. Nyuma
+    dob_edit_pin_authorization:
+      first: |-
+        CON Tafadhali weka PIN yako
+        0. Nyuma
+      retry: |-
+        CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki.
+        0. Nyuma
     gender_edit_pin_authorization:
       first: |-
         CON Tafadhali weka PIN yako

apps/contract-migration/reset.sh

@@ -76,6 +76,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	>&2 echo "add deployer address as account index writer"
 	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
+	>&2 echo "deploy contract registry contract"
 	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
@@ -87,7 +88,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
-	>&2 echo "deploy address declarator contract"
+	>&2 echo "deploy transfer auth contract"
 	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
 	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 

apps/data-seeding/README.md

@@ -136,7 +136,7 @@ First, make a note of the **block height** before running anything:
 
 To import, run to _completion_:
 
-`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
+`python eth/import_users.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
 
 After the script completes, keystore files for all generated accouts will be found in `<datadir>/keystore`, all with `foo` as password (would set it empty, but believe it or not some interfaces out there won't work unless you have one).
 
@@ -150,7 +150,7 @@ Then run:
 
 Run in sequence, in first terminal:
 
-`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
+`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> --token-symbol <token_symbol> -y ../contract-migration/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c --head out`
 
 In another terminal:
 
@@ -226,7 +226,7 @@ The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in
 
 ### Step 5 - Verify
 
-`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> <datadir>` 
+`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>` 
 
 Included checks:
 * Private key is in cic-eth keystore
@@ -262,3 +262,5 @@ Should exit with code 0 if all input data is found in the respective services.
 - MacOS BigSur issue when installing psycopg2: ld: library not found for -lssl -> https://github.com/psycopg/psycopg2/issues/1115#issuecomment-831498953
 
 - `cic_ussd` imports is poorly implemented, and consumes a lot of resources. Therefore it takes a long time to complete. Reducing the amount of polls for the phone pointer would go a long way to improve it.
+
+- A strict constraint is maintained insistin the use of postgresql-12.

apps/data-seeding/cic_eth/import_balance.py

@@ -76,6 +76,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')

apps/data-seeding/cic_eth/import_users.py

@@ -194,8 +194,7 @@ if __name__ == '__main__':
             f.write(json.dumps(o))
             f.close()
 
-            #fi.write('{},{}\n'.format(new_address, old_address))
+            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
-            meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
             meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
             os.symlink(os.path.realpath(filepath), meta_filepath)
 
@@ -221,7 +220,7 @@ if __name__ == '__main__':
 
 
             # custom data
-            custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
+            custom_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.custom')
             custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
 
             filepath = os.path.join(

apps/data-seeding/cic_meta/import_meta_preferences.js

@@ -0,0 +1,133 @@
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+
+const cic = require('@cicnet/cic-client-meta');
+const crdt = require('@cicnet/crdt-meta');
+
+//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
+
+const config = new crdt.Config('./config');
+config.process();
+console.log(config);
+
+
+function sendit(uid, envelope) {
+	const d = envelope.toJSON();
+
+	const contentLength = (new TextEncoder().encode(d)).length;
+	const opts = {
+		method: 'PUT',
+		headers: {
+			'Content-Type': 'application/json',
+			'Content-Length': contentLength,
+			'X-CIC-AUTOMERGE': 'client',
+
+		},
+	};
+	let url = config.get('META_URL');
+	url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/');
+	console.log('posting to url: ' + url + uid);
+	const req = http.request(url + uid, opts, (res) => {
+		res.on('data', process.stdout.write);
+		res.on('end', () => {
+			console.log('result', res.statusCode, res.headers);
+		});
+	});
+	if (!req.write(d)) {
+		console.error('foo', d);
+		process.exit(1);
+	}
+	req.end();
+}
+
+function doOne(keystore, filePath, identifier) {
+	const signer = new crdt.PGPSigner(keystore);
+
+	const o = JSON.parse(fs.readFileSync(filePath).toString());
+
+	cic.Custom.toKey(identifier).then((uid) => {
+		const s = new crdt.Syncable(uid, o);
+		s.setSigner(signer);
+		s.onwrap = (env) => {
+			sendit(identifier, env);
+		};
+		s.sign();
+	});
+}
+
+const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
+pk = fs.readFileSync(privateKeyPath);
+pubk = fs.readFileSync(publicKeyPath);
+
+new crdt.PGPKeyStore(
+	config.get('PGP_PASSPHRASE'),
+	pk,
+	pubk,
+	undefined,
+	undefined,
+	importMetaCustom,
+);
+
+const batchSize = 16;
+const batchDelay = 1000;
+const total = parseInt(process.argv[3]);
+const dataDir = process.argv[2];
+const workDir = path.join(dataDir, 'preferences/meta');
+const userDir = path.join(dataDir, 'preferences/new');
+let count = 0;
+let batchCount = 0;
+
+
+function importMetaCustom(keystore) {
+	let err;
+	let files;
+
+	try {
+		err, files = fs.readdirSync(workDir);
+	} catch {
+		console.error('source directory not yet ready', workDir);
+		setTimeout(importMetaCustom, batchDelay, keystore);
+		return;
+	}
+	let limit = batchSize;
+	if (files.length < limit) {
+		limit = files.length;
+	}
+	for (let i = 0; i < limit; i++) {
+		const file = files[i];
+		if (file.length < 3) {
+			console.debug('skipping file', file);
+			continue;
+		}
+		//const identifier = file.substr(0,file.length-5);
+		const identifier = file;
+		const filePath = path.join(workDir, file);
+		console.log(filePath);
+
+		//const address = fs.readFileSync(filePath).toString().substring(2).toUpperCase();
+		const custom = JSON.parse(fs.readFileSync(filePath).toString());
+		const customFilePath = path.join(
+			userDir,
+			identifier.substring(0, 2),
+			identifier.substring(2, 4),
+			identifier + '.json',
+		);
+
+		doOne(keystore, filePath, identifier);
+		fs.unlinkSync(filePath);
+		count++;
+		batchCount++;
+		if (batchCount == batchSize) {
+			console.debug('reached batch size, breathing');
+			batchCount=0;
+			setTimeout(importMetaCustom, batchDelay, keystore);
+			return;
+		}
+	}
+	if (count == total) {
+		return;
+	}
+	setTimeout(importMetaCustom, 100, keystore);
+}

apps/data-seeding/cic_ussd/import_balance.py

@@ -70,6 +70,7 @@ args_override = {
         'REDIS_DB': getattr(args, 'redis_db'),
         'META_HOST': getattr(args, 'meta_host'),
         'META_PORT': getattr(args, 'meta_port'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')
@@ -114,7 +115,7 @@ def main():
     conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
    
     ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), signer_address, signer)
-    ImportTask.balance_processor.init()
+    ImportTask.balance_processor.init(token_symbol)
 
     # TODO get decimals from token
     balances = {}
@@ -139,6 +140,7 @@ def main():
 
     ImportTask.balances = balances
     ImportTask.count = i
+    ImportTask.import_dir = user_dir
 
     s = celery.signature(
             'import_task.send_txs',

apps/data-seeding/cic_ussd/import_pins.py

@@ -39,6 +39,7 @@ elif args.vv:
 config_dir = args.c
 config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
@@ -62,9 +63,6 @@ def main():
     )
     s_import_pins.apply_async()
 
-    argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
-    celery_app.worker_main(argv)
-
 
 if __name__ == '__main__':
     main()

apps/data-seeding/cic_ussd/import_task.py

@@ -1,6 +1,7 @@
 # standard imports
 import os
 import logging
+import random
 import urllib.parse
 import urllib.error
 import urllib.request
@@ -136,6 +137,42 @@ def generate_metadata(self, address, phone):
             )
     os.symlink(os.path.realpath(filepath), meta_filepath)
 
+    # write ussd data
+    ussd_data = {
+        'phone': phone,
+        'is_activated': 1,
+        'preferred_language': random.sample(['en', 'sw'], 1)[0],
+        'is_disabled': False
+    }
+    ussd_data_dir = os.path.join(self.import_dir, 'ussd')
+    ussd_data_file_path = os.path.join(ussd_data_dir, f'{old_address}.json')
+    f = open(ussd_data_file_path, 'w')
+    f.write(json.dumps(ussd_data))
+    f.close()
+
+    # write preferences data
+    preferences_dir = os.path.join(self.import_dir, 'preferences')
+    preferences_data = {
+        'preferred_language': ussd_data['preferred_language']
+    }
+
+    preferences_key = generate_metadata_pointer(bytes.fromhex(new_address_clean[2:]), ':cic.preferences')
+    preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
+
+    filepath = os.path.join(
+        preferences_dir,
+        'new',
+        preferences_key[:2].upper(),
+        preferences_key[2:4].upper(),
+        preferences_key.upper() + '.json'
+    )
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
+    f = open(filepath, 'w')
+    f.write(json.dumps(preferences_data))
+    f.close()
+    os.symlink(os.path.realpath(filepath), preferences_filepath)
+
     logg.debug('found metadata {} for phone {}'.format(o, phone))
 
     return address

apps/data-seeding/cic_ussd/import_users.py

@@ -1,29 +1,21 @@
 # standard imports
-import os
+import argparse
-import sys
 import json
 import logging
-import argparse
+import os
-import uuid
+import sys
-import datetime
 import time
 import urllib.request
-from glob import glob
+import uuid
+from urllib.parse import urlencode
 
-# third-party imports
+# external imports
-import redis
-import confini
 import celery
-from hexathon import (
+import confini
-        add_0x,
-        strip_0x,
-        )
-from chainlib.eth.address import to_checksum
-from cic_types.models.person import Person
-from cic_eth.api.api_task import Api
-from chainlib.chain import ChainSpec
-from cic_types.processor import generate_metadata_pointer
 import phonenumbers
+import redis
+from chainlib.chain import ChainSpec
+from cic_types.models.person import Person
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -72,6 +64,12 @@ ps = r.pubsub()
 user_new_dir = os.path.join(args.user_dir, 'new')
 os.makedirs(user_new_dir)
 
+ussd_data_dir = os.path.join(args.user_dir, 'ussd')
+os.makedirs(ussd_data_dir)
+
+preferences_dir = os.path.join(args.user_dir, 'preferences')
+os.makedirs(os.path.join(preferences_dir, 'meta'))
+
 meta_dir = os.path.join(args.user_dir, 'meta')
 os.makedirs(meta_dir)
 
@@ -87,21 +85,13 @@ chain_str = str(chain_spec)
 batch_size = args.batch_size
 batch_delay = args.batch_delay
 
-db_configs = {
-    'database': config.get('DATABASE_NAME'),
-    'host': config.get('DATABASE_HOST'),
-    'port': config.get('DATABASE_PORT'),
-    'user': config.get('DATABASE_USER'),
-    'password': config.get('DATABASE_PASSWORD')
-}
-  
 
 def build_ussd_request(phone, host, port, service_code, username, password, ssl=False):
     url = 'http'
     if ssl:
         url += 's'
     url += '://{}:{}'.format(host, port)
-    url += '/?username={}&password={}'.format(username, password) #config.get('USSD_USER'), config.get('USSD_PASS'))
+    url += '/?username={}&password={}'.format(username, password)
 
     logg.info('ussd service url {}'.format(url))
     logg.info('ussd phone {}'.format(phone))
@@ -114,9 +104,10 @@ def build_ussd_request(phone, host, port, service_code, username, password, ssl=
             'text': service_code,
         }
     req = urllib.request.Request(url)
-    data_str = json.dumps(data)
+    req.method=('POST')
+    data_str = urlencode(data)
     data_bytes = data_str.encode('utf-8')
-    req.add_header('Content-Type', 'application/json')
+    req.add_header('Content-Type', 'application/x-www-form-urlencoded')
     req.data = data_bytes
 
     return req
@@ -126,7 +117,13 @@ def register_ussd(i, u):
     phone_object = phonenumbers.parse(u.tel)
     phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
     logg.debug('tel {} {}'.format(u.tel, phone))
-    req = build_ussd_request(phone, 'localhost', 63315, '*483*46#', '', '') 
+    req = build_ussd_request(
+        phone,
+        'localhost',
+        config.get('CIC_USER_USSD_SVC_SERVICE_PORT'),
+        config.get('APP_SERVICE_CODE'),
+        '',
+        '')
     response = urllib.request.urlopen(req)
     response_data = response.read().decode('utf-8')
     state = response_data[:3]
@@ -143,59 +140,57 @@ if __name__ == '__main__':
             if y[len(y)-5:] != '.json':
                 continue
             # handle json containing person object
-            filepath = None
+            filepath = os.path.join(x[0], y)
-            if y[:15] != '_ussd_data.json':
+            f = open(filepath, 'r')
-                filepath = os.path.join(x[0], y)
+            try:
-                f = open(filepath, 'r')
+                o = json.load(f)
-                try:
+            except json.decoder.JSONDecodeError as e:
-                    o = json.load(f)
-                except json.decoder.JSONDecodeError as e:
-                    f.close()
-                    logg.error('load error for {}: {}'.format(y, e))
-                    continue
                 f.close()
-                u = Person.deserialize(o)
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person.deserialize(o)
 
-                new_address = register_ussd(i, u)
+            new_address = register_ussd(i, u)
 
-                phone_object = phonenumbers.parse(u.tel)
+            phone_object = phonenumbers.parse(u.tel)
-                phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
+            phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
 
-                s_phone = celery.signature(
+            s_phone = celery.signature(
-                        'import_task.resolve_phone',
+                    'import_task.resolve_phone',
-                        [
+                    [
-                            phone,
+                        phone,
-                            ],
+                        ],
-                        queue='cic-import-ussd',
+                    queue='cic-import-ussd',
-                        )
+                    )
 
-                s_meta = celery.signature(
+            s_meta = celery.signature(
-                        'import_task.generate_metadata',
+                    'import_task.generate_metadata',
-                        [
+                    [
-                            phone,
+                        phone,
-                            ],
+                        ],
-                        queue='cic-import-ussd',
+                    queue='cic-import-ussd',
-                        )
+                    )
 
-                s_balance = celery.signature(
+            s_balance = celery.signature(
-                        'import_task.opening_balance_tx',
+                    'import_task.opening_balance_tx',
-                        [
+                    [
-                            phone,
+                        phone,
-                            i,
+                        i,
-                            ],
+                        ],
-                        queue='cic-import-ussd',
+                    queue='cic-import-ussd',
-                        )
+                    )
 
-                s_meta.link(s_balance)
+            s_meta.link(s_balance)
-                s_phone.link(s_meta)
+            s_phone.link(s_meta)
-                # block time plus a bit of time for ussd processing
+            # block time plus a bit of time for ussd processing
-                s_phone.apply_async(countdown=7)
+            s_phone.apply_async(countdown=7)
 
-                i += 1
+            i += 1
-                sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
+            sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")
 
-                j += 1
+            j += 1
-                if j == batch_size:
+            if j == batch_size:
-                    time.sleep(batch_delay)
+                time.sleep(batch_delay)
-                    j = 0
+                j = 0
 

apps/data-seeding/cic_ussd/import_ussd_data.py

@@ -31,9 +31,9 @@ elif args.vv:
 config_dir = args.c
 config = Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
 config.process()
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
-user_old_dir = os.path.join(args.user_dir, 'old')
+ussd_data_dir = os.path.join(args.user_dir, 'ussd')
-os.stat(user_old_dir)
 
 db_configs = {
     'database': config.get('DATABASE_NAME'),
@@ -45,18 +45,15 @@ db_configs = {
 celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
 
 if __name__ == '__main__':
-    for x in os.walk(user_old_dir):
+    for x in os.walk(ussd_data_dir):
         for y in x[2]:
 
-            if y[len(y) - 5:] != '.json':
+            if y[len(y) - 5:] == '.json':
-                continue
-
-            # handle ussd_data json object
-            if y[:15] == '_ussd_data.json':
                 filepath = os.path.join(x[0], y)
                 f = open(filepath, 'r')
                 try:
                     ussd_data = json.load(f)
+                    logg.debug(f'LOADING USSD DATA: {ussd_data}')
                 except json.decoder.JSONDecodeError as e:
                     f.close()
                     logg.error('load error for {}: {}'.format(y, e))

apps/data-seeding/cic_ussd/import_util.py

@@ -6,7 +6,7 @@ from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import OverrideNonceOracle
-from chainlib.eth.erc20 import ERC20
+from eth_erc20 import ERC20
 from chainlib.eth.tx import (
         count,
         TxFormat,
@@ -37,7 +37,7 @@ class BalanceProcessor:
         self.value_multiplier = 1
     
 
-    def init(self):
+    def init(self, token_symbol):
         # Get Token registry address
         registry = Registry(self.chain_spec)
         o = registry.address_of(self.registry_address, 'TokenRegistry')
@@ -46,10 +46,10 @@ class BalanceProcessor:
         logg.info('found token index address {}'.format(self.token_index_address))
 
         token_registry = TokenUniqueSymbolIndex(self.chain_spec)
-        o = token_registry.address_of(self.token_index_address, 'SRF')
+        o = token_registry.address_of(self.token_index_address, token_symbol)
         r = self.conn.do(o)
         self.token_address = token_registry.parse_address_of(r)
-        logg.info('found SRF token address {}'.format(self.token_address))
+        logg.info('found {} token address {}'.format(token_symbol, self.token_address))
 
         tx_factory = ERC20(self.chain_spec)
         o = tx_factory.decimals(self.token_address)

apps/data-seeding/config/app.ini

@@ -22,3 +22,6 @@ TRANSITIONS=/usr/src/cic-ussd/transitions/
 host =
 port =
 ssl =
+
+[keystore]
+file_path = keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c

apps/data-seeding/config/cic.ini

@@ -7,3 +7,4 @@ approval_escrow_address =
 chain_spec = evm:bloxberg:8996
 tx_retry_delay = 
 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
+user_ussd_svc_service_port=

apps/data-seeding/config/database.ini

@@ -1,5 +1,10 @@
 [database]
-name = sempo
+NAME=sempo
-host = localhost
+USER=postgres
-port = 5432
+PASSWORD=
-user = postgres
+HOST=localhost
+PORT=5432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0
+POOL_SIZE=1

apps/data-seeding/create_import_pins.py

@@ -3,6 +3,7 @@ import argparse
 import json
 import logging
 import os
+import uuid
 
 # third-party imports
 import bcrypt
@@ -83,7 +84,7 @@ if __name__ == '__main__':
 
                 phone_object = phonenumbers.parse(u.tel)
                 phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
-                password_hash = generate_password_hash()
+                password_hash = uuid.uuid4().hex
                 pins_file.write(f'{phone},{password_hash}\n')
                 logg.info(f'Writing phone: {phone}, password_hash: {password_hash}')
 

apps/data-seeding/create_import_users.py

@@ -228,7 +228,6 @@ def prepareLocalFilePath(datadir, address):
 if __name__ == '__main__':
 
     base_dir = os.path.join(user_dir, 'old')
-    ussd_dir = os.path.join(user_dir, 'ussd')
     os.makedirs(base_dir, exist_ok=True)
 
     fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
@@ -248,23 +247,11 @@ if __name__ == '__main__':
 
         print(o)
 
-        ussd_data = {
-            'phone': phone,
-            'is_activated': 1,
-            'preferred_language': random.sample(['en', 'sw'], 1)[0],
-            'is_disabled': False
-        }
-
         d = prepareLocalFilePath(base_dir, uid)
         f = open('{}/{}'.format(d, uid + '.json'), 'w')
         json.dump(o.serialize(), f)
         f.close()
 
-        d = prepareLocalFilePath(ussd_dir, uid)
-        x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w')
-        json.dump(ussd_data, x)
-        x.close()
-
         pidx = genPhoneIndex(phone)
         d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), pidx)
         f = open('{}/{}'.format(d, pidx), 'w')

apps/data-seeding/docker/Dockerfile

@@ -1,43 +1,19 @@
 # syntax = docker/dockerfile:1.2
 FROM python:3.8.6-slim-buster as compile-image
 
+WORKDIR /root
+
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
 
-WORKDIR /root
 RUN mkdir -vp /usr/local/etc/cic
 
 COPY data-seeding/requirements.txt . 
 
+COPY data-seeding/ . 
+
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt 
 
-# -------------- begin runtime container ---------------- 
-FROM python:3.8.6-slim-buster as runtime-image
-
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends gnupg libpq-dev 
-RUN apt-get install -y jq bash iputils-ping socat telnet dnsutils
-
-COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
-COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
-COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \
-                          /usr/local/lib/python3.8/site-packages/
-
-WORKDIR root/
-
-ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433
-# RUN useradd -u 1001 --create-home grassroots
-# RUN adduser grassroots sudo && \
-#     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-# WORKDIR /home/grassroots
-
-COPY data-seeding/ . 
-
-# we copied these from the root build container.
-# this is dumb though...I guess the compile image should have the same user
-# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ 
-
-# USER grassroots
 
 ENTRYPOINT [ ]

apps/data-seeding/eth/import_balance.py

@@ -75,6 +75,7 @@ args_override = {
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
         'ETH_PROVIDER': getattr(args, 'p'),
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli flag')
 config.censor('PASSWORD', 'DATABASE')

apps/data-seeding/eth/import_users.py

@@ -59,6 +59,7 @@ config.process()
 args_override = {
         'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
         'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
         }
 config.dict_override(args_override, 'cli')
 config.add(args.user_dir, '_USERDIR', True)

apps/data-seeding/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c

@@ -0,0 +1 @@
+{"address":"eb3907ecad74a0013c259d5874ae7f22dcbcc95c","crypto":{"cipher":"aes-128-ctr","ciphertext":"b0f70a8af4071faff2267374e2423cbc7a71012096fd2215866d8de7445cc215","cipherparams":{"iv":"9ac89383a7793226446dcb7e1b45cdf3"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"299f7b5df1d08a0a7b7f9c9eb44fe4798683b78da3513fcf9603fd913ab3336f"},"mac":"6f4ed36c11345a9a48353cd2f93f1f92958c96df15f3112a192bc994250e8d03"},"id":"61a9dd88-24a9-495c-9a51-152bd1bfaa5b","version":3}

apps/data-seeding/package-lock.json

@@ -1,5 +1,5 @@
 {
-  "name": "scripts",
+  "name": "data-seeding",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
@@ -49,20 +49,20 @@
       }
     },
     "node_modules/@ethereumjs/common": {
-      "version": "2.2.0",
+      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz",
-      "integrity": "sha512-PyQiTG00MJtBRkJmv46ChZL8u2XWxNBeAthznAUIUiefxPAXjbkuiCZOuncgJS34/XkMbNc9zMt/PlgKRBElig==",
+      "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==",
       "dependencies": {
         "crc-32": "^1.2.0",
-        "ethereumjs-util": "^7.0.9"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "node_modules/@ethereumjs/tx": {
-      "version": "3.1.3",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz",
-      "integrity": "sha512-DJBu6cbwYtiPTFeCUR8DF5p+PF0jxs+0rALJZiEcTz2tiRPIEkM72GEbrkGuqzENLCzBrJHT43O0DxSYTqeo+g==",
+      "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==",
       "dependencies": {
-        "@ethereumjs/common": "^2.2.0",
+        "@ethereumjs/common": "^2.3.0",
         "ethereumjs-util": "^7.0.10"
       }
     },
@@ -75,9 +75,9 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "14.14.41",
+      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.41.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g=="
+      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw=="
     },
     "node_modules/@types/pbkdf2": {
       "version": "3.1.0",
@@ -115,6 +115,10 @@
         "fast-json-stable-stringify": "^2.0.0",
         "json-schema-traverse": "^0.4.1",
         "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
       }
     },
     "node_modules/ansi-regex": {
@@ -134,6 +138,9 @@
       },
       "engines": {
         "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
     "node_modules/aproba": {
@@ -803,9 +810,9 @@
       }
     },
     "node_modules/glob": {
-      "version": "7.1.6",
+      "version": "7.1.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
-      "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
       "dependencies": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -816,6 +823,9 @@
       },
       "engines": {
         "node": "*"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
       }
     },
     "node_modules/graceful-fs": {
@@ -836,6 +846,7 @@
       "version": "5.1.5",
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.5.tgz",
       "integrity": "sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==",
+      "deprecated": "this library is no longer supported",
       "optional": true,
       "dependencies": {
         "ajv": "^6.12.3",
@@ -909,9 +920,9 @@
       }
     },
     "node_modules/ignore-walk": {
-      "version": "3.0.3",
+      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz",
-      "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==",
+      "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==",
       "dependencies": {
         "minimatch": "^3.0.4"
       }
@@ -1037,6 +1048,7 @@
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/keccak/-/keccak-3.0.1.tgz",
       "integrity": "sha512-epq90L9jlFWCW7+pQa6JOnKn2Xgl2mtI664seYR6MHskvI9agt7AnDqmAlp9TqU4/caMYbA08Hi5DMZAl5zdkA==",
+      "hasInstallScript": true,
       "dependencies": {
         "node-addon-api": "^2.0.0",
         "node-gyp-build": "^4.2.0"
@@ -1056,21 +1068,21 @@
       }
     },
     "node_modules/mime-db": {
-      "version": "1.47.0",
+      "version": "1.48.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==",
       "optional": true,
       "engines": {
         "node": ">= 0.6"
       }
     },
     "node_modules/mime-types": {
-      "version": "2.1.30",
+      "version": "2.1.31",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==",
       "optional": true,
       "dependencies": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.48.0"
       },
       "engines": {
         "node": ">= 0.6"
@@ -1215,6 +1227,7 @@
       "version": "0.11.0",
       "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz",
       "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==",
+      "deprecated": "Please upgrade to @mapbox/node-pre-gyp: the non-scoped node-pre-gyp package is deprecated and only the @mapbox scoped package will recieve updates in the future",
       "dependencies": {
         "detect-libc": "^1.0.2",
         "mkdirp": "^0.5.1",
@@ -1273,9 +1286,9 @@
       }
     },
     "node_modules/npm-bundled": {
-      "version": "1.1.1",
+      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz",
-      "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==",
+      "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==",
       "dependencies": {
         "npm-normalize-package-bin": "^1.0.1"
       }
@@ -1426,6 +1439,14 @@
       },
       "engines": {
         "node": ">= 8.0.0"
+      },
+      "peerDependencies": {
+        "pg-native": ">=2.0.0"
+      },
+      "peerDependenciesMeta": {
+        "pg-native": {
+          "optional": true
+        }
       }
     },
     "node_modules/pg-connection-string": {
@@ -1444,7 +1465,10 @@
     "node_modules/pg-pool": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz",
-      "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg=="
+      "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==",
+      "peerDependencies": {
+        "pg": ">=8.0"
+      }
     },
     "node_modules/pg-protocol": {
       "version": "1.5.0",
@@ -1596,6 +1620,7 @@
       "version": "2.88.2",
       "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
       "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==",
+      "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142",
       "optional": true,
       "dependencies": {
         "aws-sign2": "~0.7.0",
@@ -1670,7 +1695,21 @@
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ]
     },
     "node_modules/safer-buffer": {
       "version": "2.1.2",
@@ -1691,6 +1730,7 @@
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.2.tgz",
       "integrity": "sha512-UDar4sKvWAksIlfX3xIaQReADn+WFnHvbVujpcbr+9Sf/69odMwy2MUsz5CKLQgX9nsIyrjuxL2imVyoNHa3fg==",
+      "hasInstallScript": true,
       "dependencies": {
         "elliptic": "^6.5.2",
         "node-addon-api": "^2.0.0",
@@ -1755,18 +1795,27 @@
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.0.2.tgz",
       "integrity": "sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA==",
+      "hasInstallScript": true,
       "dependencies": {
         "node-addon-api": "^3.0.0",
         "node-pre-gyp": "^0.11.0"
       },
       "optionalDependencies": {
         "node-gyp": "3.x"
+      },
+      "peerDependencies": {
+        "node-gyp": "3.x"
+      },
+      "peerDependenciesMeta": {
+        "node-gyp": {
+          "optional": true
+        }
       }
     },
     "node_modules/sqlite3/node_modules/node-addon-api": {
-      "version": "3.1.0",
+      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
-      "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
+      "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
     },
     "node_modules/sshpk": {
       "version": "1.16.1",
@@ -1784,11 +1833,6 @@
         "safer-buffer": "^2.0.2",
         "tweetnacl": "~0.14.0"
       },
-      "bin": {
-        "sshpk-conv": "bin/sshpk-conv",
-        "sshpk-sign": "bin/sshpk-sign",
-        "sshpk-verify": "bin/sshpk-verify"
-      },
       "engines": {
         "node": ">=0.10.0"
       }
@@ -1872,12 +1916,16 @@
     "node_modules/transit-immutable-js": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/transit-immutable-js/-/transit-immutable-js-0.7.0.tgz",
-      "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk="
+      "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=",
+      "peerDependencies": {
+        "immutable": ">= 3",
+        "transit-js": ">= 0.8"
+      }
     },
     "node_modules/transit-js": {
-      "version": "0.8.867",
+      "version": "0.8.874",
-      "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.867.tgz",
+      "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.874.tgz",
-      "integrity": "sha512-rOwB4K0z/WZ+E2bV42iN9UV3mvGzmwSv/IpMOKdnFpawPAZT0d1L7f91Y+tZQF7lXSDGk+oln4XyIQXo+pyTGA==",
+      "integrity": "sha512-IDJJGKRzUbJHmN0P15HBBa05nbKor3r2MmG6aSt0UxXIlJZZKcddTk67/U7WyAeW9Hv/VYI02IqLzolsC4sbPA==",
       "engines": {
         "node": ">= 0.10.0"
       }
@@ -1923,6 +1971,7 @@
       "version": "3.4.0",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
       "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
       "bin": {
         "uuid": "bin/uuid"
       }
@@ -1977,6 +2026,9 @@
       },
       "engines": {
         "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
       }
     },
     "node_modules/wrap-ansi/node_modules/ansi-regex": {
@@ -2151,20 +2203,20 @@
       }
     },
     "@ethereumjs/common": {
-      "version": "2.2.0",
+      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.3.0.tgz",
-      "integrity": "sha512-PyQiTG00MJtBRkJmv46ChZL8u2XWxNBeAthznAUIUiefxPAXjbkuiCZOuncgJS34/XkMbNc9zMt/PlgKRBElig==",
+      "integrity": "sha512-Fmi15MdVptsC85n6NcUXIFiiXCXWEfZNgPWP+OGAQOC6ZtdzoNawtxH/cYpIgEgSuIzfOeX3VKQP/qVI1wISHg==",
       "requires": {
         "crc-32": "^1.2.0",
-        "ethereumjs-util": "^7.0.9"
+        "ethereumjs-util": "^7.0.10"
       }
     },
     "@ethereumjs/tx": {
-      "version": "3.1.3",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@ethereumjs/tx/-/tx-3.2.0.tgz",
-      "integrity": "sha512-DJBu6cbwYtiPTFeCUR8DF5p+PF0jxs+0rALJZiEcTz2tiRPIEkM72GEbrkGuqzENLCzBrJHT43O0DxSYTqeo+g==",
+      "integrity": "sha512-D3X/XtZ3ldUg34hr99Jvj7NxW3NxVKdUKrwQnEWlAp4CmCQpvYoyn7NF4lk34rHEt7ScS+Agu01pcDHoOcd19A==",
       "requires": {
-        "@ethereumjs/common": "^2.2.0",
+        "@ethereumjs/common": "^2.3.0",
         "ethereumjs-util": "^7.0.10"
       }
     },
@@ -2177,9 +2229,9 @@
       }
     },
     "@types/node": {
-      "version": "14.14.41",
+      "version": "15.12.0",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.41.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.0.tgz",
-      "integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g=="
+      "integrity": "sha512-+aHJvoCsVhO2ZCuT4o5JtcPrCPyDE3+1nvbDprYes+pPkEsbjH7AGUCNtjMOXS0fqH14t+B7yLzaqSz92FPWyw=="
     },
     "@types/pbkdf2": {
       "version": "3.1.0",
@@ -2822,9 +2874,9 @@
       }
     },
     "glob": {
-      "version": "7.1.6",
+      "version": "7.1.7",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz",
-      "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+      "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==",
       "requires": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -2909,9 +2961,9 @@
       }
     },
     "ignore-walk": {
-      "version": "3.0.3",
+      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.4.tgz",
-      "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==",
+      "integrity": "sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==",
       "requires": {
         "minimatch": "^3.0.4"
       }
@@ -3037,18 +3089,18 @@
       }
     },
     "mime-db": {
-      "version": "1.47.0",
+      "version": "1.48.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.47.0.tgz",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.48.0.tgz",
-      "integrity": "sha512-QBmA/G2y+IfeS4oktet3qRZ+P5kPhCKRXxXnQEudYqUaEioAU1/Lq2us3D/t1Jfo4hE9REQPrbB7K5sOczJVIw==",
+      "integrity": "sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ==",
       "optional": true
     },
     "mime-types": {
-      "version": "2.1.30",
+      "version": "2.1.31",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.30.tgz",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.31.tgz",
-      "integrity": "sha512-crmjA4bLtR8m9qLpHvgxSChT+XoSlZi8J4n/aIdn3z92e/U47Z0V/yl+Wh9W046GgFVAmoNR/fmdbZYcSSIUeg==",
+      "integrity": "sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg==",
       "optional": true,
       "requires": {
-        "mime-db": "1.47.0"
+        "mime-db": "1.48.0"
       }
     },
     "minimalistic-assert": {
@@ -3209,9 +3261,9 @@
       }
     },
     "npm-bundled": {
-      "version": "1.1.1",
+      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.2.tgz",
-      "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==",
+      "integrity": "sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==",
       "requires": {
         "npm-normalize-package-bin": "^1.0.1"
       }
@@ -3350,7 +3402,8 @@
     "pg-pool": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz",
-      "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg=="
+      "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==",
+      "requires": {}
     },
     "pg-protocol": {
       "version": "1.5.0",
@@ -3610,9 +3663,9 @@
       },
       "dependencies": {
         "node-addon-api": {
-          "version": "3.1.0",
+          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz",
-          "integrity": "sha512-flmrDNB06LIl5lywUz7YlNGZH/5p0M7W28k8hzd9Lshtdh1wshD2Y+U4h9LD6KObOy1f+fEVdgprPrEymjM5uw=="
+          "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A=="
         }
       }
     },
@@ -3696,12 +3749,13 @@
     "transit-immutable-js": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/transit-immutable-js/-/transit-immutable-js-0.7.0.tgz",
-      "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk="
+      "integrity": "sha1-mT4lCJtjEf9AIUD1VidtbSUwBdk=",
+      "requires": {}
     },
     "transit-js": {
-      "version": "0.8.867",
+      "version": "0.8.874",
-      "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.867.tgz",
+      "resolved": "https://registry.npmjs.org/transit-js/-/transit-js-0.8.874.tgz",
-      "integrity": "sha512-rOwB4K0z/WZ+E2bV42iN9UV3mvGzmwSv/IpMOKdnFpawPAZT0d1L7f91Y+tZQF7lXSDGk+oln4XyIQXo+pyTGA=="
+      "integrity": "sha512-IDJJGKRzUbJHmN0P15HBBa05nbKor3r2MmG6aSt0UxXIlJZZKcddTk67/U7WyAeW9Hv/VYI02IqLzolsC4sbPA=="
     },
     "tunnel-agent": {
       "version": "0.6.0",

apps/data-seeding/verify.py

@@ -9,6 +9,7 @@ import sys
 import urllib
 import urllib.request
 import uuid
+import urllib.parse
 
 # external imports
 import celery
@@ -72,7 +73,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
 argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
 argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
 argparser.add_argument('--include', action='append', type=str, help='include specified verification')
-argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
+argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
 argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
 argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@@ -185,9 +186,9 @@ def send_ussd_request(address, data_dir):
     }
 
     req = urllib.request.Request(config.get('_USSD_PROVIDER'))
-    data_str = json.dumps(data)
+    urlencoded_data = urllib.parse.urlencode(data)
-    data_bytes = data_str.encode('utf-8')
+    data_bytes = urlencoded_data.encode('utf-8')
-    req.add_header('Content-Type', 'application/json')
+    req.add_header('Content-Type', 'application/x-www-form-urlencoded')
     req.data = data_bytes
     response = urllib.request.urlopen(req)
     return response.read().decode('utf-8')
@@ -388,10 +389,9 @@ class Verifier:
 
     def verify_ussd_pins(self, address, balance):
         response_data = send_ussd_request(address, self.data_dir)
-        if response_data[:11] != 'CON Balance':
+        if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
             raise VerifierError(response_data, 'pins')
 
-
     def verify(self, address, balance, debug_stem=None):
   
         for k in active_tests:

docker-compose.yml

@@ -484,7 +484,7 @@ services:
       restart_policy:
         condition: on-failure
     volumes:
-      - ${LOCAL_VOLUME_DIR:-/tmp/cic}/pgp:/tmp/cic/pgp
+      - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp
     # command: "/root/start_server.sh -vv"
 
   cic-user-ussd-server: