From bf80443c8d98fe2d7dd331a0bb8502ecb1969f73 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 10 Feb 2021 17:56:31 +0100 Subject: [PATCH] Add user generation script, import script --- apps/cic-eth/docker/Dockerfile | 6 + .../scripts/generate_users.py | 220 ++++++++++++++++++ .../scripts/import_users.py | 115 +++++++++ .../scripts/requirements.txt | 1 + 4 files changed, 342 insertions(+) create mode 100644 apps/contract-migration/scripts/generate_users.py create mode 100644 apps/contract-migration/scripts/import_users.py create mode 100644 apps/contract-migration/scripts/requirements.txt diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 88dfea8c..f0aecb3a 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -41,3 +41,9 @@ COPY cic-eth/tests/ tests/ COPY cic-eth/config/ /usr/local/etc/cic-eth/ COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ + +RUN apt-get install -y git && \ + git clone https://gitlab.com/grassrootseconomics/cic-contracts.git &&\ + mkdir -p /usr/local/share/cic/solidity/abi && \ + cp cic-contracts/abis/* /usr/local/share/cic/solidity/abi/ + diff --git a/apps/contract-migration/scripts/generate_users.py b/apps/contract-migration/scripts/generate_users.py new file mode 100644 index 00000000..89d6b8c2 --- /dev/null +++ b/apps/contract-migration/scripts/generate_users.py @@ -0,0 +1,220 @@ +#!python3 + +# standard imports +import json +import time +import datetime +import random +import logging +import os +import base64 +import hashlib +import sys + +import vobject + +import celery +import web3 +from faker import Faker +import cic_registry +import confini +from cic_eth.api import Api + +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + +fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) + +config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') + +argparser = argparse.ArgumentParser() +argparser.add_argument('-c', type=str, default=config_dir, help='config file') +argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') +argparser.add_argument('-z', '--zero-balance', dest='z', action='store_true', help='do not generate initial balances') +argparser.add_argument('source', type=str, help='input directory to process') +args = argparser.parse_args() + +config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) +config.process() +args_override = { + 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'ETH_PROVIDER': getattr(args, 'p'), + } +config.dict_override(args_override, 'cli flag') +logg.info('loaded config\n{}'.format(config)) + + +# registration time generation +dt_now = datetime.datetime.utcnow() +dt_then = dt_now - datetime.timedelta(weeks=150) +ts_now = int(dt_now.timestamp()) +ts_then = int(dt_then.timestamp()) + +celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) + +api = Api(config.get('CIC_CHAIN_SPEC')) + +gift_activate = not args.z +gift_max = 10000 +gift_factor = (10**9) + + +categories = [ + "food/water", + "fuel/energy", + "education", + "health", + "shop", + "environment", + "transport", + "farming/labor", + "savingsgroup", + ] + +phone_idx = [] + + +def genPhoneIndex(phone): + h = hashlib.new('sha256') + h.update(phone.encode('utf-8')) + h.update(b'cic.msisdn') + return h.digest().hex() + + +def genId(addr, typ): + h = hashlib.new('sha256') + h.update(bytes.fromhex(addr[2:])) + h.update(typ.encode('utf-8')) + return h.digest().hex() + + +def genDate(): + + logg.info(ts_then) + ts = random.randint(ts_then, ts_now) + return datetime.datetime.fromtimestamp(ts).timestamp() + + +def genPhone(): + return fake.msisdn() + + +def genPersonal(phone): + fn = fake.first_name() + ln = fake.last_name() + e = fake.email() + + v = vobject.vCard() + first_name = fake.first_name() + last_name = fake.last_name() + v.add('n') + v.n.value = vobject.vcard.Name(family=last_name, given=first_name) + v.add('fn') + v.fn.value = '{} {}'.format(first_name, last_name) + v.add('tel') + v.tel.typ_param = 'CELL' + v.tel.value = phone + v.add('email') + v.email.value = fake.email() + + vcard_serialized = v.serialize() + vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8')) + + return vcard_base64.decode('utf-8') + + +def genCats(): + i = random.randint(0, 3) + return random.choices(categories, k=i) + + +def genAmount(): + return random.randint(0, gift_max) * gift_factor + + +def gen(): + old_blockchain_address = '0x' + os.urandom(20).hex() + accounts_index_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') + if not accounts_index_account: + accounts_index_account = None + logg.debug('accounts indexwriter {}'.format(accounts_index_account)) + t = api.create_account() + new_blockchain_address = t.get() + gender = random.choice(['female', 'male', 'other']) + phone = genPhone() + v = genPersonal(phone) + o = { + 'date_registered': genDate(), + 'vcard': v, + 'gender': gender, + 'key': { + 'ethereum': [ + old_blockchain_address, + new_blockchain_address, + ], + }, + 'location': { + 'latitude': str(fake.latitude()), + 'longitude': str(fake.longitude()), + 'external': { # add osm lookup + } + }, + 'selling': genCats(), + } + uid = genId(new_blockchain_address, 'cic.person') + + #logg.info('gifting {} to {}'.format(amount, new_blockchain_address)) + + return (uid, phone, o) + + +def prepareLocalFilePath(datadir, address): + parts = [ + address[:2], + address[2:4], + ] + dirs = '{}/{}/{}'.format( + datadir, + parts[0], + parts[1], + ) + os.makedirs(dirs, exist_ok=True) + return dirs + + +def main(): + os.makedirs('data/person', exist_ok=True) + os.makedirs('data/phone', exist_ok=True) + + fa = open('./data/amounts', 'w') + fb = open('./data/addresses', 'w') + + for i in range(int(sys.argv[1])): + + (uid, phone, o) = gen() + eth = o['key']['ethereum'][1] + + print(o) + + d = prepareLocalFilePath('./data/person', uid) + f = open('{}/{}'.format(d, uid), 'w') + json.dump(o, f) + f.close() + + pidx = genPhoneIndex(phone) + d = prepareLocalFilePath('./data/phone', uid) + f = open('{}/{}'.format(d, pidx), 'w') + f.write(eth) + f.close() + + amount = genAmount() + fa.write('{},{}\n'.format(eth,amount)) + fb.write('{}\n'.format(eth)) + logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount)) + + fb.close() + fa.close() + + +if __name__ == '__main__': + main() diff --git a/apps/contract-migration/scripts/import_users.py b/apps/contract-migration/scripts/import_users.py new file mode 100644 index 00000000..66c990d5 --- /dev/null +++ b/apps/contract-migration/scripts/import_users.py @@ -0,0 +1,115 @@ +# standard imports +import os +import sys +import json +import logging +import argparse +import uuid +from glob import glob + +# third-party imports +import redis +import confini +import celery +from cic_types.models.person import Person +from cic_eth.api.api_task import Api +from cic_registry.chain import ChainSpec +from cic_tools.eth.address import to_checksum + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + +default_config_dir = '/usr/local/etc/cic' + +argparser = argparse.ArgumentParser() +argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') +argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string') +argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') +argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') +argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') +argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') +argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') +argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') +argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') +argparser.add_argument('-v', action='store_true', help='Be verbose') +argparser.add_argument('-vv', action='store_true', help='Be more verbose') +argparser.add_argument('user_dir', type=str, help='path to users export dir tree') +args = argparser.parse_args() + +if args.v: + logg.setLevel(logging.INFO) +elif args.vv: + logg.setLevel(logging.DEBUG) + +config_dir = args.c +config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) +config.process() +args_override = { + 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'REDIS_HOST': getattr(args, 'redis_host'), + 'REDIS_PORT': getattr(args, 'redis_port'), + 'REDIS_DB': getattr(args, 'redis_db'), + } +config.dict_override(args_override, 'cli') +celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) + +redis_host = config.get('REDIS_HOST') +redis_port = config.get('REDIS_PORT') +redis_db = config.get('REDIS_DB') +r = redis.Redis(redis_host, redis_port, redis_db) + +ps = r.pubsub() + +user_dir = args.user_dir + + +def register_eth(u): + redis_channel = str(uuid.uuid4()) + ps.subscribe(redis_channel) + ps.get_message() + api = Api( + config.get('CIC_CHAIN_SPEC'), + queue=args.q, + callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + callback_task='cic_eth.callbacks.redis.redis', + callback_queue=args.q, + ) + t = api.create_account(register=True) + + ps.get_message() + m = ps.get_message(timeout=args.timeout) + address = json.loads(m['data']) + logg.debug('register eth {} {}'.format(u, address)) + + return address + + +def register_ussd(u): + pass + + +if __name__ == '__main__': + + fi = open(os.path.join(user_dir, 'addresses.csv'), 'a') # old,new + + for x in os.walk(user_dir): + for y in x[2]: + if y[len(y)-5:] != '.json': + continue + filepath = os.path.join(x[0], y) + f = open(filepath, 'r') + try: + o = json.load(f) + except json.decoder.JSONDecodeError as e: + f.close() + logg.error('load error for {}: {}'.format(y, e)) + continue + f.close() + u = Person(o) + + new_address = register_eth(u) + + old_address = u.identities['evm']['bloxberg:8995'][0] + fi.write('{}.{}\n'.format(old_address, new_address)) + + fi.close() diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt new file mode 100644 index 00000000..9f2cd1f0 --- /dev/null +++ b/apps/contract-migration/scripts/requirements.txt @@ -0,0 +1 @@ +cic-types==0.1.0