diff --git a/apps/cic-meta/scripts/server/server.ts b/apps/cic-meta/scripts/server/server.ts index 8599cf23..67490a80 100755 --- a/apps/cic-meta/scripts/server/server.ts +++ b/apps/cic-meta/scripts/server/server.ts @@ -137,6 +137,7 @@ async function processRequest(req, res) { console.debug('mode', mod); let content = ''; let contentType = 'application/json'; + console.debug('handling data', data); let r:any = undefined; try { switch (mod) { diff --git a/apps/contract-migration/scripts/create_users.py b/apps/contract-migration/scripts/create_users.py deleted file mode 100644 index 1cdaede3..00000000 --- a/apps/contract-migration/scripts/create_users.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/python - -# standard imports -import json -import time -import datetime -import random -import logging -import os -import base64 -import hashlib -import sys -import argparse -import random - -# external imports -import vobject -import celery -import web3 -from faker import Faker -import cic_registry -import confini -from cic_eth.api import Api -from cic_types.models.person import ( - Person, - generate_vcard_from_contact_data, - get_contact_data_from_vcard, - ) - -logging.basicConfig(level=logging.WARNING) -logg = logging.getLogger() - -fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) - -config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') -argparser.add_argument('--gift-threshold', action='store_true', help='If set, users will be funded with additional random balance (in token integer units)') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('--skip-identities', action='store_true', help='do not include generated ethereum address in user data') -argparser.add_argument('--dir', default='out', type=str, help='path to users export dir tree') -argparser.add_argument('user_count', type=int, help='amount of users to generate') -args = argparser.parse_args() - -if args.v: - logg.setLevel(logging.INFO) -elif args.vv: - logg.setLevel(logging.DEBUG) - -config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -logg.info('loaded config\n{}'.format(config)) - - -dt_now = datetime.datetime.utcnow() -dt_then = dt_now - datetime.timedelta(weeks=150) -ts_now = int(dt_now.timestamp()) -ts_then = int(dt_then.timestamp()) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - -api = Api(config.get('CIC_CHAIN_SPEC')) - -gift_max = args.gift_threshold or 0 -gift_factor = (10**6) - -categories = [ - "food/water", - "fuel/energy", - "education", - "health", - "shop", - "environment", - "transport", - "farming/labor", - "savingsgroup", - ] - -phone_idx = [] - -user_dir = args.dir -user_count = args.user_count - -def genPhoneIndex(phone): - h = hashlib.new('sha256') - h.update(phone.encode('utf-8')) - h.update(b'cic.msisdn') - return h.digest().hex() - - -def genId(addr, typ): - h = hashlib.new('sha256') - h.update(bytes.fromhex(addr[2:])) - h.update(typ.encode('utf-8')) - return h.digest().hex() - - -def genDate(): - - logg.info(ts_then) - ts = random.randint(ts_then, ts_now) - return datetime.datetime.fromtimestamp(ts).timestamp() - - -def genPhone(): - return fake.msisdn() - - -def genPersonal(phone): - fn = fake.first_name() - ln = fake.last_name() - e = fake.email() - - return generate_vcard_from_contact_data(ln, fn, phone, e) - -# v = vobject.vCard() -# first_name = fake.first_name() -# last_name = fake.last_name() -# v.add('n') -# v.n.value = vobject.vcard.Name(family=last_name, given=first_name) -# v.add('fn') -# v.fn.value = '{} {}'.format(first_name, last_name) -# v.add('tel') -# v.tel.typ_param = 'CELL' -# v.tel.value = phone -# v.add('email') -# v.email.value = fake.email() -# -# vcard_serialized = v.serialize() -# vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8')) - -# return vcard_base64.decode('utf-8') - - -def genCats(): - i = random.randint(0, 3) - return random.choices(categories, k=i) - - -def genAmount(): - return random.randint(0, gift_max) * gift_factor - - -def gen(): - old_blockchain_address = '0x' + os.urandom(20).hex() - #accounts_index_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') - #if not accounts_index_account: - # accounts_index_account = None - #logg.debug('accounts indexwriter {}'.format(accounts_index_account)) - #t = api.create_account(register=True) - #new_blockchain_address = t.get() - gender = random.choice(['female', 'male', 'other']) - phone = genPhone() - #phone = '254{}'.format(random.randint(4791000000, 4792000000)) - #phone = '254791549131' - logg.debug('phone {}'.format(phone)) - v = genPersonal(phone) - contact_data = get_contact_data_from_vcard(v) - - p = Person() - p.load_vcard(contact_data) - p.year = '0000' -# o = { -# 'date_registered': genDate(), -# 'vcard': v, -# 'gender': gender, -# 'key': { -# 'ethereum': [ -# old_blockchain_address, -# new_blockchain_address, -# ], -# }, -# 'location': { -# 'latitude': str(fake.latitude()), -# 'longitude': str(fake.longitude()), -# 'external': { # add osm lookup -# } -# }, -# 'selling': genCats(), -# } -# uid = genId(new_blockchain_address, 'cic.person') - - #logg.info('gifting {} to {}'.format(amount, new_blockchain_address)) - - return (old_blockchain_address, phone, p) - - -def prepareLocalFilePath(datadir, address): - parts = [ - address[:2], - address[2:4], - ] - dirs = '{}/old/{}/{}'.format( - datadir, - parts[0], - parts[1], - ) - os.makedirs(dirs, exist_ok=True) - return dirs - - -if __name__ == '__main__': - - base_dir = os.path.join(user_dir, 'old') - os.makedirs(base_dir, exist_ok=True) - - fa = open(os.path.join(base_dir, 'balances.csv'), 'w') - - for i in range(user_count): - - (eth, phone, o) = gen() - uid = eth[2:].upper() - - print(o) - - d = prepareLocalFilePath(base_dir, uid + '.json') - f = open('{}/{}'.format(d, uid), 'w') - json.dump(o.serialize(), f) - f.close() - - pidx = genPhoneIndex(phone) - d = prepareLocalFilePath(os.path.join(user_dir, 'phone'), uid) - f = open('{}/{}'.format(d, pidx), 'w') - f.write(eth) - f.close() - - amount = genAmount() - fa.write('{},{}\n'.format(eth,amount)) - logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount)) - - fa.close() diff --git a/apps/contract-migration/scripts/import_balance.py b/apps/contract-migration/scripts/import_balance.py index 224fbf8d..2aa9de88 100644 --- a/apps/contract-migration/scripts/import_balance.py +++ b/apps/contract-migration/scripts/import_balance.py @@ -48,6 +48,7 @@ argparser = argparse.ArgumentParser(description='daemon that monitors transactio argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='oldchain:1', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') argparser.add_argument('--head', action='store_true', help='start at current block height (overrides --offset)') @@ -98,6 +99,7 @@ else: block_offset = args.offset chain_spec = ChainSpec.from_chain_str('evm:' + chain_str) +old_chain_spec_str = args.old_chain_spec user_dir = args.user_dir # user_out_dir from import_users.py @@ -141,7 +143,7 @@ class Handler: logg.error('no import record of address {}'.format(recipient)) return u = Person(o) - original_address = u.identities['evm']['xdai:1'][0] + original_address = u.identities['evm'][old_chain_spec_str][0] balance = self.balances[original_address] logg.info('registered {} originally {} ({}) tx hash {} balance {}'.format(recipient, original_address, u, tx.hash, balance)) diff --git a/apps/contract-migration/scripts/import_meta.js b/apps/contract-migration/scripts/import_meta.js index 58190e83..e35f3254 100644 --- a/apps/contract-migration/scripts/import_meta.js +++ b/apps/contract-migration/scripts/import_meta.js @@ -14,25 +14,29 @@ console.log(config); function sendit(uid, envelope) { const d = envelope.toJSON(); + const contentLength = (new TextEncoder().encode(d)).length; const opts = { method: 'PUT', headers: { 'Content-Type': 'application/json', - 'Content-Length': d.length, + 'Content-Length': contentLength, 'X-CIC-AUTOMERGE': 'client', }, }; let url = config.get('META_URL'); url = url.replace(new RegExp('^(.+://[^/]+)/*$'), '$1/'); - console.debug('url: ' + url + uid); + //console.debug('url: ' + url + uid); const req = http.request(url + uid, opts, (res) => { res.on('data', process.stdout.write); res.on('end', () => { console.log('result', res.statusCode, res.headers); }); }); - req.write(d); + if (!req.write(d)) { + console.error('foo', d); + process.exit(1); + } req.end(); } @@ -44,7 +48,7 @@ function doOne(keystore, filePath) { cic.User.toKey('0x' + ethereum_address).then((uid) => { const d = fs.readFileSync(filePath, 'utf-8'); const o = JSON.parse(d); - console.log(o); + //console.log(o); fs.unlinkSync(filePath); const s = new cic.Syncable(uid, o); @@ -99,7 +103,7 @@ function importMeta(keystore) { console.debug('skipping file', file); } const filePath = path.join(workDir, file); - console.log('file', count, filePath); + //console.log('file', count, filePath); doOne(keystore, filePath); count++; batchCount++; diff --git a/apps/contract-migration/scripts/import_users.py b/apps/contract-migration/scripts/import_users.py index 70125fed..8a06cf72 100644 --- a/apps/contract-migration/scripts/import_users.py +++ b/apps/contract-migration/scripts/import_users.py @@ -154,7 +154,7 @@ if __name__ == '__main__': #fi.write('{},{}\n'.format(new_address, old_address)) meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person') meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper())) - os.symlink(filepath, meta_filepath) + os.symlink(os.path.realpath(filepath), meta_filepath) i += 1 sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")