Rename cic-tools to new location

apps/cic-eth/docker/Dockerfile

@@ -41,3 +41,9 @@ COPY cic-eth/tests/ tests/
 COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+RUN apt-get install -y git && \
+	git clone https://gitlab.com/grassrootseconomics/cic-contracts.git &&\
+	mkdir -p /usr/local/share/cic/solidity/abi && \
+	cp cic-contracts/abis/* /usr/local/share/cic/solidity/abi/
+	

apps/contract-migration/scripts/generate_users.py

@@ -0,0 +1,220 @@
+#!python3
+
+# standard imports
+import json
+import time
+import datetime
+import random
+import logging
+import os
+import base64
+import hashlib
+import sys
+
+import vobject
+
+import celery
+import web3
+from faker import Faker
+import cic_registry
+import confini
+from cic_eth.api import Api
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
+
+config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('-z', '--zero-balance', dest='z', action='store_true', help='do not generate initial balances')
+argparser.add_argument('source', type=str, help='input directory to process')
+args = argparser.parse_args()
+
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'ETH_PROVIDER': getattr(args, 'p'),
+        }
+config.dict_override(args_override, 'cli flag')
+logg.info('loaded config\n{}'.format(config))
+
+
+# registration time generation
+dt_now = datetime.datetime.utcnow()
+dt_then = dt_now - datetime.timedelta(weeks=150)
+ts_now = int(dt_now.timestamp())
+ts_then = int(dt_then.timestamp())
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+api = Api(config.get('CIC_CHAIN_SPEC'))
+
+gift_activate = not args.z
+gift_max = 10000
+gift_factor = (10**9)
+
+
+categories = [
+        "food/water",
+        "fuel/energy",
+        "education",
+        "health",
+        "shop",
+        "environment",
+        "transport",
+        "farming/labor",
+        "savingsgroup",
+        ]
+
+phone_idx = []
+
+
+def genPhoneIndex(phone):
+    h = hashlib.new('sha256')
+    h.update(phone.encode('utf-8'))
+    h.update(b'cic.msisdn')
+    return h.digest().hex()
+
+
+def genId(addr, typ):
+    h = hashlib.new('sha256')
+    h.update(bytes.fromhex(addr[2:]))
+    h.update(typ.encode('utf-8'))
+    return h.digest().hex()
+
+
+def genDate():
+
+    logg.info(ts_then)
+    ts = random.randint(ts_then, ts_now)
+    return datetime.datetime.fromtimestamp(ts).timestamp()
+
+
+def genPhone():
+    return fake.msisdn()
+
+
+def genPersonal(phone):
+    fn = fake.first_name()
+    ln = fake.last_name()
+    e = fake.email()
+
+    v = vobject.vCard()
+    first_name = fake.first_name()
+    last_name = fake.last_name()
+    v.add('n')
+    v.n.value = vobject.vcard.Name(family=last_name, given=first_name)
+    v.add('fn')
+    v.fn.value = '{} {}'.format(first_name, last_name)
+    v.add('tel')
+    v.tel.typ_param = 'CELL'
+    v.tel.value = phone
+    v.add('email')
+    v.email.value = fake.email()
+
+    vcard_serialized = v.serialize()
+    vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8'))
+
+    return vcard_base64.decode('utf-8')
+
+
+def genCats():
+    i = random.randint(0, 3)
+    return random.choices(categories, k=i)
+
+
+def genAmount():
+    return random.randint(0, gift_max) * gift_factor
+
+
+def gen():
+    old_blockchain_address = '0x' + os.urandom(20).hex()
+    accounts_index_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER')
+    if not accounts_index_account:
+        accounts_index_account = None
+    logg.debug('accounts indexwriter {}'.format(accounts_index_account))
+    t = api.create_account()
+    new_blockchain_address = t.get()
+    gender = random.choice(['female', 'male', 'other'])
+    phone = genPhone()
+    v = genPersonal(phone)
+    o = {
+        'date_registered': genDate(),
+        'vcard': v,
+        'gender': gender,
+        'key': {
+            'ethereum': [
+                old_blockchain_address,
+                new_blockchain_address,
+                ],
+            },
+        'location': {
+            'latitude': str(fake.latitude()),
+            'longitude': str(fake.longitude()),
+            'external': { # add osm lookup
+                }
+            },
+        'selling': genCats(),
+            }
+    uid = genId(new_blockchain_address, 'cic.person')
+
+    #logg.info('gifting {} to {}'.format(amount, new_blockchain_address))
+
+    return (uid, phone, o)
+
+
+def prepareLocalFilePath(datadir, address):
+    parts = [
+                address[:2],
+                address[2:4],
+            ]
+    dirs = '{}/{}/{}'.format(
+            datadir,
+            parts[0],
+            parts[1],
+            )
+    os.makedirs(dirs, exist_ok=True)
+    return dirs
+
+
+def main():
+    os.makedirs('data/person', exist_ok=True)
+    os.makedirs('data/phone', exist_ok=True)
+
+    fa = open('./data/amounts', 'w')
+    fb = open('./data/addresses', 'w')
+
+    for i in range(int(sys.argv[1])):
+    
+        (uid, phone, o) = gen()
+        eth = o['key']['ethereum'][1]
+
+        print(o)
+
+        d = prepareLocalFilePath('./data/person', uid)
+        f = open('{}/{}'.format(d, uid), 'w')
+        json.dump(o, f)
+        f.close()
+
+        pidx = genPhoneIndex(phone)
+        d = prepareLocalFilePath('./data/phone', uid)
+        f = open('{}/{}'.format(d, pidx), 'w')
+        f.write(eth)
+        f.close()
+
+        amount = genAmount()
+        fa.write('{},{}\n'.format(eth,amount))
+        fb.write('{}\n'.format(eth))
+        logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+
+    fb.close()
+    fa.close()
+
+
+if __name__ == '__main__':
+    main()

apps/contract-migration/scripts/import_users.py

@@ -0,0 +1,115 @@
+# standard imports
+import os
+import sys
+import json
+import logging
+import argparse
+import uuid
+from glob import glob
+
+# third-party imports
+import redis
+import confini
+import celery
+from chainlib.eth.address import to_checksum
+from cic_types.models.person import Person
+from cic_eth.api.api_task import Api
+from cic_registry.chain import ChainSpec
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+default_config_dir = '/usr/local/etc/cic'
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
+argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
+argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
+argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('user_dir', type=str, help='path to users export dir tree')
+args = argparser.parse_args()
+
+if args.v:
+    logg.setLevel(logging.INFO)
+elif args.vv:
+    logg.setLevel(logging.DEBUG)
+
+config_dir = args.c
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        'REDIS_HOST': getattr(args, 'redis_host'),
+        'REDIS_PORT': getattr(args, 'redis_port'),
+        'REDIS_DB': getattr(args, 'redis_db'),
+        }
+config.dict_override(args_override, 'cli')
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+r = redis.Redis(redis_host, redis_port, redis_db)
+
+ps = r.pubsub()
+
+user_dir = args.user_dir
+
+
+def register_eth(u):
+    redis_channel = str(uuid.uuid4())
+    ps.subscribe(redis_channel)
+    ps.get_message()
+    api = Api(
+        config.get('CIC_CHAIN_SPEC'),
+        queue=args.q,
+        callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+        callback_task='cic_eth.callbacks.redis.redis',
+        callback_queue=args.q,
+        )
+    t = api.create_account(register=True)
+
+    ps.get_message()
+    m = ps.get_message(timeout=args.timeout)
+    address = json.loads(m['data'])
+    logg.debug('register eth {} {}'.format(u, address))
+
+    return address
+   
+
+def register_ussd(u):
+    pass
+
+
+if __name__ == '__main__':
+
+    fi = open(os.path.join(user_dir, 'addresses.csv'), 'a') # old,new
+
+    for x in os.walk(user_dir):
+        for y in x[2]:
+            if y[len(y)-5:] != '.json':
+                continue
+            filepath = os.path.join(x[0], y)
+            f = open(filepath, 'r')
+            try:
+                o = json.load(f)
+            except json.decoder.JSONDecodeError as e:
+                f.close()
+                logg.error('load error for {}: {}'.format(y, e))
+                continue
+            f.close()
+            u = Person(o)
+
+            new_address = register_eth(u) 
+    
+            old_address = u.identities['evm']['bloxberg:8995'][0]
+            fi.write('{}.{}\n'.format(old_address, new_address))
+
+    fi.close()

apps/contract-migration/scripts/requirements.txt

@@ -0,0 +1,2 @@
+cic-types==0.1.0
+chainlib==0.0.1a4