Rename cic-tools to chainlib

Philip/ussd notify

See merge request grassrootseconomics/cic-internal-integration!11

apps/cic-eth/docker/Dockerfile

@@ -41,9 +41,3 @@ COPY cic-eth/tests/ tests/
 COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
-
-RUN apt-get install -y git && \
-	git clone https://gitlab.com/grassrootseconomics/cic-contracts.git &&\
-	mkdir -p /usr/local/share/cic/solidity/abi && \
-	cp cic-contracts/abis/* /usr/local/share/cic/solidity/abi/
-	

apps/cic-ussd/.gitlab-ci.yml

@@ -18,5 +18,5 @@ build-push-cic-ussd:
     extends:
         - .py_build_push
         - .cic_ussd_variables
-        
+
 

apps/cic-ussd/docker/Dockerfile

@@ -39,6 +39,8 @@ COPY cic-ussd/docker/db.sh \
      /root/
 
 RUN chmod +x /root/*.sh
+RUN cd cic-ussd && \
+    pip install $pip_extra_index_url_flag .
 
 
 # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts

apps/cic-ussd/docker/start_tasker.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-. ./db.sh
+. /root/db.sh
 
 /usr/local/bin/cic-ussd-tasker -vv "$@"

apps/cic-ussd/docker/start_uwsgi.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-. ./db.sh
+. /root/db.sh
 
-/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :80 --pyargv "-vv"
+/usr/local/bin/uwsgi --wsgi-file /usr/local/lib/python3.8/site-packages/cic_ussd/runnable/server.py --http :9000 --pyargv "-vv"

apps/cic-ussd/requirements.txt

@@ -6,10 +6,10 @@ betterpath==0.2.2
 billiard==3.6.3.0
 celery==4.4.7
 cffi==1.14.3
-cic-eth~=0.10.0a9
+cic-eth~=0.10.0a22
 cic-notify==0.3.1
 click==7.1.2
-confini==0.3.5
+confini~=0.3.6a1
 cryptography==3.2.1
 faker==4.17.1
 iniconfig==1.1.1

apps/contract-migration/scripts/generate_users.py

@@ -1,220 +0,0 @@
-#!python3
-
-# standard imports
-import json
-import time
-import datetime
-import random
-import logging
-import os
-import base64
-import hashlib
-import sys
-
-import vobject
-
-import celery
-import web3
-from faker import Faker
-import cic_registry
-import confini
-from cic_eth.api import Api
-
-logging.basicConfig(level=logging.DEBUG)
-logg = logging.getLogger()
-
-fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
-
-config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
-argparser.add_argument('-z', '--zero-balance', dest='z', action='store_true', help='do not generate initial balances')
-argparser.add_argument('source', type=str, help='input directory to process')
-args = argparser.parse_args()
-
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'ETH_PROVIDER': getattr(args, 'p'),
-        }
-config.dict_override(args_override, 'cli flag')
-logg.info('loaded config\n{}'.format(config))
-
-
-# registration time generation
-dt_now = datetime.datetime.utcnow()
-dt_then = dt_now - datetime.timedelta(weeks=150)
-ts_now = int(dt_now.timestamp())
-ts_then = int(dt_then.timestamp())
-
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-api = Api(config.get('CIC_CHAIN_SPEC'))
-
-gift_activate = not args.z
-gift_max = 10000
-gift_factor = (10**9)
-
-
-categories = [
-        "food/water",
-        "fuel/energy",
-        "education",
-        "health",
-        "shop",
-        "environment",
-        "transport",
-        "farming/labor",
-        "savingsgroup",
-        ]
-
-phone_idx = []
-
-
-def genPhoneIndex(phone):
-    h = hashlib.new('sha256')
-    h.update(phone.encode('utf-8'))
-    h.update(b'cic.msisdn')
-    return h.digest().hex()
-
-
-def genId(addr, typ):
-    h = hashlib.new('sha256')
-    h.update(bytes.fromhex(addr[2:]))
-    h.update(typ.encode('utf-8'))
-    return h.digest().hex()
-
-
-def genDate():
-
-    logg.info(ts_then)
-    ts = random.randint(ts_then, ts_now)
-    return datetime.datetime.fromtimestamp(ts).timestamp()
-
-
-def genPhone():
-    return fake.msisdn()
-
-
-def genPersonal(phone):
-    fn = fake.first_name()
-    ln = fake.last_name()
-    e = fake.email()
-
-    v = vobject.vCard()
-    first_name = fake.first_name()
-    last_name = fake.last_name()
-    v.add('n')
-    v.n.value = vobject.vcard.Name(family=last_name, given=first_name)
-    v.add('fn')
-    v.fn.value = '{} {}'.format(first_name, last_name)
-    v.add('tel')
-    v.tel.typ_param = 'CELL'
-    v.tel.value = phone
-    v.add('email')
-    v.email.value = fake.email()
-
-    vcard_serialized = v.serialize()
-    vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8'))
-
-    return vcard_base64.decode('utf-8')
-
-
-def genCats():
-    i = random.randint(0, 3)
-    return random.choices(categories, k=i)
-
-
-def genAmount():
-    return random.randint(0, gift_max) * gift_factor
-
-
-def gen():
-    old_blockchain_address = '0x' + os.urandom(20).hex()
-    accounts_index_account = config.get('DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER')
-    if not accounts_index_account:
-        accounts_index_account = None
-    logg.debug('accounts indexwriter {}'.format(accounts_index_account))
-    t = api.create_account()
-    new_blockchain_address = t.get()
-    gender = random.choice(['female', 'male', 'other'])
-    phone = genPhone()
-    v = genPersonal(phone)
-    o = {
-        'date_registered': genDate(),
-        'vcard': v,
-        'gender': gender,
-        'key': {
-            'ethereum': [
-                old_blockchain_address,
-                new_blockchain_address,
-                ],
-            },
-        'location': {
-            'latitude': str(fake.latitude()),
-            'longitude': str(fake.longitude()),
-            'external': { # add osm lookup
-                }
-            },
-        'selling': genCats(),
-            }
-    uid = genId(new_blockchain_address, 'cic.person')
-
-    #logg.info('gifting {} to {}'.format(amount, new_blockchain_address))
-
-    return (uid, phone, o)
-
-
-def prepareLocalFilePath(datadir, address):
-    parts = [
-                address[:2],
-                address[2:4],
-            ]
-    dirs = '{}/{}/{}'.format(
-            datadir,
-            parts[0],
-            parts[1],
-            )
-    os.makedirs(dirs, exist_ok=True)
-    return dirs
-
-
-def main():
-    os.makedirs('data/person', exist_ok=True)
-    os.makedirs('data/phone', exist_ok=True)
-
-    fa = open('./data/amounts', 'w')
-    fb = open('./data/addresses', 'w')
-
-    for i in range(int(sys.argv[1])):
-    
-        (uid, phone, o) = gen()
-        eth = o['key']['ethereum'][1]
-
-        print(o)
-
-        d = prepareLocalFilePath('./data/person', uid)
-        f = open('{}/{}'.format(d, uid), 'w')
-        json.dump(o, f)
-        f.close()
-
-        pidx = genPhoneIndex(phone)
-        d = prepareLocalFilePath('./data/phone', uid)
-        f = open('{}/{}'.format(d, pidx), 'w')
-        f.write(eth)
-        f.close()
-
-        amount = genAmount()
-        fa.write('{},{}\n'.format(eth,amount))
-        fb.write('{}\n'.format(eth))
-        logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
-
-    fb.close()
-    fa.close()
-
-
-if __name__ == '__main__':
-    main()

apps/contract-migration/scripts/import_users.py

@@ -1,115 +0,0 @@
-# standard imports
-import os
-import sys
-import json
-import logging
-import argparse
-import uuid
-from glob import glob
-
-# third-party imports
-import redis
-import confini
-import celery
-from chainlib.eth.address import to_checksum
-from cic_types.models.person import Person
-from cic_eth.api.api_task import Api
-from cic_registry.chain import ChainSpec
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-default_config_dir = '/usr/local/etc/cic'
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
-argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
-argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
-argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
-argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
-argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
-argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
-argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout')
-argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
-argparser.add_argument('-v', action='store_true', help='Be verbose')
-argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-argparser.add_argument('user_dir', type=str, help='path to users export dir tree')
-args = argparser.parse_args()
-
-if args.v:
-    logg.setLevel(logging.INFO)
-elif args.vv:
-    logg.setLevel(logging.DEBUG)
-
-config_dir = args.c
-config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
-config.process()
-args_override = {
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        'REDIS_HOST': getattr(args, 'redis_host'),
-        'REDIS_PORT': getattr(args, 'redis_port'),
-        'REDIS_DB': getattr(args, 'redis_db'),
-        }
-config.dict_override(args_override, 'cli')
-celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
-
-redis_host = config.get('REDIS_HOST')
-redis_port = config.get('REDIS_PORT')
-redis_db = config.get('REDIS_DB')
-r = redis.Redis(redis_host, redis_port, redis_db)
-
-ps = r.pubsub()
-
-user_dir = args.user_dir
-
-
-def register_eth(u):
-    redis_channel = str(uuid.uuid4())
-    ps.subscribe(redis_channel)
-    ps.get_message()
-    api = Api(
-        config.get('CIC_CHAIN_SPEC'),
-        queue=args.q,
-        callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
-        callback_task='cic_eth.callbacks.redis.redis',
-        callback_queue=args.q,
-        )
-    t = api.create_account(register=True)
-
-    ps.get_message()
-    m = ps.get_message(timeout=args.timeout)
-    address = json.loads(m['data'])
-    logg.debug('register eth {} {}'.format(u, address))
-
-    return address
-   
-
-def register_ussd(u):
-    pass
-
-
-if __name__ == '__main__':
-
-    fi = open(os.path.join(user_dir, 'addresses.csv'), 'a') # old,new
-
-    for x in os.walk(user_dir):
-        for y in x[2]:
-            if y[len(y)-5:] != '.json':
-                continue
-            filepath = os.path.join(x[0], y)
-            f = open(filepath, 'r')
-            try:
-                o = json.load(f)
-            except json.decoder.JSONDecodeError as e:
-                f.close()
-                logg.error('load error for {}: {}'.format(y, e))
-                continue
-            f.close()
-            u = Person(o)
-
-            new_address = register_eth(u) 
-    
-            old_address = u.identities['evm']['bloxberg:8995'][0]
-            fi.write('{}.{}\n'.format(old_address, new_address))
-
-    fi.close()

apps/contract-migration/scripts/requirements.txt

@@ -1,2 +0,0 @@
-cic-types==0.1.0
-chainlib==0.0.1a4

apps/contract-migration/seed_cic_eth.sh

@@ -24,7 +24,7 @@ truncate $env_out_file -s 0
 set -e
 set -a
 
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a25 cic-tools==0.0.1a4
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a25 chainlib==0.0.1a4
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER

docker-compose.yml

@@ -458,49 +458,49 @@ services:
       - ${LOCAL_VOLUME_DIR:-/tmp/cic}/pgp:/tmp/cic/pgp
     command: "/root/start_server.sh -vv"
 
-#  cic-ussd-server:
-#    # image: grassrootseconomics:cic-ussd
-#    build:
-#      context: apps/
-#      dockerfile: cic-ussd/docker/Dockerfile
-#    environment:
-#      DATABASE_USER: grassroots
-#      DATABASE_HOST: postgres
-#      DATABASE_PORT: 5432
-#      DATABASE_PASSWORD: tralala
-#      DATABASE_NAME: cic_ussd
-#      DATABASE_ENGINE: postgresql
-#      DATABASE_DRIVER: psycopg2
-#      SERVER_PORT: 80
-#    ports:
-#      - 80:8082
-#    depends_on:
-#      - postgres
-#      - redis
-#    deploy:
-#      restart_policy:
-#        condition: on-failure
-#    command: "/root/start_uwsgi.sh"
-#
-#  cic-ussd-tasker:
-#    # image: grassrootseconomics:cic-ussd
-#    build:
-#      context: apps
-#      dockerfile: cic-ussd/docker/Dockerfile
-#    environment:
-#      DATABASE_USER: grassroots
-#      DATABASE_HOST: postgres
-#      DATABASE_PORT: 5432
-#      DATABASE_PASSWORD: tralala
-#      DATABASE_NAME: cic_ussd
-#      DATABASE_ENGINE: postgresql
-#      DATABASE_DRIVER: psycopg2
-#      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
-#      CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis}
-#    depends_on:
-#      - postgres
-#      - redis
-#    deploy:
-#      restart_policy:
-#        condition: on-failure
-#    command: "/root/start_tasker.sh -q cic-ussd"
+  cic-ussd-server:
+    # image: grassrootseconomics:cic-ussd
+    build:
+      context: apps/
+      dockerfile: cic-ussd/docker/Dockerfile
+    environment:
+      DATABASE_USER: grassroots
+      DATABASE_HOST: postgres
+      DATABASE_PORT: 5432
+      DATABASE_PASSWORD: tralala
+      DATABASE_NAME: cic_ussd
+      DATABASE_ENGINE: postgresql
+      DATABASE_DRIVER: psycopg2
+      SERVER_PORT: 9000
+    ports:
+      - 9000:8082
+    depends_on:
+      - postgres
+      - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
+    command: "/root/start_uwsgi.sh"
+
+  cic-ussd-tasker:
+    # image: grassrootseconomics:cic-ussd
+    build:
+      context: apps
+      dockerfile: cic-ussd/docker/Dockerfile
+    environment:
+      DATABASE_USER: grassroots
+      DATABASE_HOST: postgres
+      DATABASE_PORT: 5432
+      DATABASE_PASSWORD: tralala
+      DATABASE_NAME: cic_ussd
+      DATABASE_ENGINE: postgresql
+      DATABASE_DRIVER: psycopg2
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
+      CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis}
+    depends_on:
+      - postgres
+      - redis
+    deploy:
+      restart_policy:
+        condition: on-failure
+    command: "/root/start_tasker.sh -q cic-ussd"