Add more better mocked data

This commit is contained in:
Geoff Turk 2021-06-02 14:53:53 +02:00
parent 546d69f1e9
commit e2946052e0

View File

@ -2,27 +2,24 @@
# standard imports # standard imports
import json import json
import time
import datetime import datetime
import random import random
import logging import logging
import os import os
import base64
import hashlib import hashlib
import sys
import argparse import argparse
import random import random
# external imports # external imports
import vobject
import celery import celery
from faker import Faker from faker import Faker
from collections import OrderedDict
import confini import confini
from cic_types.models.person import ( from cic_types.models.person import (
Person, Person,
generate_vcard_from_contact_data, generate_vcard_from_contact_data,
get_contact_data_from_vcard, get_contact_data_from_vcard,
) )
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
import phonenumbers import phonenumbers
@ -32,17 +29,21 @@ logg = logging.getLogger()
fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
script_dir = os.path.realpath(os.path.dirname(__file__)) script_dir = os.path.realpath(os.path.dirname(__file__))
#config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') # config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config')) config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record') argparser.add_argument('--tag', type=str, action='append',
argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)') help='Tags to add to record')
argparser.add_argument('--gift-threshold', type=int,
help='If set, users will be funded with additional random balance (in token integer units)')
argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', action='store_true', help='Be more verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose')
argparser.add_argument('--dir', default='out', type=str, help='path to users export dir tree') argparser.add_argument('--dir', default='out', type=str,
argparser.add_argument('user_count', type=int, help='amount of users to generate') help='path to users export dir tree')
argparser.add_argument('user_count', type=int,
help='amount of users to generate')
args = argparser.parse_args() args = argparser.parse_args()
if args.v: if args.v:
@ -60,22 +61,23 @@ dt_then = dt_now - datetime.timedelta(weeks=150)
ts_now = int(dt_now.timestamp()) ts_now = int(dt_now.timestamp())
ts_then = int(dt_then.timestamp()) ts_then = int(dt_then.timestamp())
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) celery_app = celery.Celery(broker=config.get(
'CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
gift_max = args.gift_threshold or 0 gift_max = args.gift_threshold or 0
gift_factor = (10**6) gift_factor = (10**6)
categories = [ categories = [
"food/water", "food/water",
"fuel/energy", "fuel/energy",
"education", "education",
"health", "health",
"shop", "shop",
"environment", "environment",
"transport", "transport",
"farming/labor", "farming/labor",
"savingsgroup", "savingsgroup",
] ]
phone_idx = [] phone_idx = []
@ -88,6 +90,7 @@ if tags == None or len(tags) == 0:
random.seed() random.seed()
def genPhoneIndex(phone): def genPhoneIndex(phone):
h = hashlib.new('sha256') h = hashlib.new('sha256')
h.update(phone.encode('utf-8')) h.update(phone.encode('utf-8'))
@ -141,16 +144,16 @@ def genDob():
if random.random() > 0.5: if random.random() > 0.5:
dob['month'] = dob_src.month dob['month'] = dob_src.month
dob['day'] = dob_src.day dob['day'] = dob_src.day
return dob return dob
def gen(): def gen():
old_blockchain_address = '0x' + os.urandom(20).hex() old_blockchain_address = '0x' + os.urandom(20).hex()
old_blockchain_checksum_address = to_checksum_address(old_blockchain_address) old_blockchain_checksum_address = to_checksum_address(
old_blockchain_address)
gender = random.choice(['female', 'male', 'other']) gender = random.choice(['female', 'male', 'other'])
phone = genPhone() phone = genPhone()
city = fake.city_name()
v = genPersonal(phone) v = genPersonal(phone)
contact_data = get_contact_data_from_vcard(v) contact_data = get_contact_data_from_vcard(v)
@ -161,31 +164,63 @@ def gen():
p.date_of_birth = genDob() p.date_of_birth = genDob()
p.gender = gender p.gender = gender
p.identities = { p.identities = {
'evm': { 'evm': {
'oldchain:1': [ 'oldchain:1': [
old_blockchain_checksum_address, old_blockchain_checksum_address,
], ],
}, },
} }
p.location['area_name'] = city p.products = [fake.random_element(elements=OrderedDict(
[('fruit', 0.25),
('maji', 0.05),
('milk', 0.1),
('teacher', 0.1),
('doctor', 0.05),
('boutique', 0.15),
('recycling', 0.05),
('farmer', 0.05),
('oil', 0.05),
('pastor', 0.1),
('chama', 0.03),
('pastor', 0.01),
('bzrTsuZieaq', 0.01)
]))]
p.location['area_name'] = fake.random_element(elements=OrderedDict(
[('mnarani', 0.05),
('chilumani', 0.1),
('madewani', 0.1),
('kisauni', 0.05),
('bangla', 0.1),
('kabiro', 0.1),
('manyani', 0.05),
('ruben', 0.15),
('makupa', 0.05),
('kingston', 0.05),
('rangala', 0.05),
('homabay', 0.1),
('nakuru', 0.03),
('kajiado', 0.01),
('zurtWicKtily', 0.01)
]))
if random.randint(0, 1): if random.randint(0, 1):
p.location['latitude'] = (random.random() + 180) - 90 #fake.local_latitude() # fake.local_latitude()
p.location['longitude'] = (random.random() + 360) - 180 #fake.local_latitude() p.location['latitude'] = (random.random() + 180) - 90
# fake.local_latitude()
p.location['longitude'] = (random.random() + 360) - 180
return (old_blockchain_checksum_address, phone, p) return (old_blockchain_checksum_address, phone, p)
def prepareLocalFilePath(datadir, address): def prepareLocalFilePath(datadir, address):
parts = [ parts = [
address[:2], address[:2],
address[2:4], address[2:4],
] ]
dirs = '{}/{}/{}'.format( dirs = '{}/{}/{}'.format(
datadir, datadir,
parts[0], parts[0],
parts[1], parts[1],
) )
os.makedirs(dirs, exist_ok=True) os.makedirs(dirs, exist_ok=True)
return dirs return dirs
@ -238,9 +273,10 @@ if __name__ == '__main__':
ft.write('{}:{}\n'.format(eth, ','.join(tags))) ft.write('{}:{}\n'.format(eth, ','.join(tags)))
amount = genAmount() amount = genAmount()
fa.write('{},{}\n'.format(eth,amount)) fa.write('{},{}\n'.format(eth, amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone)) logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(
pidx, uid, eth, amount, phone))
i += 1 i += 1
ft.close() ft.close()