Add custom meta import script and generate from import users scripts

This commit is contained in:
nolash 2021-04-23 21:01:24 +02:00
parent 41a3b2e060
commit 306b11ee22
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
4 changed files with 2190 additions and 16 deletions

View File

@ -32,6 +32,7 @@ default_config_dir = '/usr/local/etc/cic'
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission')
argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@ -76,6 +77,11 @@ os.makedirs(user_new_dir)
meta_dir = os.path.join(args.user_dir, 'meta') meta_dir = os.path.join(args.user_dir, 'meta')
os.makedirs(meta_dir) os.makedirs(meta_dir)
custom_dir = os.path.join(args.user_dir, 'custom')
os.makedirs(custom_dir)
os.makedirs(os.path.join(custom_dir, 'new'))
os.makedirs(os.path.join(custom_dir, 'meta'))
phone_dir = os.path.join(args.user_dir, 'phone') phone_dir = os.path.join(args.user_dir, 'phone')
os.makedirs(os.path.join(phone_dir, 'meta')) os.makedirs(os.path.join(phone_dir, 'meta'))
@ -85,6 +91,11 @@ os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs') txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir) os.makedirs(txs_dir)
user_dir = args.user_dir
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec) chain_str = str(chain_spec)
@ -133,7 +144,17 @@ def register_eth(i, u):
if __name__ == '__main__': if __name__ == '__main__':
#fi = open(os.path.join(user_out_dir, 'addresses.csv'), 'a') user_tags = {}
f = open(os.path.join(user_dir, 'tags.csv'), 'r')
while True:
r = f.readline().rstrip()
if len(r) == 0:
break
(old_address, tags_csv) = r.split(':')
old_address = strip_0x(old_address)
user_tags[old_address] = tags_csv.split(',')
logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
i = 0 i = 0
j = 0 j = 0
@ -198,6 +219,29 @@ if __name__ == '__main__':
os.symlink(os.path.realpath(filepath), meta_phone_filepath) os.symlink(os.path.realpath(filepath), meta_phone_filepath)
# custom data
custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
filepath = os.path.join(
custom_dir,
'new',
custom_key[:2].upper(),
custom_key[2:4].upper(),
custom_key.upper() + '.json',
)
os.makedirs(os.path.dirname(filepath), exist_ok=True)
sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
f = open(filepath, 'w')
k = u.identities['evm'][sub_old_chain_str][0]
tag_data = {'tags': user_tags[strip_0x(k)]}
f.write(json.dumps(tag_data))
f.close()
os.symlink(os.path.realpath(filepath), custom_filepath)
i += 1 i += 1
sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r") sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")

View File

@ -37,6 +37,7 @@ config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config'))
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') argparser.add_argument('-c', type=str, default=config_dir, help='Config dir')
argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record')
argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)') argparser.add_argument('--gift-threshold', type=int, help='If set, users will be funded with additional random balance (in token integer units)')
argparser.add_argument('-v', action='store_true', help='Be verbose') argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', action='store_true', help='Be more verbose') argparser.add_argument('-vv', action='store_true', help='Be more verbose')
@ -81,6 +82,10 @@ phone_idx = []
user_dir = args.dir user_dir = args.dir
user_count = args.user_count user_count = args.user_count
tags = args.tag
if tags == None or len(tags) == 0:
tags = ['individual']
random.seed() random.seed()
def genPhoneIndex(phone): def genPhoneIndex(phone):
@ -189,6 +194,7 @@ if __name__ == '__main__':
os.makedirs(base_dir, exist_ok=True) os.makedirs(base_dir, exist_ok=True)
fa = open(os.path.join(user_dir, 'balances.csv'), 'w') fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
ft = open(os.path.join(user_dir, 'tags.csv'), 'w')
i = 0 i = 0
while i < user_count: while i < user_count:
@ -215,10 +221,12 @@ if __name__ == '__main__':
f.write(eth) f.write(eth)
f.close() f.close()
ft.write('{}:{}\n'.format(eth, ','.join(tags)))
amount = genAmount() amount = genAmount()
fa.write('{},{}\n'.format(eth,amount)) fa.write('{},{}\n'.format(eth,amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone)) logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone))
i += 1 i += 1
ft.close()
fa.close() fa.close()

View File

@ -38,6 +38,7 @@ argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address') argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node') argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node')
@ -68,6 +69,11 @@ os.makedirs(user_new_dir)
meta_dir = os.path.join(args.user_dir, 'meta') meta_dir = os.path.join(args.user_dir, 'meta')
os.makedirs(meta_dir) os.makedirs(meta_dir)
custom_dir = os.path.join(args.user_dir, 'custom')
os.makedirs(custom_dir)
os.makedirs(os.path.join(custom_dir, 'new'))
os.makedirs(os.path.join(custom_dir, 'meta'))
phone_dir = os.path.join(args.user_dir, 'phone') phone_dir = os.path.join(args.user_dir, 'phone')
os.makedirs(os.path.join(phone_dir, 'meta')) os.makedirs(os.path.join(phone_dir, 'meta'))
@ -77,9 +83,14 @@ os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs') txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir) os.makedirs(txs_dir)
user_dir = args.user_dir
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec) chain_str = str(chain_spec)
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec)
batch_size = args.batch_size batch_size = args.batch_size
batch_delay = args.batch_delay batch_delay = args.batch_delay
@ -127,12 +138,18 @@ def register_eth(i, u):
return address return address
def register_ussd(u):
pass
if __name__ == '__main__': if __name__ == '__main__':
user_tags = {}
f = open(os.path.join(user_dir, 'tags.csv'), 'r')
while True:
r = f.readline().rstrip()
if len(r) == 0:
break
(old_address, tags_csv) = r.split(':')
old_address = strip_0x(old_address)
user_tags[old_address] = tags_csv.split(',')
logg.debug('read tags {} for old address {}'.format(user_tags[old_address], old_address))
i = 0 i = 0
j = 0 j = 0
@ -150,6 +167,7 @@ if __name__ == '__main__':
continue continue
f.close() f.close()
u = Person.deserialize(o) u = Person.deserialize(o)
logg.debug('u {}'.format(o))
new_address = register_eth(i, u) new_address = register_eth(i, u)
if u.identities.get('evm') == None: if u.identities.get('evm') == None:
@ -157,8 +175,6 @@ if __name__ == '__main__':
sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
u.identities['evm'][sub_chain_str] = [new_address] u.identities['evm'][sub_chain_str] = [new_address]
register_ussd(u)
new_address_clean = strip_0x(new_address) new_address_clean = strip_0x(new_address)
filepath = os.path.join( filepath = os.path.join(
user_new_dir, user_new_dir,
@ -198,6 +214,29 @@ if __name__ == '__main__':
os.symlink(os.path.realpath(filepath), meta_phone_filepath) os.symlink(os.path.realpath(filepath), meta_phone_filepath)
# custom data
custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
filepath = os.path.join(
custom_dir,
'new',
custom_key[:2].upper(),
custom_key[2:4].upper(),
custom_key.upper() + '.json',
)
os.makedirs(os.path.dirname(filepath), exist_ok=True)
sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
f = open(filepath, 'w')
k = u.identities['evm'][sub_old_chain_str][0]
tag_data = {'tags': user_tags[strip_0x(k)]}
f.write(json.dumps(tag_data))
f.close()
os.symlink(os.path.realpath(filepath), custom_filepath)
i += 1 i += 1
sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r") sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")

File diff suppressed because it is too large Load Diff