kamikazechaser
/
cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clean ups.

This commit is contained in:
PhilipWafula 2021-10-20 15:52:40 +03:00
parent 0e9c4e5199
commit 92ef656cab
Signed by untrusted user: mango-habanero
GPG Key ID: B00CE9034DA19FB7
4 changed files with 42 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apps/data-seeding/cic_ussd/import_balance.py
View File

@ -57,8 +57,8 @@ elif args.v:
config = Config(args.c, args.env_prefix)
config.process()
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CHAIN_SPEC': getattr(args, 'i'),
'RPC_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'REDIS_HOST': getattr(args, 'redis_host'),
'REDIS_PORT': getattr(args, 'redis_port'),
@ -90,7 +90,7 @@ signer = EIP155Signer(keystore)
block_offset = -1 if args.head else args.offset
chain_str = config.get('CIC_CHAIN_SPEC')
chain_str = config.get('CHAIN_SPEC')
chain_spec = ChainSpec.from_chain_str(chain_str)
ImportTask.chain_spec = chain_spec
old_chain_spec_str = args.old_chain_spec
@ -99,16 +99,12 @@ old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str)
MetadataTask.meta_host = config.get('META_HOST')
MetadataTask.meta_port = config.get('META_PORT')
txs_dir = os.path.join(args.import_dir, 'txs')
os.makedirs(txs_dir, exist_ok=True)
sys.stdout.write(f'created txs dir: {txs_dir}')
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
get_celery_worker_status(celery_app)
def main():
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
ImportTask.balance_processor = BalanceProcessor(conn,
chain_spec,
config.get('CIC_REGISTRY_ADDRESS'),

22
apps/data-seeding/cic_ussd/import_task.py
View File

@ -14,7 +14,9 @@ from celery import Task
from chainlib.chain import ChainSpec
from chainlib.eth.address import to_checksum_address
from chainlib.eth.tx import raw, unpack
from cic_types.models.person import Person, generate_metadata_pointer
from cic_types.models.person import Person, identity_tag
from cic_types.processor import generate_metadata_pointer
from cic_types.condiments import MetadataPointer
from hexathon import add_0x, strip_0x
# local imports
@ -55,7 +57,7 @@ class MetadataTask(ImportTask):
def old_address_from_phone(base_path: str, phone_number: str):
pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}')
with open(phone_idx_path, 'r') as f:
old_address = f.read()
@ -73,9 +75,13 @@ def generate_person_metadata(self, blockchain_address: str, phone_number: str):
person = Person.deserialize(person_metadata)
if not person.identities.get('evm'):
person.identities['evm'] = {}
sub_chain_str = f'{self.chain_spec.common_name()}:{self.chain_spec.network_id()}'
person.identities['evm'][sub_chain_str] = [add_0x(blockchain_address)]
blockchain_address = strip_0x(blockchain_address)
chain_spec = self.chain_spec.asdict()
arch = chain_spec.get('arch')
fork = chain_spec.get('fork')
tag = identity_tag(chain_spec)
person.identities[arch][fork] = {
tag: [blockchain_address]
}
file_path = os.path.join(
self.import_dir,
'new',
@ -102,7 +108,7 @@ def generate_preferences_data(self, data: tuple):
blockchain_address: str = data[0]
preferences = data[1]
preferences_dir = os.path.join(self.import_dir, 'preferences')
preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), ':cic.preferences')
preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES)
preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
filepath = os.path.join(
preferences_dir,
@ -137,7 +143,7 @@ def generate_ussd_data(self, blockchain_address: str, phone_number: str):
preferred_language = random.sample(["en", "sw"], 1)[0]
preferences = {'preferred_language': preferred_language}
with open(ussd_data_file, file_op) as ussd_data_file:
ussd_data_file.write(f'{phone_number}, { 1}, {preferred_language}, {False}\n')
ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n')
logg.debug(f'written ussd data for address: {blockchain_address}')
return blockchain_address, preferences
@ -163,7 +169,7 @@ def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial:
@celery_app.task(bind=True, base=MetadataTask)
def resolve_phone(self, phone_number: str):
identifier = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone')
identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
url = parse.urljoin(self.meta_url(), identifier)
logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}')
r = request.urlopen(url)

20
apps/data-seeding/cic_ussd/import_users.py
View File

@ -17,6 +17,7 @@ from cic_types.models.person import Person
from confini import Config
# local imports
from common.dirs import initialize_dirs
from import_util import get_celery_worker_status
default_config_dir = './config'
@ -37,6 +38,7 @@ arg_parser.add_argument('--env-prefix',
dest='env_prefix',
type=str,
help='environment prefix for variables to overwrite configuration.')
arg_parser.add_argument('-f', action='store_true', help='force clear previous state')
arg_parser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit data seeding tasks to.')
arg_parser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
@ -70,21 +72,7 @@ config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug(f'config loaded from {args.c}:\n{config}')
old_account_dir = os.path.join(args.import_dir, 'old')
os.stat(old_account_dir)
logg.debug(f'created old system data dir: {old_account_dir}')
new_account_dir = os.path.join(args.import_dir, 'new')
os.makedirs(new_account_dir, exist_ok=True)
logg.debug(f'created new system data dir: {new_account_dir}')
person_metadata_dir = os.path.join(args.import_dir, 'meta')
os.makedirs(person_metadata_dir, exist_ok=True)
logg.debug(f'created person metadata dir: {person_metadata_dir}')
preferences_dir = os.path.join(args.import_dir, 'preferences')
os.makedirs(os.path.join(preferences_dir, 'meta'), exist_ok=True)
logg.debug(f'created preferences metadata dir: {preferences_dir}')
dirs = initialize_dirs(args.import_dir, force_reset=args.f)
valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
@ -157,7 +145,7 @@ def register_account(person: Person):
if __name__ == '__main__':
i = 0
j = 0
for x in os.walk(old_account_dir):
for x in os.walk(dirs['old']):
for y in x[2]:
if y[len(y) - 5:] != '.json':
continue

34
apps/data-seeding/verify.py
View File

@ -25,10 +25,9 @@ from chainlib.eth.gas import (
from chainlib.eth.tx import TxFactory
from chainlib.hash import keccak256_string_to_hex
from chainlib.jsonrpc import JSONRPCRequest
from cic_types.models.person import (
Person,
generate_metadata_pointer,
)
from cic_types.models.person import Person, identity_tag
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
from erc20_faucet import Faucet
from eth_erc20 import ERC20
from hexathon.parse import strip_0x, add_0x
@ -74,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-c', type=str, help='config override dir')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec')
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
@ -108,7 +107,7 @@ config.process()
# override args
args_override = {
'CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'RPC_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
}
config.dict_override(args_override, 'cli flag')
@ -307,7 +306,7 @@ class Verifier:
def verify_gas(self, address, balance_token=None):
o = balance(address)
o = balance(add_0x(address))
r = self.conn.do(o)
logg.debug('wtf {}'.format(r))
actual_balance = int(strip_0x(r), 16)
@ -323,7 +322,7 @@ class Verifier:
def verify_metadata(self, address, balance=None):
k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person')
k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), MetadataPointer.PERSON)
url = os.path.join(config.get('_META_PROVIDER'), k)
logg.debug('verify metadata url {}'.format(url))
try:
@ -367,7 +366,7 @@ class Verifier:
p = Person.deserialize(o)
k = generate_metadata_pointer(p.tel.encode('utf-8'), ':cic.phone')
k = generate_metadata_pointer(p.tel.encode('utf-8'), MetadataPointer.PHONE)
url = os.path.join(config.get('_META_PROVIDER'), k)
logg.debug('verify metadata phone url {}'.format(url))
try:
@ -427,7 +426,7 @@ class Verifier:
def main():
global chain_str, block_offset, user_dir
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
# Get Token registry address
@ -505,10 +504,17 @@ def main():
u = Person.deserialize(o)
#logg.debug('data {}'.format(u.identities['evm']))
subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
new_address = u.identities['evm'][subchain_str][0]
subchain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id())
old_address = u.identities['evm'][subchain_str][0]
new_chain_spec = chain_spec.asdict()
arch = new_chain_spec.get('arch')
fork = new_chain_spec.get('fork')
tag = identity_tag(new_chain_spec)
new_address = u.identities[arch][fork][tag][0]
old_chainspec = old_chain_spec.asdict()
arch = old_chainspec.get('arch')
fork = old_chainspec.get('fork')
tag = identity_tag(old_chainspec)
old_address = u.identities[arch][fork][tag][0]
balance = 0
try:
balance = balances[old_address]