cic-stack/apps/data-seeding/cic_ussd/import_task.py

# standard imports
import csv
import json
import logging
import os
import random
import uuid
from urllib import error, parse, request

# external imports
import celery
import psycopg2
from celery import Task
from chainlib.chain import ChainSpec
from chainlib.eth.address import to_checksum_address
from chainlib.eth.tx import raw, unpack
from cic_types.models.person import Person, identity_tag
from cic_types.processor import generate_metadata_pointer
from cic_types.condiments import MetadataPointer
from hexathon import add_0x, strip_0x

# local imports


celery_app = celery.current_app
logg = logging.getLogger()


class ImportTask(Task):
    balances = None
    balance_processor = None
    chain_spec: ChainSpec = None
    count = 0
    db_config: dict = None
    import_dir = ''
    include_balances = False
    max_retries = None


class MetadataTask(ImportTask):
    meta_host = None
    meta_port = None
    meta_path = ''
    meta_ssl = False
    autoretry_for = (error.HTTPError, OSError,)
    retry_jitter = True
    retry_backoff = True
    retry_backoff_max = 60

    @classmethod
    def meta_url(cls):
        scheme = 'http'
        if cls.meta_ssl:
            scheme += 's'
        url = parse.urlparse(f'{scheme}://{cls.meta_host}:{cls.meta_port}/{cls.meta_path}')
        return parse.urlunparse(url)


def old_address_from_phone(base_path: str, phone_number: str):
    pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
    phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}')
    with open(phone_idx_path, 'r') as f:
        old_address = f.read()
    return old_address


@celery_app.task(bind=True, base=MetadataTask)
def generate_person_metadata(self, blockchain_address: str, phone_number: str):
    logg.debug(f'blockchain address: {blockchain_address}')
    old_blockchain_address = old_address_from_phone(self.import_dir, phone_number)
    old_address_upper = strip_0x(old_blockchain_address).upper()
    metadata_path = f'{self.import_dir}/old/{old_address_upper[:2]}/{old_address_upper[2:4]}/{old_address_upper}.json'
    with open(metadata_path, 'r') as metadata_file:
        person_metadata = json.load(metadata_file)
    person = Person.deserialize(person_metadata)
    if not person.identities.get('evm'):
        person.identities['evm'] = {}
    chain_spec = self.chain_spec.asdict()
    arch = chain_spec.get('arch')
    fork = chain_spec.get('fork')
    tag = identity_tag(chain_spec)
    person.identities[arch][fork] = {
        tag: [blockchain_address]
    }
    file_path = os.path.join(
        self.import_dir,
        'new',
        blockchain_address[:2].upper(),
        blockchain_address[2:4].upper(),
        blockchain_address.upper() + '.json'
    )
    os.makedirs(os.path.dirname(file_path), exist_ok=True)
    serialized_person_metadata = person.serialize()
    with open(file_path, 'w') as metadata_file:
        metadata_file.write(json.dumps(serialized_person_metadata))
        logg.debug(f'written person metadata for address: {blockchain_address}')
    meta_filepath = os.path.join(
        self.import_dir,
        'meta',
        '{}.json'.format(blockchain_address.upper()),
    )
    os.symlink(os.path.realpath(file_path), meta_filepath)
    return blockchain_address


@celery_app.task(bind=True, base=MetadataTask)
def generate_preferences_data(self, data: tuple):
    blockchain_address: str = data[0]
    preferences = data[1]
    preferences_dir = os.path.join(self.import_dir, 'preferences')
    preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES)
    preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key)
    filepath = os.path.join(
        preferences_dir,
        'new',
        preferences_key[:2].upper(),
        preferences_key[2:4].upper(),
        preferences_key.upper() + '.json'
    )
    os.makedirs(os.path.dirname(filepath), exist_ok=True)
    with open(filepath, 'w') as preferences_file:
        preferences_file.write(json.dumps(preferences))
        logg.debug(f'written preferences metadata: {preferences} for address: {blockchain_address}')
    os.symlink(os.path.realpath(filepath), preferences_filepath)
    return blockchain_address


@celery_app.task(bind=True, base=MetadataTask)
def generate_pins_data(self, blockchain_address: str, phone_number: str):
    pins_file = f'{self.import_dir}/pins.csv'
    file_op = 'a' if os.path.exists(pins_file) else 'w'
    with open(pins_file, file_op) as pins_file:
        password_hash = uuid.uuid4().hex
        pins_file.write(f'{phone_number},{password_hash}\n')
        logg.debug(f'written pin data for address: {blockchain_address}')
    return blockchain_address


@celery_app.task(bind=True, base=MetadataTask)
def generate_ussd_data(self, blockchain_address: str, phone_number: str):
    ussd_data_file = f'{self.import_dir}/ussd_data.csv'
    file_op = 'a' if os.path.exists(ussd_data_file) else 'w'
    preferred_language = random.sample(["en", "sw"], 1)[0]
    preferences = {'preferred_language': preferred_language}
    with open(ussd_data_file, file_op) as ussd_data_file:
        ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n')
        logg.debug(f'written ussd data for address: {blockchain_address}')
    return blockchain_address, preferences


@celery_app.task(bind=True, base=MetadataTask)
def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial: str):
    old_blockchain_address = old_address_from_phone(self.import_dir, phone_number)
    address = to_checksum_address(strip_0x(old_blockchain_address))
    balance = self.balances[address]
    logg.debug(f'found balance: {balance} for address: {address} phone: {phone_number}')
    decimal_balance = self.balance_processor.get_decimal_amount(int(balance))
    tx_hash_hex, o = self.balance_processor.get_rpc_tx(blockchain_address, decimal_balance, serial)
    tx = unpack(bytes.fromhex(strip_0x(o)), self.chain_spec)
    logg.debug(f'generated tx token value: {decimal_balance}: {blockchain_address} tx hash {tx_hash_hex}')
    tx_path = os.path.join(self.import_dir, 'txs', strip_0x(tx_hash_hex))
    with open(tx_path, 'w') as tx_file:
        tx_file.write(strip_0x(o))
        logg.debug(f'written tx with tx hash: {tx["hash"]} for address: {blockchain_address}')
    tx_nonce_path = os.path.join(self.import_dir, 'txs', '.' + str(tx['nonce']))
    os.symlink(os.path.realpath(tx_path), tx_nonce_path)
    return tx['hash']


@celery_app.task(bind=True, base=MetadataTask)
def resolve_phone(self, phone_number: str):
    identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE)
    url = parse.urljoin(self.meta_url(), identifier)
    logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}')
    r = request.urlopen(url)
    address = json.load(r)
    address = address.replace('"', '')
    logg.debug(f'address: {address} for phone: {phone_number}')
    return address


@celery_app.task(autoretry_for=(FileNotFoundError,),
                 bind=True,
                 base=ImportTask,
                 max_retries=None,
                 default_retry_delay=0.1)
def send_txs(self, nonce):
    queue = self.request.delivery_info.get('routing_key')
    if nonce == self.count + self.balance_processor.nonce_offset:
        logg.info(f'reached nonce {nonce} (offset {self.balance_processor.nonce_offset} + count {self.count}).')
        celery_app.control.broadcast('shutdown', destination=[f'celery@{queue}'])

    logg.debug(f'attempt to open symlink for nonce {nonce}')
    tx_nonce_path = os.path.join(self.import_dir, 'txs', '.' + str(nonce))
    with open(tx_nonce_path, 'r') as tx_nonce_file:
        tx_signed_raw_hex = tx_nonce_file.read()
    os.unlink(tx_nonce_path)
    o = raw(add_0x(tx_signed_raw_hex))
    if self.include_balances:
        tx_hash_hex = self.balance_processor.conn.do(o)
        logg.info(f'sent nonce {nonce} tx hash {tx_hash_hex}')
    nonce += 1
    s = celery.signature('import_task.send_txs', [nonce], queue=queue)
    s.apply_async()
    return nonce


@celery_app.task()
def set_pin_data(config: dict, phone_to_pins: list):
    db_conn = psycopg2.connect(
        database=config.get('database'),
        host=config.get('host'),
        port=config.get('port'),
        user=config.get('user'),
        password=config.get('password')
    )
    db_cursor = db_conn.cursor()
    sql = 'UPDATE account SET password_hash = %s WHERE phone_number = %s'
    for element in phone_to_pins:
        db_cursor.execute(sql, (element[1], element[0]))
        logg.debug(f'Updating: {element[0]} with: {element[1]}')
    db_conn.commit()
    db_cursor.close()
    db_conn.close()


@celery_app.task
def set_ussd_data(config: dict, ussd_data: list):
    db_conn = psycopg2.connect(
        database=config.get('database'),
        host=config.get('host'),
        port=config.get('port'),
        user=config.get('user'),
        password=config.get('password')
    )
    db_cursor = db_conn.cursor()
    sql = 'UPDATE account SET status = %s, preferred_language = %s WHERE phone_number = %s'
    for element in ussd_data:
        status = 2 if int(element[1]) == 1 else 1
        preferred_language = element[2]
        phone_number = element[0]
        db_cursor.execute(sql, (status, preferred_language, phone_number))
    db_conn.commit()
    db_cursor.close()
    db_conn.close()