cic-internal-integration/apps/data-seeding/verify.py

563 lines
20 KiB
Python
Raw Normal View History

2021-02-21 16:41:37 +01:00
# standard imports
import argparse
2021-05-15 09:40:34 +02:00
import copy
2021-02-21 16:41:37 +01:00
import hashlib
import json
2021-05-15 09:40:34 +02:00
import logging
import os
import sys
2021-02-21 16:41:37 +01:00
import urllib
import urllib.request
2021-05-15 09:40:34 +02:00
import uuid
2021-06-23 06:29:38 +02:00
import urllib.parse
2021-02-21 16:41:37 +01:00
# external imports
2021-02-21 16:41:37 +01:00
import celery
import confini
from chainlib.chain import ChainSpec
2021-05-15 09:40:34 +02:00
from chainlib.eth.address import to_checksum_address
from chainlib.eth.connection import EthHTTPConnection
2021-02-21 16:41:37 +01:00
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.gas import (
2021-05-15 09:40:34 +02:00
OverrideGasOracle,
balance,
)
2021-02-21 16:41:37 +01:00
from chainlib.eth.tx import TxFactory
2021-05-15 09:40:34 +02:00
from chainlib.hash import keccak256_string_to_hex
2021-06-30 16:44:17 +02:00
from chainlib.jsonrpc import JSONRPCRequest
from cic_types.models.person import Person, identity_tag
from cic_types.condiments import MetadataPointer
from cic_types.processor import generate_metadata_pointer
2021-05-05 18:25:21 +02:00
from erc20_faucet import Faucet
from eth_erc20 import ERC20
2021-05-15 09:40:34 +02:00
from hexathon.parse import strip_0x, add_0x
2021-07-14 16:43:32 +02:00
from eth_contract_registry import Registry
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
2021-02-21 16:41:37 +01:00
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
script_dir = os.path.dirname(os.path.realpath(__file__))
base_config_dir = os.path.join(script_dir, 'config')
2021-02-21 16:41:37 +01:00
custodial_tests = [
'local_key',
'gas',
'faucet',
2021-07-14 16:43:32 +02:00
'ussd',
'ussd_pins',
]
metadata_tests = [
'metadata',
'metadata_phone',
]
eth_tests = [
'accounts_index',
'balance',
]
phone_tests = [
# 'ussd',
2021-05-15 09:40:34 +02:00
'ussd_pins'
]
2021-09-01 18:54:10 +02:00
admin_tests = [
'local_key',
]
cache_tests = [
'cache_tx_user',
]
all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests + cache_tests
2021-02-21 16:41:37 +01:00
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
argparser.add_argument('-c', type=str, help='config override dir')
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec')
2021-02-21 16:41:37 +01:00
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url')
argparser.add_argument('--cache-provider', type=str, dest='cache_provider', default='http://localhost:63313', help='cic-cache url')
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
2021-09-01 18:54:10 +02:00
argparser.add_argument('--skip-ussd', dest='skip_ussd', action='store_true', help='skip all ussd verifications')
2021-07-14 16:43:32 +02:00
argparser.add_argument('--skip-metadata', dest='skip_metadata', action='store_true', help='skip all metadata verifications')
argparser.add_argument('--skip-cache', dest='skip_cache', action='store_true', help='skip all cache verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--list-verifications', action='store_true', help='print a list of verification check identifiers')
2021-06-23 06:29:38 +02:00
argparser.add_argument('--token-symbol', default='GFT', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
2021-02-21 16:41:37 +01:00
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
2021-03-06 18:55:51 +01:00
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
2021-02-21 16:41:37 +01:00
argparser.add_argument('-v', help='be verbose', action='store_true')
argparser.add_argument('-vv', help='be more verbose', action='store_true')
argparser.add_argument('user_dir', type=str, help='user export directory')
args = argparser.parse_args(sys.argv[1:])
if args.v == True:
logging.getLogger().setLevel(logging.INFO)
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config = None
logg.debug('config dir {}'.format(base_config_dir))
if args.c != None:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
else:
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
2021-02-21 16:41:37 +01:00
config.process()
2021-02-21 16:41:37 +01:00
# override args
args_override = {
'CHAIN_SPEC': getattr(args, 'i'),
'RPC_PROVIDER': getattr(args, 'p'),
2021-02-21 16:41:37 +01:00
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True)
config.add(args.cache_provider, '_CACHE_PROVIDER', True)
2021-05-20 23:25:14 +02:00
token_symbol = args.token_symbol
2021-02-21 16:41:37 +01:00
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
2021-02-21 16:41:37 +01:00
chain_str = str(chain_spec)
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
old_chain_str = str(old_chain_spec)
user_dir = args.user_dir # user_out_dir from import_users.py
2021-03-06 18:55:51 +01:00
exit_on_error = args.x
active_tests = []
exclude = []
include = args.include
if args.include == None:
include = all_tests
for t in args.exclude:
if t not in all_tests:
raise ValueError('Cannot exclude unknown verification "{}"'.format(t))
exclude.append(t)
if args.skip_custodial:
logg.info('will skip all custodial verifications ({})'.format(','.join(custodial_tests)))
for t in custodial_tests:
if t not in exclude:
exclude.append(t)
2021-09-01 18:54:10 +02:00
if args.skip_ussd:
logg.info('will skip all ussd verifications ({})'.format(','.join(phone_tests)))
for t in phone_tests:
if t not in exclude:
exclude.append(t)
2021-07-14 16:43:32 +02:00
if args.skip_metadata:
logg.info('will skip all metadata verifications ({})'.format(','.join(metadata_tests)))
for t in metadata_tests:
if t not in exclude:
exclude.append(t)
for t in include:
if t not in all_tests:
raise ValueError('Cannot include unknown verification "{}"'.format(t))
if t not in exclude:
active_tests.append(t)
logg.info('will perform verification "{}"'.format(t))
api = None
for t in custodial_tests:
if t in active_tests:
2021-07-14 16:43:32 +02:00
from cic_eth.api.admin import AdminApi
api = AdminApi(None)
logg.info('activating custodial module'.format(t))
break
2021-09-21 08:04:44 +02:00
outfunc = logg.debug
2021-03-06 18:55:51 +01:00
2021-05-15 09:40:34 +02:00
def send_ussd_request(address, data_dir):
upper_address = strip_0x(address).upper()
f = open(os.path.join(
data_dir,
'new',
upper_address[:2],
upper_address[2:4],
upper_address + '.json',
), 'r'
)
o = json.load(f)
f.close()
p = Person.deserialize(o)
phone = p.tel
session = uuid.uuid4().hex
2021-08-29 11:55:47 +02:00
valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
2021-05-15 09:40:34 +02:00
data = {
'sessionId': session,
2021-08-29 11:55:47 +02:00
'serviceCode': valid_service_codes[0],
2021-05-15 09:40:34 +02:00
'phoneNumber': phone,
'text': '',
}
req = urllib.request.Request(config.get('_USSD_PROVIDER'))
2021-06-23 06:29:38 +02:00
urlencoded_data = urllib.parse.urlencode(data)
data_bytes = urlencoded_data.encode('utf-8')
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
2021-05-15 09:40:34 +02:00
req.data = data_bytes
response = urllib.request.urlopen(req)
return response.read().decode('utf-8')
2021-03-06 18:55:51 +01:00
class VerifierState:
2021-08-24 17:25:11 +02:00
def __init__(self, item_keys, target_count, active_tests=None):
2021-03-06 18:55:51 +01:00
self.items = {}
2021-08-24 17:25:11 +02:00
self.target_count = target_count
2021-03-06 18:55:51 +01:00
for k in item_keys:
self.items[k] = 0
if active_tests == None:
self.active_tests = copy.copy(item_keys)
else:
self.active_tests = copy.copy(active_tests)
2021-03-06 18:55:51 +01:00
def poke(self, item_key):
self.items[item_key] += 1
def __str__(self):
r = ''
for k in self.items.keys():
if k in self.active_tests:
2021-08-24 17:25:11 +02:00
r += '{}: {}/{}\n'.format(k, self.items[k], self.target_count)
else:
r += '{}: skipped\n'.format(k)
2021-03-06 18:55:51 +01:00
return r
2021-02-21 16:41:37 +01:00
class VerifierError(Exception):
def __init__(self, e, c):
super(VerifierError, self).__init__(e)
self.c = c
def __str__(self):
super_error = super(VerifierError, self).__str__()
return '[{}] {}'.format(self.c, super_error)
class Verifier:
2021-03-06 18:55:51 +01:00
# TODO: what an awful function signature
2021-08-24 17:25:11 +02:00
def __init__(self, conn, target_count, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
2021-02-21 16:41:37 +01:00
self.conn = conn
self.gas_oracle = gas_oracle
self.chain_spec = chain_spec
self.index_address = index_address
self.token_address = token_address
self.faucet_address = faucet_address
2021-04-04 14:40:59 +02:00
self.erc20_tx_factory = ERC20(chain_spec, gas_oracle=gas_oracle)
self.tx_factory = TxFactory(chain_spec, gas_oracle=gas_oracle)
2021-02-21 16:41:37 +01:00
self.api = cic_eth_api
self.data_dir = data_dir
2021-03-06 18:55:51 +01:00
self.exit_on_error = exit_on_error
2021-05-05 18:25:21 +02:00
self.faucet_tx_factory = Faucet(chain_spec, gas_oracle=gas_oracle)
2021-03-06 18:55:51 +01:00
verifymethods = []
for k in dir(self):
if len(k) > 7 and k[:7] == 'verify_':
logg.debug('verifier has verify method {}'.format(k))
2021-03-06 18:55:51 +01:00
verifymethods.append(k[7:])
o = self.faucet_tx_factory.token_amount(self.faucet_address, sender_address=ZERO_ADDRESS)
r = self.conn.do(o)
self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
2021-02-21 16:41:37 +01:00
2021-08-24 17:25:11 +02:00
self.state = VerifierState(verifymethods, target_count, active_tests=active_tests)
2021-02-21 16:41:37 +01:00
2021-03-06 18:55:51 +01:00
def verify_accounts_index(self, address, balance=None):
2021-07-14 16:43:32 +02:00
accounts_index = AccountsIndex(self.chain_spec)
o = accounts_index.have(self.index_address, address)
2021-02-21 16:41:37 +01:00
r = self.conn.do(o)
2021-07-14 16:43:32 +02:00
n = accounts_index.parse_have(r)
logg.debug('index check for {}: {}'.format(address, n))
2021-02-21 16:41:37 +01:00
if n != 1:
raise VerifierError(n, 'accounts index')
def verify_balance(self, address, balance):
o = self.erc20_tx_factory.balance(self.token_address, address)
2021-02-21 16:41:37 +01:00
r = self.conn.do(o)
2021-05-20 23:25:14 +02:00
try:
actual_balance = int(strip_0x(r), 16)
except ValueError:
actual_balance = int(r)
2021-03-01 21:15:17 +01:00
balance = int(balance / 1000000) * 1000000
balance += self.faucet_amount
logg.info('balance for {}: {}'.format(address, balance))
2021-02-21 16:41:37 +01:00
if balance != actual_balance:
raise VerifierError((actual_balance, balance), 'balance')
2021-03-06 18:55:51 +01:00
def verify_local_key(self, address, balance=None):
t = self.api.have_account(address, self.chain_spec)
r = t.get()
2021-02-21 16:41:37 +01:00
logg.debug('verify local key result {}'.format(r))
if r != address:
raise VerifierError((address, r), 'local key')
def verify_gas(self, address, balance_token=None):
o = balance(add_0x(address))
r = self.conn.do(o)
logg.debug('wtf {}'.format(r))
actual_balance = int(strip_0x(r), 16)
if actual_balance == 0:
raise VerifierError((address, actual_balance), 'gas')
def verify_faucet(self, address, balance_token=None):
o = self.faucet_tx_factory.usable_for(self.faucet_address, address)
r = self.conn.do(o)
if self.faucet_tx_factory.parse_usable_for(r):
raise VerifierError((address, r), 'faucet')
2021-03-06 18:55:51 +01:00
def verify_metadata(self, address, balance=None):
k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), MetadataPointer.PERSON)
url = os.path.join(config.get('_META_PROVIDER'), k)
2021-02-21 16:41:37 +01:00
logg.debug('verify metadata url {}'.format(url))
try:
res = urllib.request.urlopen(url)
except urllib.error.HTTPError as e:
raise VerifierError(
'({}) {}'.format(url, e),
'metadata (person)',
)
b = res.read()
o_retrieved = json.loads(b.decode('utf-8'))
upper_address = strip_0x(address).upper()
f = open(os.path.join(
self.data_dir,
'new',
upper_address[:2],
upper_address[2:4],
upper_address + '.json',
), 'r'
)
o_original = json.load(f)
f.close()
if o_original != o_retrieved:
raise VerifierError(o_retrieved, 'metadata (person)')
def verify_cache_tx_user(self, address, balance=None):
url = os.path.join(config.get('_CACHE_PROVIDER'), 'txa', 'user', address)
req = urllib.request.Request(url)
req.add_header('X_CIC_CACHE_MODE', 'all')
try:
res = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
raise VerifierError(
'({}) {}'.format(url, e),
'cache (tx user)',
)
r = json.load(res)
if len(r['data']) == 0:
raise VerifierError('empty tx list for address {}'.format(address), 'cache (tx user)')
for tx in r['data']:
logg.warning('found tx {} for {} but not checking validity'.format(tx['tx_hash'], address))
def verify_metadata_phone(self, address, balance=None):
upper_address = strip_0x(address).upper()
f = open(os.path.join(
self.data_dir,
'new',
upper_address[:2],
upper_address[2:4],
upper_address + '.json',
), 'r'
)
o = json.load(f)
f.close()
p = Person.deserialize(o)
k = generate_metadata_pointer(p.tel.encode('utf-8'), MetadataPointer.PHONE)
url = os.path.join(config.get('_META_PROVIDER'), k)
logg.debug('verify metadata phone url {}'.format(url))
try:
res = urllib.request.urlopen(url)
except urllib.error.HTTPError as e:
raise VerifierError(
'({}) {}'.format(url, e),
'metadata (phone)',
)
b = res.read()
address_recovered = json.loads(b.decode('utf-8'))
address_recovered = address_recovered.replace('"', '')
2021-04-14 10:17:47 +02:00
try:
2021-04-28 11:11:39 +02:00
upper_address_recovered = strip_0x(address_recovered).upper()
2021-04-14 10:17:47 +02:00
except ValueError:
raise VerifierError(address_recovered, 'metadata (phone) address {} address recovered {}'.format(address, address_recovered))
2021-04-28 11:11:39 +02:00
if upper_address != upper_address_recovered:
raise VerifierError(address_recovered, 'metadata (phone)')
def verify_ussd(self, address, balance=None):
2021-05-15 09:40:34 +02:00
response_data = send_ussd_request(address, self.data_dir)
state = response_data[:3]
out = response_data[4:]
m = '{} {}'.format(state, out[:7])
if m != 'CON Welcome':
raise VerifierError(response_data, 'ussd')
2021-05-15 09:40:34 +02:00
def verify_ussd_pins(self, address, balance):
response_data = send_ussd_request(address, self.data_dir)
2021-06-23 06:29:38 +02:00
if response_data[:11] != 'CON Balance' and response_data[:9] != 'CON Salio':
2021-05-15 09:40:34 +02:00
raise VerifierError(response_data, 'pins')
def verify(self, address, balance, debug_stem=None):
2021-03-06 18:55:51 +01:00
for k in active_tests:
s = '{} {}'.format(debug_stem, k)
outfunc(s)
2021-03-06 18:55:51 +01:00
try:
m = getattr(self, 'verify_{}'.format(k))
m(address, balance)
except VerifierError as e:
logline = 'verification {} failed for {}: {}'.format(k, address, str(e))
if self.exit_on_error:
logg.critical(logline)
sys.exit(1)
logg.error(logline)
self.state.poke(k)
def __str__(self):
return str(self.state)
2021-02-21 16:41:37 +01:00
def main():
global chain_str, block_offset, user_dir
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
2021-02-21 16:41:37 +01:00
# Get Token registry address
2021-07-14 16:43:32 +02:00
registry = Registry(chain_spec)
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'TokenRegistry')
2021-02-21 16:41:37 +01:00
r = conn.do(o)
2021-07-14 16:43:32 +02:00
token_index_address = registry.parse_address_of(r)
token_index_address = to_checksum_address(token_index_address)
2021-02-21 16:41:37 +01:00
logg.info('found token index address {}'.format(token_index_address))
2021-07-14 16:43:32 +02:00
# Get Account registry address
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'AccountRegistry')
2021-02-21 16:41:37 +01:00
r = conn.do(o)
2021-07-14 16:43:32 +02:00
account_index_address = registry.parse_address_of(r)
account_index_address = to_checksum_address(account_index_address)
2021-02-21 16:41:37 +01:00
logg.info('found account index address {}'.format(account_index_address))
2021-07-14 16:43:32 +02:00
# Get Faucet address
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'Faucet')
r = conn.do(o)
2021-07-14 16:43:32 +02:00
faucet_address = registry.parse_address_of(r)
faucet_index_address = to_checksum_address(token_index_address)
logg.info('found faucet {}'.format(faucet_address))
2021-07-14 16:43:32 +02:00
# Get Sarafu token address
token_index = TokenUniqueSymbolIndex(chain_spec)
o = token_index.address_of(token_index_address, token_symbol)
2021-02-21 16:41:37 +01:00
r = conn.do(o)
2021-07-14 16:43:32 +02:00
token_address = token_index.parse_address_of(r)
try:
token_address = to_checksum_address(token_address)
except ValueError as e:
logg.critical('lookup failed for token {}: {}'.format(token_symbol, e))
sys.exit(1)
logg.info('found token address {}'.format(token_address))
2021-02-21 16:41:37 +01:00
balances = {}
f = open('{}/balances.csv'.format(user_dir, 'r'))
i = 0
while True:
l = f.readline()
if l == None:
break
r = l.split(',')
try:
address = to_checksum_address(r[0])
2021-03-06 18:55:51 +01:00
#sys.stdout.write('loading balance {} {}'.format(i, address).ljust(200) + "\r")
outfunc('loading balance {} {}'.format(i, address)) #.ljust(200))
2021-02-21 16:41:37 +01:00
except ValueError:
break
balance = int(r[1].rstrip())
2021-02-21 16:41:37 +01:00
balances[address] = balance
i += 1
f.close()
2021-08-24 17:25:11 +02:00
verifier = Verifier(conn, i, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
2021-02-21 16:41:37 +01:00
user_new_dir = os.path.join(user_dir, 'new')
i = 0
2021-02-21 16:41:37 +01:00
for x in os.walk(user_new_dir):
for y in x[2]:
if y[len(y)-5:] != '.json':
continue
filepath = os.path.join(x[0], y)
f = open(filepath, 'r')
try:
o = json.load(f)
except json.decoder.JSONDecodeError as e:
f.close()
logg.error('load error for {}: {}'.format(y, e))
continue
f.close()
u = Person.deserialize(o)
#logg.debug('data {}'.format(u.identities['evm']))
2021-02-21 16:41:37 +01:00
new_chain_spec = chain_spec.asdict()
arch = new_chain_spec.get('arch')
fork = new_chain_spec.get('fork')
tag = identity_tag(new_chain_spec)
new_address = u.identities[arch][fork][tag][0]
old_chainspec = old_chain_spec.asdict()
arch = old_chainspec.get('arch')
fork = old_chainspec.get('fork')
tag = identity_tag(old_chainspec)
old_address = u.identities[arch][fork][tag][0]
2021-03-06 18:55:51 +01:00
balance = 0
try:
balance = balances[old_address]
except KeyError:
logg.info('no old balance found for {}, assuming 0'.format(old_address))
2021-02-21 16:41:37 +01:00
s = 'checking {}: {} -> {} = {}'.format(i, old_address, new_address, balance)
verifier.verify(new_address, balance, debug_stem=s)
i += 1
2021-02-21 16:41:37 +01:00
2021-08-24 17:25:11 +02:00
print()
2021-03-06 18:55:51 +01:00
print(verifier)
2021-02-21 16:41:37 +01:00
if __name__ == '__main__':
main()