Compare commits
6 Commits
master
...
lash/clean
Author | SHA1 | Date | |
---|---|---|---|
|
d9c06d0eb2 | ||
|
d21315b79b | ||
|
e966da6175 | ||
|
97b459d21e | ||
|
7dbb735fc1 | ||
|
b27fd5326f |
@ -88,7 +88,7 @@ run = True
|
|||||||
|
|
||||||
class DispatchSyncer:
|
class DispatchSyncer:
|
||||||
|
|
||||||
yield_delay = 0.005
|
yield_delay = 0.0005
|
||||||
|
|
||||||
def __init__(self, chain_spec):
|
def __init__(self, chain_spec):
|
||||||
self.chain_spec = chain_spec
|
self.chain_spec = chain_spec
|
||||||
|
@ -17,9 +17,7 @@ import random
|
|||||||
import vobject
|
import vobject
|
||||||
import celery
|
import celery
|
||||||
from faker import Faker
|
from faker import Faker
|
||||||
import cic_eth_registry
|
|
||||||
import confini
|
import confini
|
||||||
from cic_eth.api import Api
|
|
||||||
from cic_types.models.person import (
|
from cic_types.models.person import (
|
||||||
Person,
|
Person,
|
||||||
generate_vcard_from_contact_data,
|
generate_vcard_from_contact_data,
|
||||||
@ -62,8 +60,6 @@ ts_then = int(dt_then.timestamp())
|
|||||||
|
|
||||||
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
|
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
|
||||||
|
|
||||||
api = Api(config.get('CIC_CHAIN_SPEC'))
|
|
||||||
|
|
||||||
gift_max = args.gift_threshold or 0
|
gift_max = args.gift_threshold or 0
|
||||||
gift_factor = (10**6)
|
gift_factor = (10**6)
|
||||||
|
|
||||||
|
@ -23,7 +23,7 @@ from chainlib.eth.gas import RPCGasOracle
|
|||||||
from chainlib.eth.nonce import RPCNonceOracle
|
from chainlib.eth.nonce import RPCNonceOracle
|
||||||
from cic_types.processor import generate_metadata_pointer
|
from cic_types.processor import generate_metadata_pointer
|
||||||
from eth_accounts_index import AccountRegistry
|
from eth_accounts_index import AccountRegistry
|
||||||
from cic_eth_registry import CICRegistry
|
from contract_registry import Registry
|
||||||
from crypto_dev_signer.keystore.dict import DictKeystore
|
from crypto_dev_signer.keystore.dict import DictKeystore
|
||||||
from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer
|
from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer
|
||||||
from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
|
from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
|
||||||
@ -88,9 +88,11 @@ signer = EIP155Signer(keystore)
|
|||||||
|
|
||||||
nonce_oracle = RPCNonceOracle(signer_address, rpc)
|
nonce_oracle = RPCNonceOracle(signer_address, rpc)
|
||||||
|
|
||||||
CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
|
registry = Registry()
|
||||||
registry = CICRegistry(chain_spec, rpc)
|
o = registry.address_of(config.get('CIC_REGISTRY_ADDRESS'), 'AccountRegistry')
|
||||||
account_registry_address = registry.by_name('AccountRegistry')
|
r = rpc.do(o)
|
||||||
|
account_registry_address = registry.parse_address_of(r)
|
||||||
|
logg.info('using account registry {}'.format(account_registry_address))
|
||||||
|
|
||||||
keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
|
keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
|
||||||
os.makedirs(keyfile_dir)
|
os.makedirs(keyfile_dir)
|
||||||
|
@ -10,6 +10,7 @@ import hashlib
|
|||||||
import csv
|
import csv
|
||||||
import json
|
import json
|
||||||
import urllib
|
import urllib
|
||||||
|
import copy
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import celery
|
import celery
|
||||||
@ -39,7 +40,6 @@ from chainlib.eth.gas import (
|
|||||||
from chainlib.eth.tx import TxFactory
|
from chainlib.eth.tx import TxFactory
|
||||||
from chainlib.eth.rpc import jsonrpc_template
|
from chainlib.eth.rpc import jsonrpc_template
|
||||||
from chainlib.eth.error import EthException
|
from chainlib.eth.error import EthException
|
||||||
from cic_eth.api.api_admin import AdminApi
|
|
||||||
from cic_types.models.person import (
|
from cic_types.models.person import (
|
||||||
Person,
|
Person,
|
||||||
generate_metadata_pointer,
|
generate_metadata_pointer,
|
||||||
@ -51,12 +51,27 @@ logg = logging.getLogger()
|
|||||||
|
|
||||||
config_dir = '/usr/local/etc/cic-syncer'
|
config_dir = '/usr/local/etc/cic-syncer'
|
||||||
|
|
||||||
|
custodial_tests = [
|
||||||
|
'local_key',
|
||||||
|
'gas',
|
||||||
|
'faucet',
|
||||||
|
]
|
||||||
|
|
||||||
|
all_tests = custodial_tests + [
|
||||||
|
'accounts_index',
|
||||||
|
'balance',
|
||||||
|
'metadata',
|
||||||
|
]
|
||||||
|
|
||||||
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
|
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
|
||||||
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
|
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
|
||||||
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
|
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
|
||||||
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
|
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
|
||||||
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
|
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
|
||||||
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
|
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
|
||||||
|
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
|
||||||
|
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
|
||||||
|
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
|
||||||
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
|
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
|
||||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||||
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
|
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
|
||||||
@ -95,14 +110,61 @@ user_dir = args.user_dir # user_out_dir from import_users.py
|
|||||||
meta_url = args.meta_provider
|
meta_url = args.meta_provider
|
||||||
exit_on_error = args.x
|
exit_on_error = args.x
|
||||||
|
|
||||||
|
active_tests = []
|
||||||
|
exclude = []
|
||||||
|
include = args.include
|
||||||
|
if args.include == None:
|
||||||
|
include = all_tests
|
||||||
|
for t in args.exclude:
|
||||||
|
if t not in all_tests:
|
||||||
|
raise ValueError('Cannot exclude unknown verification "{}"'.format(t))
|
||||||
|
exclude.append(t)
|
||||||
|
if args.skip_custodial:
|
||||||
|
logg.info('will skip all custodial verifications ({})'.format(','.join(custodial_tests)))
|
||||||
|
for t in custodial_tests:
|
||||||
|
if t not in exclude:
|
||||||
|
exclude.append(t)
|
||||||
|
for t in include:
|
||||||
|
if t not in all_tests:
|
||||||
|
raise ValueError('Cannot include unknown verification "{}"'.format(t))
|
||||||
|
if t not in exclude:
|
||||||
|
active_tests.append(t)
|
||||||
|
logg.info('will perform verification "{}"'.format(t))
|
||||||
|
|
||||||
|
api = None
|
||||||
|
for t in custodial_tests:
|
||||||
|
if t in active_tests:
|
||||||
|
from cic_eth.api.api_admin import AdminApi
|
||||||
|
api = AdminApi(None)
|
||||||
|
logg.info('activating custodial module'.format(t))
|
||||||
|
break
|
||||||
|
|
||||||
|
cols = os.get_terminal_size().columns
|
||||||
|
|
||||||
|
|
||||||
|
def to_terminalwidth(s):
|
||||||
|
ss = s.ljust(int(cols)-1)
|
||||||
|
ss += "\r"
|
||||||
|
return ss
|
||||||
|
|
||||||
|
def default_outfunc(s):
|
||||||
|
ss = to_terminalwidth(s)
|
||||||
|
sys.stdout.write(ss)
|
||||||
|
outfunc = default_outfunc
|
||||||
|
if logg.isEnabledFor(logging.DEBUG):
|
||||||
|
outfunc = logg.debug
|
||||||
|
|
||||||
|
|
||||||
class VerifierState:
|
class VerifierState:
|
||||||
|
|
||||||
def __init__(self, item_keys):
|
def __init__(self, item_keys, active_tests=None):
|
||||||
self.items = {}
|
self.items = {}
|
||||||
for k in item_keys:
|
for k in item_keys:
|
||||||
logg.info('k {}'.format(k))
|
|
||||||
self.items[k] = 0
|
self.items[k] = 0
|
||||||
|
if active_tests == None:
|
||||||
|
self.active_tests = copy.copy(item_keys)
|
||||||
|
else:
|
||||||
|
self.active_tests = copy.copy(active_tests)
|
||||||
|
|
||||||
|
|
||||||
def poke(self, item_key):
|
def poke(self, item_key):
|
||||||
@ -112,7 +174,10 @@ class VerifierState:
|
|||||||
def __str__(self):
|
def __str__(self):
|
||||||
r = ''
|
r = ''
|
||||||
for k in self.items.keys():
|
for k in self.items.keys():
|
||||||
|
if k in self.active_tests:
|
||||||
r += '{}: {}\n'.format(k, self.items[k])
|
r += '{}: {}\n'.format(k, self.items[k])
|
||||||
|
else:
|
||||||
|
r += '{}: skipped\n'.format(k)
|
||||||
return r
|
return r
|
||||||
|
|
||||||
|
|
||||||
@ -148,10 +213,10 @@ class Verifier:
|
|||||||
verifymethods = []
|
verifymethods = []
|
||||||
for k in dir(self):
|
for k in dir(self):
|
||||||
if len(k) > 7 and k[:7] == 'verify_':
|
if len(k) > 7 and k[:7] == 'verify_':
|
||||||
logg.info('adding verify method {}'.format(k))
|
logg.debug('verifier has verify method {}'.format(k))
|
||||||
verifymethods.append(k[7:])
|
verifymethods.append(k[7:])
|
||||||
|
|
||||||
self.state = VerifierState(verifymethods)
|
self.state = VerifierState(verifymethods, active_tests=active_tests)
|
||||||
|
|
||||||
|
|
||||||
def verify_accounts_index(self, address, balance=None):
|
def verify_accounts_index(self, address, balance=None):
|
||||||
@ -233,26 +298,14 @@ class Verifier:
|
|||||||
raise VerifierError(o_retrieved, 'metadata (person)')
|
raise VerifierError(o_retrieved, 'metadata (person)')
|
||||||
|
|
||||||
|
|
||||||
def verify(self, address, balance):
|
def verify(self, address, balance, debug_stem=None):
|
||||||
logg.debug('verify {} {}'.format(address, balance))
|
|
||||||
|
|
||||||
methods = [
|
for k in active_tests:
|
||||||
'local_key',
|
s = '{} {}'.format(debug_stem, k)
|
||||||
'accounts_index',
|
outfunc(s)
|
||||||
'balance',
|
|
||||||
'metadata',
|
|
||||||
'gas',
|
|
||||||
'faucet',
|
|
||||||
]
|
|
||||||
|
|
||||||
for k in methods:
|
|
||||||
try:
|
try:
|
||||||
m = getattr(self, 'verify_{}'.format(k))
|
m = getattr(self, 'verify_{}'.format(k))
|
||||||
m(address, balance)
|
m(address, balance)
|
||||||
# self.verify_local_key(address)
|
|
||||||
# self.verify_accounts_index(address)
|
|
||||||
# self.verify_balance(address, balance)
|
|
||||||
# self.verify_metadata(address)
|
|
||||||
except VerifierError as e:
|
except VerifierError as e:
|
||||||
logline = 'verification {} failed for {}: {}'.format(k, address, str(e))
|
logline = 'verification {} failed for {}: {}'.format(k, address, str(e))
|
||||||
if self.exit_on_error:
|
if self.exit_on_error:
|
||||||
@ -266,10 +319,6 @@ class Verifier:
|
|||||||
return str(self.state)
|
return str(self.state)
|
||||||
|
|
||||||
|
|
||||||
class MockClient:
|
|
||||||
|
|
||||||
w3 = None
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
global chain_str, block_offset, user_dir
|
global chain_str, block_offset, user_dir
|
||||||
|
|
||||||
@ -291,7 +340,6 @@ def main():
|
|||||||
o['params'].append(txf.normalize(tx))
|
o['params'].append(txf.normalize(tx))
|
||||||
o['params'].append('latest')
|
o['params'].append('latest')
|
||||||
r = conn.do(o)
|
r = conn.do(o)
|
||||||
print('r {}'.format(r))
|
|
||||||
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
|
token_index_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
|
||||||
logg.info('found token index address {}'.format(token_index_address))
|
logg.info('found token index address {}'.format(token_index_address))
|
||||||
|
|
||||||
@ -320,6 +368,7 @@ def main():
|
|||||||
logg.info('found faucet {}'.format(faucet_address))
|
logg.info('found faucet {}'.format(faucet_address))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# Get Sarafu token address
|
# Get Sarafu token address
|
||||||
tx = txf.template(ZERO_ADDRESS, token_index_address)
|
tx = txf.template(ZERO_ADDRESS, token_index_address)
|
||||||
data = add_0x(registry_addressof_method)
|
data = add_0x(registry_addressof_method)
|
||||||
@ -333,7 +382,6 @@ def main():
|
|||||||
o['params'].append(txf.normalize(tx))
|
o['params'].append(txf.normalize(tx))
|
||||||
o['params'].append('latest')
|
o['params'].append('latest')
|
||||||
r = conn.do(o)
|
r = conn.do(o)
|
||||||
print('r {}'.format(r))
|
|
||||||
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
|
sarafu_token_address = to_checksum_address(eth_abi.decode_single('address', bytes.fromhex(strip_0x(r))))
|
||||||
logg.info('found token address {}'.format(sarafu_token_address))
|
logg.info('found token address {}'.format(sarafu_token_address))
|
||||||
|
|
||||||
@ -348,7 +396,7 @@ def main():
|
|||||||
try:
|
try:
|
||||||
address = to_checksum_address(r[0])
|
address = to_checksum_address(r[0])
|
||||||
#sys.stdout.write('loading balance {} {}'.format(i, address).ljust(200) + "\r")
|
#sys.stdout.write('loading balance {} {}'.format(i, address).ljust(200) + "\r")
|
||||||
logg.debug('loading balance {} {}'.format(i, address).ljust(200))
|
outfunc('loading balance {} {}'.format(i, address)) #.ljust(200))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
break
|
break
|
||||||
balance = int(r[1].rstrip())
|
balance = int(r[1].rstrip())
|
||||||
@ -357,11 +405,10 @@ def main():
|
|||||||
|
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
api = AdminApi(MockClient())
|
|
||||||
|
|
||||||
verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, faucet_address, user_dir, exit_on_error)
|
verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, sarafu_token_address, faucet_address, user_dir, exit_on_error)
|
||||||
|
|
||||||
user_new_dir = os.path.join(user_dir, 'new')
|
user_new_dir = os.path.join(user_dir, 'new')
|
||||||
|
i = 0
|
||||||
for x in os.walk(user_new_dir):
|
for x in os.walk(user_new_dir):
|
||||||
for y in x[2]:
|
for y in x[2]:
|
||||||
if y[len(y)-5:] != '.json':
|
if y[len(y)-5:] != '.json':
|
||||||
@ -377,7 +424,7 @@ def main():
|
|||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
u = Person.deserialize(o)
|
u = Person.deserialize(o)
|
||||||
logg.debug('data {}'.format(u.identities['evm']))
|
#logg.debug('data {}'.format(u.identities['evm']))
|
||||||
|
|
||||||
subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
|
subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id())
|
||||||
new_address = u.identities['evm'][subchain_str][0]
|
new_address = u.identities['evm'][subchain_str][0]
|
||||||
@ -388,9 +435,11 @@ def main():
|
|||||||
balance = balances[old_address]
|
balance = balances[old_address]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
logg.info('no old balance found for {}, assuming 0'.format(old_address))
|
logg.info('no old balance found for {}, assuming 0'.format(old_address))
|
||||||
logg.debug('checking {} -> {} = {}'.format(old_address, new_address, balance))
|
|
||||||
|
|
||||||
verifier.verify(new_address, balance)
|
s = 'checking {}: {} -> {} = {}'.format(i, old_address, new_address, balance)
|
||||||
|
|
||||||
|
verifier.verify(new_address, balance, debug_stem=s)
|
||||||
|
i += 1
|
||||||
|
|
||||||
print(verifier)
|
print(verifier)
|
||||||
|
|
||||||
|
@ -373,7 +373,7 @@ services:
|
|||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
|
if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
|
||||||
./start_retry.sh -v
|
./start_retry.sh -vv
|
||||||
# command: "/root/start_retry.sh -q cic-eth -vv"
|
# command: "/root/start_retry.sh -q cic-eth -vv"
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user