Add data seeding preparation step, rehabilitation of non-custodial seeding
This commit is contained in:
nolash
parent
ea3c75e755
commit
677fb346fd
GPG Key ID:
21D2E7BB88C2A746
|
|
@ -4,9 +4,9 @@ billiard==3.6.4.0
|
|||
bcrypt==3.2.0
|
||||
celery==4.4.7
|
||||
cffi==1.14.6
|
||||
cic-eth[services]~=0.12.4a11
|
||||
cic-notify~=0.4.0a10
|
||||
cic-types~=0.2.0a3
|
||||
#cic-eth[services]~=0.12.4a11
|
||||
#cic-notify~=0.4.0a10
|
||||
#cic-types~=0.2.0a3
|
||||
confini>=0.3.6rc4,<0.5.0
|
||||
phonenumbers==8.12.12
|
||||
psycopg2==2.8.6
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $accounts_index_writer
|
|||
|
||||
|
||||
# Assign system writer for accounts index
|
||||
>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to writer to accounts index contract at $accounts_index_address\033[;39m"
|
||||
>&2 echo -e "\033[;96mEnable accounts index writer $accounts_index_writer to write to accounts index contract at $accounts_index_address\033[;39m"
|
||||
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $accounts_index_writer`
|
||||
add_pending_tx_hash $r
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
. util.sh
|
||||
|
||||
set -a
|
||||
|
||||
. ${DEV_DATA_DIR}/env_reset
|
||||
|
||||
WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60}
|
||||
|
||||
set -e
|
||||
|
||||
if [ ! -z $DEV_ETH_GAS_PRICE ]; then
|
||||
gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE"
|
||||
fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE"
|
||||
fi
|
||||
|
||||
must_address "$CIC_REGISTRY_ADDRESS" "registry"
|
||||
must_eth_rpc
|
||||
|
||||
|
||||
accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`
|
||||
|
||||
|
||||
>&2 echo -e "\033[;96mEnable default wallet $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER to write to accounts index contract at $accounts_index_address\033[;39m"
|
||||
r=`eth-accounts-index-writer -s -w -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $accounts_index_address $DEV_DEBUG_FLAG $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
|
||||
add_pending_tx_hash $r
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
|
||||
ARG DEV_DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
|
||||
|
||||
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
|
||||
FROM $DEV_DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@ unset CONFINI_DIR
|
|||
|
||||
set +a
|
||||
|
||||
LAST_BIT_POS=4
|
||||
files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial)
|
||||
description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine")
|
||||
LAST_BIT_POS=5
|
||||
files=(deploy_contract_root deploy_contract_instance deploy_token init_custodial data_seeding)
|
||||
description=("global contracts" "instance specific contracts" "token deployment" "initialize custodial engine" "data seeding for development")
|
||||
|
||||
>&2 echo -e "\033[;96mRUNNING configurations\033[;39m"
|
||||
source ./config.sh
|
||||
|
|
|
|||
|
|
@ -47,12 +47,15 @@ from erc20_faucet import Faucet
|
|||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
config_dir = './config'
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.dirname(script_dir)
|
||||
base_config_dir = os.path.join(root_dir, 'config')
|
||||
|
||||
|
||||
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
|
||||
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
|
||||
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
|
||||
argparser.add_argument('-c', type=str, help='config override directory')
|
||||
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
|
||||
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
|
||||
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
|
||||
|
|
@ -71,21 +74,24 @@ if args.v == True:
|
|||
elif args.vv == True:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
config_dir = os.path.join(args.c)
|
||||
os.makedirs(config_dir, 0o777, True)
|
||||
config = confini.Config(config_dir, args.env_prefix)
|
||||
config = None
|
||||
logg.debug('config dir {}'.format(base_config_dir))
|
||||
if args.c != None:
|
||||
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
|
||||
else:
|
||||
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
|
||||
config.process()
|
||||
|
||||
# override args
|
||||
args_override = {
|
||||
'CIC_CHAIN_SPEC': getattr(args, 'i'),
|
||||
'ETH_PROVIDER': getattr(args, 'p'),
|
||||
'CHAIN_SPEC': getattr(args, 'i'),
|
||||
'RPC_PROVIDER': getattr(args, 'p'),
|
||||
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
|
||||
'KEYSTORE_FILE_PATH': getattr(args, 'y')
|
||||
}
|
||||
config.dict_override(args_override, 'cli flag')
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
|
||||
|
||||
#app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
|
||||
|
||||
|
|
@ -98,7 +104,7 @@ if args.y != None:
|
|||
signer = EIP155Signer(keystore)
|
||||
|
||||
queue = args.q
|
||||
chain_str = config.get('CIC_CHAIN_SPEC')
|
||||
chain_str = config.get('CHAIN_SPEC')
|
||||
block_offset = 0
|
||||
if args.head:
|
||||
block_offset = -1
|
||||
|
|
@ -203,7 +209,7 @@ def progress_callback(block_number, tx_index):
|
|||
def main():
|
||||
global chain_str, block_offset, user_dir
|
||||
|
||||
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
|
||||
conn = EthHTTPConnection(config.get('RPC_PROVIDER'))
|
||||
gas_oracle = OverrideGasOracle(conn=conn, limit=8000000)
|
||||
nonce_oracle = RPCNonceOracle(signer_address, conn)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import uuid
|
|||
import datetime
|
||||
import time
|
||||
import phonenumbers
|
||||
import shutil
|
||||
from glob import glob
|
||||
|
||||
# external imports
|
||||
|
|
@ -23,6 +24,7 @@ from chainlib.eth.connection import EthHTTPConnection
|
|||
from chainlib.eth.gas import RPCGasOracle
|
||||
from chainlib.eth.nonce import RPCNonceOracle
|
||||
from cic_types.processor import generate_metadata_pointer
|
||||
from cic_types import MetadataPointer
|
||||
from eth_accounts_index.registry import AccountRegistry
|
||||
from eth_contract_registry import Registry
|
||||
from crypto_dev_signer.keystore.dict import DictKeystore
|
||||
|
|
@ -32,12 +34,15 @@ from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict
|
|||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
default_config_dir = '/usr/local/etc/cic'
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.dirname(script_dir)
|
||||
base_config_dir = os.path.join(root_dir, 'config')
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
|
||||
argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
|
||||
argparser.add_argument('-c', type=str, default=default_config_dir, help='config file')
|
||||
argparser.add_argument('-c', type=str, help='config override directory')
|
||||
argparser.add_argument('-f', action='store_true', help='force clear previous state')
|
||||
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
|
||||
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string')
|
||||
argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address')
|
||||
|
|
@ -53,8 +58,11 @@ if args.v:
|
|||
elif args.vv:
|
||||
logg.setLevel(logging.DEBUG)
|
||||
|
||||
config_dir = args.c
|
||||
config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
|
||||
config = None
|
||||
if args.c != None:
|
||||
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'), override_config_dir=args.c)
|
||||
else:
|
||||
config = confini.Config(base_config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
|
||||
config.process()
|
||||
args_override = {
|
||||
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
|
||||
|
|
@ -64,25 +72,26 @@ args_override = {
|
|||
config.dict_override(args_override, 'cli')
|
||||
config.add(args.user_dir, '_USERDIR', True)
|
||||
|
||||
user_new_dir = os.path.join(args.user_dir, 'new')
|
||||
os.makedirs(user_new_dir)
|
||||
|
||||
meta_dir = os.path.join(args.user_dir, 'meta')
|
||||
os.makedirs(meta_dir)
|
||||
|
||||
custom_dir = os.path.join(args.user_dir, 'custom')
|
||||
os.makedirs(custom_dir)
|
||||
os.makedirs(os.path.join(custom_dir, 'new'))
|
||||
os.makedirs(os.path.join(custom_dir, 'meta'))
|
||||
|
||||
phone_dir = os.path.join(args.user_dir, 'phone')
|
||||
os.makedirs(os.path.join(phone_dir, 'meta'))
|
||||
|
||||
user_old_dir = os.path.join(args.user_dir, 'old')
|
||||
os.stat(user_old_dir)
|
||||
|
||||
user_new_dir = os.path.join(args.user_dir, 'new')
|
||||
meta_dir = os.path.join(args.user_dir, 'meta')
|
||||
custom_dir = os.path.join(args.user_dir, 'custom')
|
||||
phone_dir = os.path.join(args.user_dir, 'phone')
|
||||
txs_dir = os.path.join(args.user_dir, 'txs')
|
||||
os.makedirs(txs_dir)
|
||||
|
||||
try:
|
||||
os.stat(user_old_dir)
|
||||
except FileNotFoundError:
|
||||
sys.stderr.write('no users to import. please run create_import_users.py first\n')
|
||||
sys.exit(1)
|
||||
|
||||
#os.makedirs(user_new_dir)
|
||||
#os.makedirs(meta_dir)
|
||||
#os.makedirs(custom_dir)
|
||||
#os.makedirs(os.path.join(custom_dir, 'new'))
|
||||
#os.makedirs(os.path.join(custom_dir, 'meta'))
|
||||
#os.makedirs(os.path.join(phone_dir, 'meta'))
|
||||
#os.makedirs(txs_dir)
|
||||
|
||||
user_dir = args.user_dir
|
||||
|
||||
|
|
@ -114,7 +123,26 @@ account_registry_address = registry.parse_address_of(r)
|
|||
logg.info('using account registry {}'.format(account_registry_address))
|
||||
|
||||
keyfile_dir = os.path.join(config.get('_USERDIR'), 'keystore')
|
||||
os.makedirs(keyfile_dir)
|
||||
|
||||
result_dirs = [
|
||||
user_new_dir,
|
||||
meta_dir,
|
||||
custom_dir,
|
||||
os.path.join(custom_dir, 'new'),
|
||||
os.path.join(custom_dir, 'meta'),
|
||||
os.path.join(phone_dir, 'meta'),
|
||||
txs_dir,
|
||||
keyfile_dir,
|
||||
]
|
||||
if args.f:
|
||||
for d in result_dirs:
|
||||
try:
|
||||
shutil.rmtree(d)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
for d in result_dirs:
|
||||
os.makedirs(d)
|
||||
|
||||
|
||||
def register_eth(i, u):
|
||||
|
||||
|
|
@ -190,14 +218,14 @@ if __name__ == '__main__':
|
|||
f.write(json.dumps(o))
|
||||
f.close()
|
||||
|
||||
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
|
||||
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), MetadataPointer.PERSON)
|
||||
meta_filepath = os.path.join(meta_dir, '{}.json'.format(new_address_clean.upper()))
|
||||
os.symlink(os.path.realpath(filepath), meta_filepath)
|
||||
|
||||
phone_object = phonenumbers.parse(u.tel)
|
||||
phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164)
|
||||
logg.debug('>>>>> Using phone {}'.format(phone))
|
||||
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
|
||||
meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE)
|
||||
meta_phone_filepath = os.path.join(phone_dir, 'meta', meta_phone_key)
|
||||
|
||||
filepath = os.path.join(
|
||||
|
|
@ -217,7 +245,7 @@ if __name__ == '__main__':
|
|||
|
||||
|
||||
# custom data
|
||||
custom_key = generate_metadata_pointer(phone.encode('utf-8'), ':cic.custom')
|
||||
custom_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.CUSTOM)
|
||||
custom_filepath = os.path.join(custom_dir, 'meta', custom_key)
|
||||
|
||||
filepath = os.path.join(
|
||||
|
|
|
|||
|
|
@ -12,3 +12,4 @@ eth-erc20~=0.1.2a3
|
|||
erc20-faucet~=0.3.2a2
|
||||
psycopg2==2.8.6
|
||||
liveness~=0.0.1a7
|
||||
confini>=0.4.2rc3,<0.5.0
|
||||
|
|
|
|||
|
|
@ -39,7 +39,8 @@ from eth_token_index import TokenUniqueSymbolIndex
|
|||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
config_dir = '/usr/local/etc/cic-syncer'
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
base_config_dir = os.path.join(script_dir, 'config')
|
||||
|
||||
custodial_tests = [
|
||||
'local_key',
|
||||
|
|
@ -72,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests
|
|||
|
||||
argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
|
||||
argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address')
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
|
||||
argparser.add_argument('-c', type=str, help='config override dir')
|
||||
argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec')
|
||||
argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
|
||||
argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url')
|
||||
|
|
@ -96,13 +97,17 @@ if args.v == True:
|
|||
elif args.vv == True:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
config_dir = os.path.join(args.c)
|
||||
os.makedirs(config_dir, 0o777, True)
|
||||
config = confini.Config(config_dir, args.env_prefix)
|
||||
config = None
|
||||
logg.debug('config dir {}'.format(base_config_dir))
|
||||
if args.c != None:
|
||||
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'), override_dirs=args.c)
|
||||
else:
|
||||
config = confini.Config(base_config_dir, env_prefix=os.environ.get('CONFINI_ENV_PREFIX'))
|
||||
config.process()
|
||||
|
||||
# override args
|
||||
args_override = {
|
||||
'CIC_CHAIN_SPEC': getattr(args, 'i'),
|
||||
'CHAIN_SPEC': getattr(args, 'i'),
|
||||
'ETH_PROVIDER': getattr(args, 'p'),
|
||||
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
|
||||
}
|
||||
|
|
@ -114,11 +119,9 @@ config.add(args.ussd_provider, '_USSD_PROVIDER', True)
|
|||
|
||||
token_symbol = args.token_symbol
|
||||
|
||||
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
|
||||
|
||||
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
chain_str = str(chain_spec)
|
||||
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
|
||||
old_chain_str = str(old_chain_spec)
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ services:
|
|||
args:
|
||||
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
|
||||
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
|
||||
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
|
||||
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
|
||||
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
|
||||
environment:
|
||||
DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
|
||||
|
|
|
|||
Loading…
Reference in New Issue