WIP Add queue recover runnable in cic-eth
This commit is contained in:
nolash
parent
d01bf3cce0
commit
bbf38a5fe5
GPG Key ID:
21D2E7BB88C2A746
@ -58,6 +58,7 @@ class SessionBase(Model):
|
||||
def connect(dsn, pool_size=16, debug=False):
|
||||
"""Create new database connection engine and connect to database backend.
|
||||
|
||||
|
||||
:param dsn: DSN string defining connection.
|
||||
:type dsn: str
|
||||
"""
|
||||
|
||||
153
apps/cic-eth/cic_eth/runnable/recovery/rebuild.py
Normal file
153
apps/cic-eth/cic_eth/runnable/recovery/rebuild.py
Normal file
@ -0,0 +1,153 @@
|
||||
# standard imports
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
import argparse
|
||||
import tempfile
|
||||
import re
|
||||
import urllib
|
||||
import websocket
|
||||
import stat
|
||||
import importlib
|
||||
|
||||
# external imports
|
||||
from chainlib.connection import (
|
||||
RPCConnection,
|
||||
ConnType,
|
||||
)
|
||||
from chainlib.eth.address import to_checksum_address
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainqueue.db.models.otx import Otx
|
||||
from eth_accounts_index import AccountsIndex
|
||||
from moolb import Bloom
|
||||
from hexathon import strip_0x
|
||||
from chainlib.eth.block import (
|
||||
block_latest,
|
||||
)
|
||||
from chainsyncer.backend.sql import SQLBackend
|
||||
from chainsyncer.db.models.base import SessionBase
|
||||
from chainsyncer.driver.head import HeadSyncer
|
||||
from chainsyncer.driver.history import HistorySyncer
|
||||
|
||||
# local imports
|
||||
from cic_eth_registry.error import UnknownContractError
|
||||
from cic_eth_registry.erc20 import ERC20Token
|
||||
from cic_eth.registry import (
|
||||
connect as connect_registry,
|
||||
)
|
||||
import cic_eth.cli
|
||||
from cic_eth.db import dsn_from_config
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
arg_flags = cic_eth.cli.argflag_std_read
|
||||
local_arg_flags = cic_eth.cli.argflag_local_sync
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
# set up rpc
|
||||
rpc = cic_eth.cli.RPC.from_config(config)
|
||||
conn = rpc.get_default()
|
||||
|
||||
dsn = dsn_from_config(config)
|
||||
SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
|
||||
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
|
||||
registry = None
|
||||
try:
|
||||
registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
except UnknownContractError as e:
|
||||
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
|
||||
sys.exit(1)
|
||||
logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
|
||||
|
||||
|
||||
class QueueRecoveryFilter:
|
||||
|
||||
def __init__(self, chain_spec, rpc, account_registry_address):
|
||||
self.account_registry_address = account_registry_address
|
||||
self.account_registry = AccountsIndex(chain_spec)
|
||||
self.account_count = 0
|
||||
self.bloom = None
|
||||
self.rpc = rpc
|
||||
|
||||
|
||||
def load(self):
|
||||
o = self.account_registry.count(self.account_registry_address)
|
||||
r = self.rpc.do(o)
|
||||
self.account_count = self.account_registry.parse_entry_count(r)
|
||||
logg.debug('loading account registry {} with {} entries'.format(self.account_registry_address, self.account_count))
|
||||
|
||||
self.bloom = Bloom(8192, 3)
|
||||
|
||||
for i in range(self.account_count):
|
||||
o = self.account_registry.entry(self.account_registry_address, i)
|
||||
r = self.rpc.do(o)
|
||||
entry = self.account_registry.parse_account(r)
|
||||
entry_bytes = bytes.fromhex(strip_0x(entry))
|
||||
self.bloom.add(entry_bytes)
|
||||
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, db_session=None):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
o = block_latest()
|
||||
r = conn.do(o)
|
||||
block_current = int(r, 16)
|
||||
block_offset = block_current + 1
|
||||
|
||||
loop_interval = config.get('SYNCER_LOOP_INTERVAL')
|
||||
if loop_interval == None:
|
||||
stat = init_chain_stat(conn, block_start=block_current)
|
||||
loop_interval = stat.block_average()
|
||||
|
||||
logg.debug('current block height {}'.format(block_offset))
|
||||
|
||||
syncers = []
|
||||
|
||||
#if SQLBackend.first(chain_spec):
|
||||
# backend = SQLBackend.initial(chain_spec, block_offset)
|
||||
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
||||
|
||||
if len(syncer_backends) == 0:
|
||||
initial_block_start = config.get('SYNCER_OFFSET')
|
||||
initial_block_offset = block_offset
|
||||
if config.true('SYNCER_NO_HISTORY'):
|
||||
initial_block_start = block_offset
|
||||
initial_block_offset += 1
|
||||
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
||||
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
||||
else:
|
||||
for syncer_backend in syncer_backends:
|
||||
logg.info('resuming sync session {}'.format(syncer_backend))
|
||||
|
||||
syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1))
|
||||
|
||||
for syncer_backend in syncer_backends:
|
||||
try:
|
||||
syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface))
|
||||
logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
|
||||
except AttributeError:
|
||||
logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
|
||||
syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface))
|
||||
|
||||
account_registry_address = registry.by_name('AccountRegistry')
|
||||
account_filter = QueueRecoveryFilter(chain_spec, conn, account_registry_address)
|
||||
account_filter.load()
|
||||
|
||||
for syncer in syncers:
|
||||
syncer.add_filter(account_filter)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
0
apps/contract-migration/config.sh
Normal file → Executable file
0
apps/contract-migration/config.sh
Normal file → Executable file
@ -39,8 +39,8 @@ if [ "$TOKEN_TYPE" == "giftable_erc20_token" ]; then
|
||||
TOKEN_NAME=$TOKEN_SYMBOL
|
||||
fi
|
||||
>&2 echo deploying default token $TOKEN_TYPE
|
||||
echo giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
|
||||
DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
|
||||
echo giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
|
||||
DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
|
||||
elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
|
||||
if [ -z "$TOKEN_SYMBOL" ]; then
|
||||
>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
|
||||
@ -56,55 +56,55 @@ elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
|
||||
>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
|
||||
fi
|
||||
fi
|
||||
DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s`
|
||||
DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s`
|
||||
else
|
||||
>&2 echo unknown token type $TOKEN_TYPE
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
|
||||
giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
|
||||
echo "giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
|
||||
giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
|
||||
|
||||
>&2 echo "deploy account index contract"
|
||||
DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w`
|
||||
DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w`
|
||||
>&2 echo "add deployer address as account index writer"
|
||||
eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
|
||||
eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
|
||||
|
||||
>&2 echo "deploy contract registry contract"
|
||||
CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
|
||||
CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
|
||||
|
||||
# Deploy address declarator registry
|
||||
>&2 echo "deploy address declarator contract"
|
||||
declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
|
||||
DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv $declarator_description`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
|
||||
DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w -vv $declarator_description`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
|
||||
|
||||
# Deploy transfer authorization contact
|
||||
>&2 echo "deploy transfer auth contract"
|
||||
DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
|
||||
DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
|
||||
|
||||
# Deploy token index contract
|
||||
>&2 echo "deploy token index contract"
|
||||
DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS
|
||||
DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w -vv`
|
||||
eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS
|
||||
>&2 echo "add reserve token to token index"
|
||||
eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
|
||||
eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
|
||||
|
||||
# Sarafu faucet contract
|
||||
>&2 echo "deploy token faucet contract"
|
||||
DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
|
||||
DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
|
||||
|
||||
>&2 echo "set token faucet amount"
|
||||
sarafu-faucet-set $fee_price_arg -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
|
||||
sarafu-faucet-set $fee_price_arg -w -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
|
||||
|
||||
>&2 echo "register faucet in registry"
|
||||
eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
|
||||
eth-contract-registry-set -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
|
||||
|
||||
>&2 echo "set faucet as token minter"
|
||||
giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv $DEV_FAUCET_ADDRESS
|
||||
giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERVE_ADDRESS -i $CHAIN_SPEC -p $RPC_PROVIDER -vv $DEV_FAUCET_ADDRESS
|
||||
|
||||
|
||||
#echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset
|
||||
|
||||
@ -58,7 +58,7 @@ config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
|
||||
config.process()
|
||||
args_override = {
|
||||
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
|
||||
'CIC_CHAIN_SPEC': getattr(args, 'i'),
|
||||
'CHAIN_SPEC': getattr(args, 'i'),
|
||||
'KEYSTORE_FILE_PATH': getattr(args, 'y')
|
||||
}
|
||||
config.dict_override(args_override, 'cli')
|
||||
@ -86,7 +86,7 @@ os.makedirs(txs_dir)
|
||||
|
||||
user_dir = args.user_dir
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
chain_str = str(chain_spec)
|
||||
|
||||
old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user