kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Ensure serial send of txs to node

Browse Source
This commit is contained in:
nolash
2021-04-08 07:09:38 +02:00
parent 2f0a95d2b6
commit 8f2d3997d3
23 changed files with 261 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/contract-migration/scripts/create_import_users.py
View File

@@ -215,7 +215,7 @@ if __name__ == '__main__':
amount = genAmount()
fa.write('{},{}\n'.format(eth,amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
logg.debug('pidx {}, uid {}, eth {}, amount {}, phone {}'.format(pidx, uid, eth, amount, phone))
i += 1

94
apps/contract-migration/scripts/import_task.py
View File

@@ -13,6 +13,10 @@ from hexathon import (
add_0x,
)
from chainlib.eth.address import to_checksum_address
from chainlib.eth.tx import (
unpack,
raw,
)
from cic_types.processor import generate_metadata_pointer
from cic_types.models.person import Person
@@ -23,9 +27,10 @@ celery_app = celery.current_app
class MetadataTask(celery.Task):
count = 0
balances = None
chain_spec = None
import_path = 'out'
import_dir = 'out'
meta_host = None
meta_port = None
meta_path = ''
@@ -33,11 +38,12 @@ class MetadataTask(celery.Task):
balance_processor = None
autoretry_for = (
urllib.error.HTTPError,
OSError,
)
retry_kwargs = {
'countdown': 3,
'max_retries': 100,
}
retry_jitter = True
retry_backoff = True
retry_backoff_max = 60
max_retries = None
@classmethod
def meta_url(self):
@@ -79,12 +85,12 @@ def resolve_phone(self, phone):
@celery_app.task(bind=True, base=MetadataTask)
def generate_metadata(self, address, phone):
old_address = old_address_from_phone(self.import_path, phone)
old_address = old_address_from_phone(self.import_dir, phone)
logg.debug('address {}'.format(address))
old_address_upper = strip_0x(old_address).upper()
metadata_path = '{}/old/{}/{}/{}.json'.format(
self.import_path,
self.import_dir,
old_address_upper[:2],
old_address_upper[2:4],
old_address_upper,
@@ -103,7 +109,7 @@ def generate_metadata(self, address, phone):
new_address_clean = strip_0x(address)
filepath = os.path.join(
self.import_path,
self.import_dir,
'new',
new_address_clean[:2].upper(),
new_address_clean[2:4].upper(),
@@ -118,7 +124,7 @@ def generate_metadata(self, address, phone):
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
meta_filepath = os.path.join(
self.import_path,
self.import_dir,
'meta',
'{}.json'.format(new_address_clean.upper()),
)
@@ -130,19 +136,77 @@ def generate_metadata(self, address, phone):
@celery_app.task(bind=True, base=MetadataTask)
def transfer_opening_balance(self, address, phone, serial):
def opening_balance_tx(self, address, phone, serial):
old_address = old_address_from_phone(self.import_path, phone)
old_address = old_address_from_phone(self.import_dir, phone)
k = to_checksum_address(strip_0x(old_address))
balance = self.balances[k]
logg.debug('found balance {} for address {} phone {}'.format(balance, old_address, phone))
decimal_balance = self.balance_processor.get_decimal_amount(balance)
decimal_balance = self.balance_processor.get_decimal_amount(int(balance))
tx_hash_hex = self.balance_processor.get_rpc_tx(address, decimal_balance, serial)
logg.debug('sending {} to {} tx hash {}'.format(decimal_balance, address, tx_hash_hex))
(tx_hash_hex, o) = self.balance_processor.get_rpc_tx(address, decimal_balance, serial)
return tx_hash_hex
tx = unpack(bytes.fromhex(strip_0x(o)), self.chain_spec)
logg.debug('generated tx token value {} to {} tx hash {}'.format(decimal_balance, address, tx_hash_hex))
tx_path = os.path.join(
self.import_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(o)
f.close()
tx_nonce_path = os.path.join(
self.import_dir,
'txs',
'.' + str(tx['nonce']),
)
os.symlink(os.path.realpath(tx_path), tx_nonce_path)
return tx['hash']
@celery_app.task(bind=True, base=MetadataTask, autoretry_for=(FileNotFoundError,), max_retries=None, countdown=0.1)
def send_txs(self, nonce):
if nonce == self.count + self.balance_processor.nonce_offset:
logg.info('reached nonce {} (offset {} + count {}) exiting'.format(nonce, self.balance_processor.nonce_offset, self.count))
return
tx_nonce_path = os.path.join(
self.import_dir,
'txs',
'.' + str(nonce),
)
f = open(tx_nonce_path, 'r')
tx_signed_raw_hex = f.read()
f.close()
os.unlink(tx_nonce_path)
o = raw(tx_signed_raw_hex)
tx_hash_hex = self.balance_processor.conn.do(o)
logg.info('sent nonce {} tx hash {}'.format(nonce, tx_hash_hex)) #tx_signed_raw_hex))
nonce += 1
queue = self.request.delivery_info.get('routing_key')
s = celery.signature(
'import_task.send_txs',
[
nonce,
],
queue=queue,
)
s.apply_async()
return nonce

12
apps/contract-migration/scripts/import_ussd_balance.py
View File

@@ -138,7 +138,17 @@ def main():
f.close()
MetadataTask.balances = balances
MetadataTask.count = i
s = celery.signature(
'import_task.send_txs',
[
MetadataTask.balance_processor.nonce_offset,
],
queue='cic-import-ussd',
)
s.apply_async()
argv = ['worker', '-Q', 'cic-import-ussd', '--loglevel=DEBUG']
celery_app.worker_main(argv)

9
apps/contract-migration/scripts/import_ussd_users.py
View File

@@ -37,7 +37,7 @@ argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis
argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback')
argparser.add_argument('--batch-size', dest='batch_size', default=100, type=int, help='burst size of sending transactions to node') # batch size should be slightly below cumulative gas limit worth, eg 80000 gas txs with 8000000 limit is a bit less than 100 batch size
argparser.add_argument('--batch-delay', dest='batch_delay', default=2, type=int, help='seconds delay between batches')
argparser.add_argument('--batch-delay', dest='batch_delay', default=3, type=int, help='seconds delay between batches')
argparser.add_argument('--timeout', default=60.0, type=float, help='Callback timeout')
argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
argparser.add_argument('-v', action='store_true', help='Be verbose')
@@ -78,6 +78,9 @@ os.makedirs(meta_dir)
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec)
@@ -165,7 +168,7 @@ if __name__ == '__main__':
)
s_balance = celery.signature(
'import_task.transfer_opening_balance',
'import_task.opening_balance_tx',
[
phone,
i,
@@ -175,7 +178,7 @@ if __name__ == '__main__':
s_meta.link(s_balance)
s_phone.link(s_meta)
s_phone.apply_async()
s_phone.apply_async(countdown=7) # block time plus a bit of time for ussd processing
i += 1
sys.stdout.write('imported {} {}'.format(i, u).ljust(200) + "\r")

15
apps/contract-migration/scripts/import_util.py
View File

@@ -7,7 +7,10 @@ from eth_token_index import TokenUniqueSymbolIndex
from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import OverrideNonceOracle
from chainlib.eth.erc20 import ERC20
from chainlib.eth.tx import count
from chainlib.eth.tx import (
count,
TxFormat,
)
logg = logging.getLogger().getChild(__name__)
@@ -51,16 +54,18 @@ class BalanceProcessor:
tx_factory = ERC20(self.chain_spec)
o = tx_factory.decimals(self.token_address)
r = self.conn.do(o)
self.value_multiplier = int(r, 16) ** 10
n = tx_factory.parse_decimals(r)
self.value_multiplier = 10 ** n
def get_rpc_tx(self, recipient, value, i):
logg.debug('initiating nonce offset {} for recipient {}'.format(self.nonce_offset + i, recipient))
nonce_oracle = OverrideNonceOracle(self.signer_address, self.nonce_offset + i)
tx_factory = ERC20(self.chain_spec, signer=self.signer, nonce_oracle=nonce_oracle, gas_oracle=self.gas_oracle)
(tx_hash_hex, o) = tx_factory.transfer(self.token_address, self.signer_address, recipient, value)
self.conn.do(o)
return tx_hash_hex
return tx_factory.transfer(self.token_address, self.signer_address, recipient, value, tx_format=TxFormat.RLP_SIGNED)
#(tx_hash_hex, o) = tx_factory.transfer(self.token_address, self.signer_address, recipient, value)
#self.conn.do(o)
#return tx_hash_hex
def get_decimal_amount(self, value):