diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 2cf06a49..95fa8ac1 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,13 @@ -cic-base~=0.1.2a58 +cic-base~=0.1.2a62 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 moolb~=0.1.0 -cic-eth-registry~=0.5.4a10 +cic-eth-registry~=0.5.4a12 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainlib~=0.0.2a2 +chainlib~=0.0.2a5 chainsyncer~=0.0.1a21 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/retry.py b/apps/cic-eth/cic_eth/runnable/daemons/retry.py index 1044101b..8974c33d 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/retry.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/retry.py @@ -20,6 +20,7 @@ from cic_eth.admin.ctrl import lock_send from cic_eth.db.enum import LockEnum from cic_eth.runnable.daemons.filters.straggler import StragglerFilter from cic_eth.sync.retry import RetrySyncer +from cic_eth.stat import init_chain_stat logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -71,57 +72,21 @@ RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='def dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) - -## TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here -#def sendfail_filter(w3, tx_hash, rcpt, chain_spec): -# tx_dict = get_tx(tx_hash) -# tx = unpack(tx_dict['signed_tx'], chain_spec) -# logg.debug('submitting tx {} for retry'.format(tx_hash)) -# s_check = celery.signature( -# 'cic_eth.admin.ctrl.check_lock', -# [ -# tx_hash, -# chain_str, -# LockEnum.QUEUE, -# tx['from'], -# ], -# queue=queue, -# ) -## s_resume = celery.signature( -## 'cic_eth.eth.tx.resume_tx', -## [ -## chain_str, -## ], -## queue=queue, -## ) -# -## s_retry_status = celery.signature( -## 'cic_eth.queue.state.set_ready', -## [], -## queue=queue, -## ) -# s_resend = celery.signature( -# 'cic_eth.eth.gas.resend_with_higher_gas', -# [ -# chain_str, -# ], -# queue=queue, -# ) -# -# #s_resume.link(s_retry_status) -# #s_check.link(s_resume) -# s_check.link(s_resend) -# s_check.apply_async() - def main(): conn = RPCConnection.connect(chain_spec, 'default') + + straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) + loop_interval = config.get('SYNCER_LOOP_INTERVAL') + if loop_interval == None: + stat = init_chain_stat(conn) + loop_interval = stat.block_average() + syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE')) syncer.backend.set(0, 0) fltr = StragglerFilter(chain_spec, queue=queue) syncer.add_filter(fltr) - syncer.loop(float(straggler_delay), conn) + syncer.loop(int(loop_interval), conn) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index c2a93da1..653dd8ae 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -7,7 +7,7 @@ import argparse import sys import re -# third-party imports +# external imports import confini import celery import rlp @@ -42,6 +42,7 @@ from cic_eth.runnable.daemons.filters import ( RegistrationFilter, TransferAuthFilter, ) +from cic_eth.stat import init_chain_stat script_dir = os.path.realpath(os.path.dirname(__file__)) @@ -78,6 +79,11 @@ def main(): block_current = int(r, 16) block_offset = block_current + 1 + loop_interval = config.get('SYNCER_LOOP_INTERVAL') + if loop_interval == None: + stat = init_chain_stat(rpc, block_start=block_current) + loop_interval = stat.block_average() + logg.debug('starting at block {}'.format(block_offset)) syncers = [] @@ -140,7 +146,8 @@ def main(): for cf in callback_filters: syncer.add_filter(cf) - r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc) + #r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc) + r = syncer.loop(int(loop_interval), rpc) sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) i += 1 diff --git a/apps/cic-eth/config/docker/cic.ini b/apps/cic-eth/config/docker/cic.ini index 251a6be3..50032aa9 100644 --- a/apps/cic-eth/config/docker/cic.ini +++ b/apps/cic-eth/config/docker/cic.ini @@ -2,3 +2,4 @@ registry_address = chain_spec = evm:bloxberg:8996 trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C +tx_retry_delay = 20 diff --git a/apps/cic-eth/config/docker/syncer.ini b/apps/cic-eth/config/docker/syncer.ini index 97236743..9c452999 100644 --- a/apps/cic-eth/config/docker/syncer.ini +++ b/apps/cic-eth/config/docker/syncer.ini @@ -1,2 +1,2 @@ [SYNCER] -loop_interval = 1 +loop_interval = diff --git a/apps/cic-eth/config/syncer.ini b/apps/cic-eth/config/syncer.ini index 97236743..9c452999 100644 --- a/apps/cic-eth/config/syncer.ini +++ b/apps/cic-eth/config/syncer.ini @@ -1,2 +1,2 @@ [SYNCER] -loop_interval = 1 +loop_interval = diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index d60dac16..e5bdd2f9 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip # python merge_requirements.py | tee merged_requirements.txt #RUN cd cic-base && \ # pip install $pip_extra_index_url_flag -r ./merged_requirements.txt -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a61 +RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 COPY cic-eth/scripts/ scripts/ COPY cic-eth/setup.cfg cic-eth/setup.py ./ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index a4cf6a36..73ed26e3 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,4 +1,4 @@ -cic-base~=0.1.2a60 +cic-base~=0.1.2a62 celery==4.4.7 crypto-dev-signer~=0.4.14a17 confini~=0.3.6rc3 @@ -16,7 +16,7 @@ semver==2.13.0 websocket-client==0.57.0 moolb~=0.1.1b2 eth-address-index~=0.1.1a7 -chainlib~=0.0.2a4 +chainlib~=0.0.2a5 hexathon~=0.0.1a7 chainsyncer~=0.0.1a21 chainqueue~=0.0.1a7 diff --git a/apps/cic-notify/docker/Dockerfile b/apps/cic-notify/docker/Dockerfile index dfce433a..67b2e4c4 100644 --- a/apps/cic-notify/docker/Dockerfile +++ b/apps/cic-notify/docker/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && \ WORKDIR /usr/src/cic-notify ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44 +RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 COPY cic-notify/setup.cfg \ cic-notify/setup.py \ diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index b2282e2b..8a583ea1 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,7 +57,7 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a61 +ARG cic_base_version=0.1.2a62 ARG cic_eth_version=0.11.0b1 ARG sarafu_faucet_version=0.0.2a19 ARG cic_contracts_version=0.0.2a2