Compare commits

...

2 Commits

Author SHA1 Message Date
nolash
184885b35f
Upgrade deps 2021-04-06 21:06:24 +02:00
nolash
5a4ce54add
Add block time calc to retry, tracker 2021-04-06 21:01:50 +02:00
10 changed files with 29 additions and 56 deletions

View File

@ -1,13 +1,13 @@
cic-base~=0.1.2a58 cic-base~=0.1.2a62
alembic==1.4.2 alembic==1.4.2
confini~=0.3.6rc3 confini~=0.3.6rc3
uwsgi==2.0.19.1 uwsgi==2.0.19.1
moolb~=0.1.0 moolb~=0.1.0
cic-eth-registry~=0.5.4a10 cic-eth-registry~=0.5.4a12
SQLAlchemy==1.3.20 SQLAlchemy==1.3.20
semver==2.13.0 semver==2.13.0
psycopg2==2.8.6 psycopg2==2.8.6
celery==4.4.7 celery==4.4.7
redis==3.5.3 redis==3.5.3
chainlib~=0.0.2a2 chainlib~=0.0.2a5
chainsyncer~=0.0.1a21 chainsyncer~=0.0.1a21

View File

@ -20,6 +20,7 @@ from cic_eth.admin.ctrl import lock_send
from cic_eth.db.enum import LockEnum from cic_eth.db.enum import LockEnum
from cic_eth.runnable.daemons.filters.straggler import StragglerFilter from cic_eth.runnable.daemons.filters.straggler import StragglerFilter
from cic_eth.sync.retry import RetrySyncer from cic_eth.sync.retry import RetrySyncer
from cic_eth.stat import init_chain_stat
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
@ -71,57 +72,21 @@ RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='def
dsn = dsn_from_config(config) dsn = dsn_from_config(config)
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
## TODO: we already have the signed raw tx in get, so its a waste of cycles to get_tx here
#def sendfail_filter(w3, tx_hash, rcpt, chain_spec):
# tx_dict = get_tx(tx_hash)
# tx = unpack(tx_dict['signed_tx'], chain_spec)
# logg.debug('submitting tx {} for retry'.format(tx_hash))
# s_check = celery.signature(
# 'cic_eth.admin.ctrl.check_lock',
# [
# tx_hash,
# chain_str,
# LockEnum.QUEUE,
# tx['from'],
# ],
# queue=queue,
# )
## s_resume = celery.signature(
## 'cic_eth.eth.tx.resume_tx',
## [
## chain_str,
## ],
## queue=queue,
## )
#
## s_retry_status = celery.signature(
## 'cic_eth.queue.state.set_ready',
## [],
## queue=queue,
## )
# s_resend = celery.signature(
# 'cic_eth.eth.gas.resend_with_higher_gas',
# [
# chain_str,
# ],
# queue=queue,
# )
#
# #s_resume.link(s_retry_status)
# #s_check.link(s_resume)
# s_check.link(s_resend)
# s_check.apply_async()
def main(): def main():
conn = RPCConnection.connect(chain_spec, 'default') conn = RPCConnection.connect(chain_spec, 'default')
straggler_delay = int(config.get('CIC_TX_RETRY_DELAY'))
loop_interval = config.get('SYNCER_LOOP_INTERVAL')
if loop_interval == None:
stat = init_chain_stat(conn)
loop_interval = stat.block_average()
syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE')) syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE'))
syncer.backend.set(0, 0) syncer.backend.set(0, 0)
fltr = StragglerFilter(chain_spec, queue=queue) fltr = StragglerFilter(chain_spec, queue=queue)
syncer.add_filter(fltr) syncer.add_filter(fltr)
syncer.loop(float(straggler_delay), conn) syncer.loop(int(loop_interval), conn)
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -7,7 +7,7 @@ import argparse
import sys import sys
import re import re
# third-party imports # external imports
import confini import confini
import celery import celery
import rlp import rlp
@ -42,6 +42,7 @@ from cic_eth.runnable.daemons.filters import (
RegistrationFilter, RegistrationFilter,
TransferAuthFilter, TransferAuthFilter,
) )
from cic_eth.stat import init_chain_stat
script_dir = os.path.realpath(os.path.dirname(__file__)) script_dir = os.path.realpath(os.path.dirname(__file__))
@ -78,6 +79,11 @@ def main():
block_current = int(r, 16) block_current = int(r, 16)
block_offset = block_current + 1 block_offset = block_current + 1
loop_interval = config.get('SYNCER_LOOP_INTERVAL')
if loop_interval == None:
stat = init_chain_stat(rpc, block_start=block_current)
loop_interval = stat.block_average()
logg.debug('starting at block {}'.format(block_offset)) logg.debug('starting at block {}'.format(block_offset))
syncers = [] syncers = []
@ -140,7 +146,8 @@ def main():
for cf in callback_filters: for cf in callback_filters:
syncer.add_filter(cf) syncer.add_filter(cf)
r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc) #r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
r = syncer.loop(int(loop_interval), rpc)
sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
i += 1 i += 1

View File

@ -2,3 +2,4 @@
registry_address = registry_address =
chain_spec = evm:bloxberg:8996 chain_spec = evm:bloxberg:8996
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
tx_retry_delay = 20

View File

@ -1,2 +1,2 @@
[SYNCER] [SYNCER]
loop_interval = 1 loop_interval =

View File

@ -1,2 +1,2 @@
[SYNCER] [SYNCER]
loop_interval = 1 loop_interval =

View File

@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
# python merge_requirements.py | tee merged_requirements.txt # python merge_requirements.py | tee merged_requirements.txt
#RUN cd cic-base && \ #RUN cd cic-base && \
# pip install $pip_extra_index_url_flag -r ./merged_requirements.txt # pip install $pip_extra_index_url_flag -r ./merged_requirements.txt
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a61 RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
COPY cic-eth/scripts/ scripts/ COPY cic-eth/scripts/ scripts/
COPY cic-eth/setup.cfg cic-eth/setup.py ./ COPY cic-eth/setup.cfg cic-eth/setup.py ./

View File

@ -1,4 +1,4 @@
cic-base~=0.1.2a60 cic-base~=0.1.2a62
celery==4.4.7 celery==4.4.7
crypto-dev-signer~=0.4.14a17 crypto-dev-signer~=0.4.14a17
confini~=0.3.6rc3 confini~=0.3.6rc3
@ -16,7 +16,7 @@ semver==2.13.0
websocket-client==0.57.0 websocket-client==0.57.0
moolb~=0.1.1b2 moolb~=0.1.1b2
eth-address-index~=0.1.1a7 eth-address-index~=0.1.1a7
chainlib~=0.0.2a4 chainlib~=0.0.2a5
hexathon~=0.0.1a7 hexathon~=0.0.1a7
chainsyncer~=0.0.1a21 chainsyncer~=0.0.1a21
chainqueue~=0.0.1a7 chainqueue~=0.0.1a7

View File

@ -6,7 +6,7 @@ RUN apt-get update && \
WORKDIR /usr/src/cic-notify WORKDIR /usr/src/cic-notify
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a44 RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
COPY cic-notify/setup.cfg \ COPY cic-notify/setup.cfg \
cic-notify/setup.py \ cic-notify/setup.py \

View File

@ -57,7 +57,7 @@ WORKDIR /home/grassroots
USER grassroots USER grassroots
ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
ARG cic_base_version=0.1.2a61 ARG cic_base_version=0.1.2a62
ARG cic_eth_version=0.11.0b1 ARG cic_eth_version=0.11.0b1
ARG sarafu_faucet_version=0.0.2a19 ARG sarafu_faucet_version=0.0.2a19
ARG cic_contracts_version=0.0.2a2 ARG cic_contracts_version=0.0.2a2