From 98ddf56a1dd52f189dc92f43f3adefe4dccf26f3 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Thu, 15 Apr 2021 14:02:09 +0000 Subject: [PATCH 1/2] Iimprove cic cache service --- apps/cic-cache/docker/Dockerfile | 3 +++ apps/cic-cache/docker/db.sh | 6 ++++++ apps/cic-cache/docker/start_tracker.sh | 5 +++++ apps/cic-cache/requirements.txt | 5 ++--- docker-compose.yml | 19 ++++++++++--------- 5 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 apps/cic-cache/docker/db.sh create mode 100644 apps/cic-cache/docker/start_tracker.sh diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile index b4e4aebc..16752f45 100644 --- a/apps/cic-cache/docker/Dockerfile +++ b/apps/cic-cache/docker/Dockerfile @@ -47,6 +47,9 @@ RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \ mkdir -p /usr/local/share/cic/solidity && \ cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi +COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh +COPY cic-cache/docker/db.sh ./db.sh +RUN chmod 755 ./*.sh # Tracker # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"] # Server diff --git a/apps/cic-cache/docker/db.sh b/apps/cic-cache/docker/db.sh new file mode 100644 index 00000000..002cea14 --- /dev/null +++ b/apps/cic-cache/docker/db.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e +>&2 echo executing database migration +python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv +set +e diff --git a/apps/cic-cache/docker/start_tracker.sh b/apps/cic-cache/docker/start_tracker.sh new file mode 100644 index 00000000..cca1cb1a --- /dev/null +++ b/apps/cic-cache/docker/start_tracker.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. ./db.sh + +/usr/local/bin/cic-cache-trackerd $@ diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 5669eb42..7687b611 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,12 @@ -cic-base~=0.1.2a66 +cic-base~=0.1.2a76 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 moolb~=0.1.0 -cic-eth-registry~=0.5.4a13 +cic-eth-registry~=0.5.4a16 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainlib~=0.0.2a10 chainsyncer[sql]~=0.0.2a1 diff --git a/docker-compose.yml b/docker-compose.yml index 85dbfbce..85723630 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,7 +145,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - /usr/local/bin/cic-cache-trackerd -vv -c /usr/local/etc/cic-cache + ./start_tracker.sh -c /usr/local/etc/cic-cache -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -191,13 +191,13 @@ services: context: apps dockerfile: cic-cache/docker/Dockerfile environment: - DATABASE_USER: $DATABASE_USER - DATABASE_HOST: $DATABASE_HOST - DATABASE_PORT: $DATABASE_PORT - DATABASE_PASSWORD: $DATABASE_PASSWORD - DATABASE_NAME: $DATABASE_NAME_CIC_CACHE + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + #DATABASE_PASSWORD: ${DATABASE_PASSWORD:- + DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} DATABASE_DEBUG: 1 - PGPASSWORD: $DATABASE_PASSWORD + #PGPASSWORD: $DATABASE_PASSWORD SERVER_PORT: 8000 ports: - ${HTTP_PORT_CIC_CACHE:-63313}:8000 @@ -212,9 +212,10 @@ services: - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \ + --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \ --http :8000 \ - --pyargv -vv + --pyargv "-vv" + cic-eth-tasker: # image: grassrootseconomics:cic-eth-service From 22ec8e2e0eb3a4f304c5f75f23366fd3a263488e Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 15 Apr 2021 18:04:47 +0200 Subject: [PATCH 2/2] Update sql backend symbol name, deps --- apps/cic-cache/cic_cache/runnable/daemons/tracker.py | 12 ++++++------ apps/cic-cache/requirements.txt | 4 ++-- apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py | 6 +----- apps/cic-eth/cic_eth/runnable/daemons/tracker.py | 12 ++++++------ apps/cic-eth/cic_eth/sync/retry.py | 2 +- apps/cic-eth/docker/Dockerfile | 3 ++- apps/cic-eth/requirements.txt | 2 +- apps/contract-migration/docker/Dockerfile | 6 +++--- .../scripts/cic_eth/import_balance.py | 2 +- .../contract-migration/scripts/eth/import_balance.py | 2 +- apps/contract-migration/scripts/requirements.txt | 8 ++++---- 11 files changed, 28 insertions(+), 31 deletions(-) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index cee54144..bd03528a 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -26,7 +26,7 @@ from chainlib.eth.block import ( from hexathon import ( strip_0x, ) -from chainsyncer.backend import SyncerBackend +from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver import ( HeadSyncer, HistorySyncer, @@ -71,13 +71,13 @@ def main(): syncers = [] - #if SyncerBackend.first(chain_spec): - # backend = SyncerBackend.initial(chain_spec, block_offset) - syncer_backends = SyncerBackend.resume(chain_spec, block_offset) + #if SQLBackend.first(chain_spec): + # backend = SQLBackend.initial(chain_spec, block_offset) + syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: logg.info('found no backends to resume') - syncers.append(SyncerBackend.initial(chain_spec, block_offset)) + syncers.append(SQLBackend.initial(chain_spec, block_offset)) else: for syncer_backend in syncer_backends: logg.info('resuming sync session {}'.format(syncer_backend)) @@ -85,7 +85,7 @@ def main(): for syncer_backend in syncer_backends: syncers.append(HistorySyncer(syncer_backend)) - syncer_backend = SyncerBackend.live(chain_spec, block_offset+1) + syncer_backend = SQLBackend.live(chain_spec, block_offset+1) syncers.append(HeadSyncer(syncer_backend)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 7687b611..b538691d 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,4 +1,4 @@ -cic-base~=0.1.2a76 +cic-base~=0.1.2a77 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 @@ -9,4 +9,4 @@ semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.2a1 +chainsyncer[sql]~=0.0.2a2 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index b93f1184..95dc7031 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -15,7 +15,6 @@ from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec from chainlib.eth.tx import unpack from chainlib.connection import RPCConnection -from chainsyncer.error import SyncDone from hexathon import strip_0x from chainqueue.db.enum import ( StatusEnum, @@ -153,10 +152,7 @@ class DispatchSyncer: def main(): syncer = DispatchSyncer(chain_spec) conn = RPCConnection.connect(chain_spec, 'default') - try: - syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL'))) - except SyncDone as e: - sys.stderr.write("dispatcher done at block {}\n".format(e)) + syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL'))) sys.exit(0) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 653dd8ae..f25f28d7 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -26,7 +26,7 @@ from chainlib.eth.block import ( from hexathon import ( strip_0x, ) -from chainsyncer.backend import SyncerBackend +from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver import ( HeadSyncer, HistorySyncer, @@ -88,18 +88,18 @@ def main(): syncers = [] - #if SyncerBackend.first(chain_spec): - # backend = SyncerBackend.initial(chain_spec, block_offset) - syncer_backends = SyncerBackend.resume(chain_spec, block_offset) + #if SQLBackend.first(chain_spec): + # backend = SQLBackend.initial(chain_spec, block_offset) + syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: logg.info('found no backends to resume') - syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset)) + syncer_backends.append(SQLBackend.initial(chain_spec, block_offset)) else: for syncer_backend in syncer_backends: logg.info('resuming sync session {}'.format(syncer_backend)) - syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1)) + syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1)) for syncer_backend in syncer_backends: try: diff --git a/apps/cic-eth/cic_eth/sync/retry.py b/apps/cic-eth/cic_eth/sync/retry.py index b4cefca6..b311964d 100644 --- a/apps/cic-eth/cic_eth/sync/retry.py +++ b/apps/cic-eth/cic_eth/sync/retry.py @@ -4,7 +4,7 @@ import datetime # external imports from chainsyncer.driver import HeadSyncer -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.error import NoBlockForYou from chainlib.eth.block import ( block_by_number, diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index e5bdd2f9..408f9aa5 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -29,7 +29,8 @@ RUN /usr/local/bin/python -m pip install --upgrade pip # python merge_requirements.py | tee merged_requirements.txt #RUN cd cic-base && \ # pip install $pip_extra_index_url_flag -r ./merged_requirements.txt -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 +RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76 +RUN pip install $pip_extra_index_url_flag chainsyncer~=0.0.2a2 COPY cic-eth/scripts/ scripts/ COPY cic-eth/setup.cfg cic-eth/setup.py ./ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 92ce8c4f..708b63d6 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -18,7 +18,7 @@ moolb~=0.1.1b2 eth-address-index~=0.1.1a7 chainlib~=0.0.2a5 hexathon~=0.0.1a7 -chainsyncer[sql]~=0.0.2a1 +chainsyncer[sql]~=0.0.2a2 chainqueue~=0.0.1a7 pysha3==1.0.2 coincurve==15.0.0 diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index dd2cf26c..6b053b66 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,9 +57,9 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a67 -ARG cic_eth_version=0.11.0b1 -ARG sarafu_faucet_version=0.0.2a19 +ARG cic_base_version=0.1.2a77 +ARG cic_eth_version=0.11.0b6 +ARG sarafu_faucet_version=0.0.2a28 ARG cic_contracts_version=0.0.2a2 RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \ cic-eth==$cic_eth_version \ diff --git a/apps/contract-migration/scripts/cic_eth/import_balance.py b/apps/contract-migration/scripts/cic_eth/import_balance.py index 2067d197..6a8f6181 100644 --- a/apps/contract-migration/scripts/cic_eth/import_balance.py +++ b/apps/contract-migration/scripts/cic_eth/import_balance.py @@ -17,7 +17,7 @@ from hexathon import ( strip_0x, add_0x, ) -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.driver import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( diff --git a/apps/contract-migration/scripts/eth/import_balance.py b/apps/contract-migration/scripts/eth/import_balance.py index 2067d197..6a8f6181 100644 --- a/apps/contract-migration/scripts/eth/import_balance.py +++ b/apps/contract-migration/scripts/eth/import_balance.py @@ -17,7 +17,7 @@ from hexathon import ( strip_0x, add_0x, ) -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.driver import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index d0ddd1e6..4d02487e 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2a67 -sarafu-faucet==0.0.2a20 -cic-eth==0.11.0b3 +cic-base[full_graph]==0.1.2a77 +sarafu-faucet==0.0.2a28 +cic-eth==0.11.0b6 cic-types==0.1.0a10 -crypto-dev-signer==0.4.14b1 +crypto-dev-signer==0.4.14b2