diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index cee54144..bd03528a 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -26,7 +26,7 @@ from chainlib.eth.block import ( from hexathon import ( strip_0x, ) -from chainsyncer.backend import SyncerBackend +from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver import ( HeadSyncer, HistorySyncer, @@ -71,13 +71,13 @@ def main(): syncers = [] - #if SyncerBackend.first(chain_spec): - # backend = SyncerBackend.initial(chain_spec, block_offset) - syncer_backends = SyncerBackend.resume(chain_spec, block_offset) + #if SQLBackend.first(chain_spec): + # backend = SQLBackend.initial(chain_spec, block_offset) + syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: logg.info('found no backends to resume') - syncers.append(SyncerBackend.initial(chain_spec, block_offset)) + syncers.append(SQLBackend.initial(chain_spec, block_offset)) else: for syncer_backend in syncer_backends: logg.info('resuming sync session {}'.format(syncer_backend)) @@ -85,7 +85,7 @@ def main(): for syncer_backend in syncer_backends: syncers.append(HistorySyncer(syncer_backend)) - syncer_backend = SyncerBackend.live(chain_spec, block_offset+1) + syncer_backend = SQLBackend.live(chain_spec, block_offset+1) syncers.append(HeadSyncer(syncer_backend)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') diff --git a/apps/cic-cache/docker/Dockerfile b/apps/cic-cache/docker/Dockerfile index ea0af0e1..ed01fd54 100644 --- a/apps/cic-cache/docker/Dockerfile +++ b/apps/cic-cache/docker/Dockerfile @@ -47,6 +47,9 @@ RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \ mkdir -p /usr/local/share/cic/solidity && \ cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi +COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh +COPY cic-cache/docker/db.sh ./db.sh +RUN chmod 755 ./*.sh # Tracker # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"] # Server diff --git a/apps/cic-cache/docker/db.sh b/apps/cic-cache/docker/db.sh new file mode 100644 index 00000000..002cea14 --- /dev/null +++ b/apps/cic-cache/docker/db.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e +>&2 echo executing database migration +python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv +set +e diff --git a/apps/cic-cache/docker/start_tracker.sh b/apps/cic-cache/docker/start_tracker.sh new file mode 100644 index 00000000..cca1cb1a --- /dev/null +++ b/apps/cic-cache/docker/start_tracker.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. ./db.sh + +/usr/local/bin/cic-cache-trackerd $@ diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 5669eb42..b538691d 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,12 @@ -cic-base~=0.1.2a66 +cic-base~=0.1.2a77 alembic==1.4.2 confini~=0.3.6rc3 uwsgi==2.0.19.1 moolb~=0.1.0 -cic-eth-registry~=0.5.4a13 +cic-eth-registry~=0.5.4a16 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainlib~=0.0.2a10 -chainsyncer[sql]~=0.0.2a1 +chainsyncer[sql]~=0.0.2a2 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index b93f1184..95dc7031 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -15,7 +15,6 @@ from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec from chainlib.eth.tx import unpack from chainlib.connection import RPCConnection -from chainsyncer.error import SyncDone from hexathon import strip_0x from chainqueue.db.enum import ( StatusEnum, @@ -153,10 +152,7 @@ class DispatchSyncer: def main(): syncer = DispatchSyncer(chain_spec) conn = RPCConnection.connect(chain_spec, 'default') - try: - syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL'))) - except SyncDone as e: - sys.stderr.write("dispatcher done at block {}\n".format(e)) + syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL'))) sys.exit(0) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index ee565f55..d20dc338 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -25,7 +25,7 @@ from chainlib.eth.block import ( from hexathon import ( strip_0x, ) -from chainsyncer.backend import SyncerBackend +from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver import ( HeadSyncer, HistorySyncer, @@ -93,18 +93,18 @@ def main(): syncers = [] - #if SyncerBackend.first(chain_spec): - # backend = SyncerBackend.initial(chain_spec, block_offset) - syncer_backends = SyncerBackend.resume(chain_spec, block_offset) + #if SQLBackend.first(chain_spec): + # backend = SQLBackend.initial(chain_spec, block_offset) + syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: logg.info('found no backends to resume') - syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset)) + syncer_backends.append(SQLBackend.initial(chain_spec, block_offset)) else: for syncer_backend in syncer_backends: logg.info('resuming sync session {}'.format(syncer_backend)) - syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1)) + syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1)) for syncer_backend in syncer_backends: try: diff --git a/apps/cic-eth/cic_eth/sync/retry.py b/apps/cic-eth/cic_eth/sync/retry.py index b4cefca6..b311964d 100644 --- a/apps/cic-eth/cic_eth/sync/retry.py +++ b/apps/cic-eth/cic_eth/sync/retry.py @@ -4,7 +4,7 @@ import datetime # external imports from chainsyncer.driver import HeadSyncer -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.error import NoBlockForYou from chainlib.eth.block import ( block_by_number, diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 4b6c788a..9ca4a83a 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -29,7 +29,7 @@ RUN /usr/local/bin/python -m pip install --upgrade pip # python merge_requirements.py | tee merged_requirements.txt #RUN cd cic-base && \ # pip install $pip_extra_index_url_flag -r ./merged_requirements.txt -RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76 +RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a77 COPY cic-eth/scripts/ scripts/ COPY cic-eth/setup.cfg cic-eth/setup.py ./ diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 97848c39..15e02448 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -17,7 +17,7 @@ moolb~=0.1.1b2 eth-address-index~=0.1.1a9 chainlib~=0.0.2a13 hexathon~=0.0.1a7 -chainsyncer[sql]~=0.0.2a1 +chainsyncer[sql]~=0.0.2a2 chainqueue~=0.0.1a7 pysha3==1.0.2 coincurve==15.0.0 diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index fd02835d..6b053b66 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -57,7 +57,7 @@ WORKDIR /home/grassroots USER grassroots ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 -ARG cic_base_version=0.1.2a76 +ARG cic_base_version=0.1.2a77 ARG cic_eth_version=0.11.0b6 ARG sarafu_faucet_version=0.0.2a28 ARG cic_contracts_version=0.0.2a2 diff --git a/apps/contract-migration/scripts/cic_eth/import_balance.py b/apps/contract-migration/scripts/cic_eth/import_balance.py index 2067d197..6a8f6181 100644 --- a/apps/contract-migration/scripts/cic_eth/import_balance.py +++ b/apps/contract-migration/scripts/cic_eth/import_balance.py @@ -17,7 +17,7 @@ from hexathon import ( strip_0x, add_0x, ) -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.driver import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( diff --git a/apps/contract-migration/scripts/eth/import_balance.py b/apps/contract-migration/scripts/eth/import_balance.py index 2067d197..6a8f6181 100644 --- a/apps/contract-migration/scripts/eth/import_balance.py +++ b/apps/contract-migration/scripts/eth/import_balance.py @@ -17,7 +17,7 @@ from hexathon import ( strip_0x, add_0x, ) -from chainsyncer.backend import MemBackend +from chainsyncer.backend.memory import MemBackend from chainsyncer.driver import HeadSyncer from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.block import ( diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index 599eeea3..4d02487e 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full]==0.1.2a76 +cic-base[full_graph]==0.1.2a77 sarafu-faucet==0.0.2a28 -cic-eth==0.11.0b5 +cic-eth==0.11.0b6 cic-types==0.1.0a10 crypto-dev-signer==0.4.14b2 diff --git a/docker-compose.yml b/docker-compose.yml index 85dbfbce..85723630 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -145,7 +145,7 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - /usr/local/bin/cic-cache-trackerd -vv -c /usr/local/etc/cic-cache + ./start_tracker.sh -c /usr/local/etc/cic-cache -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -191,13 +191,13 @@ services: context: apps dockerfile: cic-cache/docker/Dockerfile environment: - DATABASE_USER: $DATABASE_USER - DATABASE_HOST: $DATABASE_HOST - DATABASE_PORT: $DATABASE_PORT - DATABASE_PASSWORD: $DATABASE_PASSWORD - DATABASE_NAME: $DATABASE_NAME_CIC_CACHE + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + #DATABASE_PASSWORD: ${DATABASE_PASSWORD:- + DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} DATABASE_DEBUG: 1 - PGPASSWORD: $DATABASE_PASSWORD + #PGPASSWORD: $DATABASE_PASSWORD SERVER_PORT: 8000 ports: - ${HTTP_PORT_CIC_CACHE:-63313}:8000 @@ -212,9 +212,10 @@ services: - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \ + --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \ --http :8000 \ - --pyargv -vv + --pyargv "-vv" + cic-eth-tasker: # image: grassrootseconomics:cic-eth-service