Merge remote-tracking branch 'origin/master' into lash/chainlib-regression

Iimprove cic cache service

See merge request grassrootseconomics/cic-internal-integration!100

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -26,7 +26,7 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend import SyncerBackend
+from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver import (
         HeadSyncer,
         HistorySyncer,
@@ -71,13 +71,13 @@ def main():
 
     syncers = []
 
-    #if SyncerBackend.first(chain_spec):
-    #    backend = SyncerBackend.initial(chain_spec, block_offset)
-    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
+    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncers.append(SyncerBackend.initial(chain_spec, block_offset))
+        syncers.append(SQLBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
@@ -85,7 +85,7 @@ def main():
     for syncer_backend in syncer_backends:
         syncers.append(HistorySyncer(syncer_backend))
 
-    syncer_backend = SyncerBackend.live(chain_spec, block_offset+1)
+    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
     syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')

apps/cic-cache/docker/Dockerfile

@@ -47,6 +47,9 @@ RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
 	mkdir -p /usr/local/share/cic/solidity && \
 	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
 
+COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
+COPY cic-cache/docker/db.sh ./db.sh
+RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server

apps/cic-cache/docker/db.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+>&2 echo executing database migration
+python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
+set +e

apps/cic-cache/docker/start_tracker.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. ./db.sh
+
+/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/requirements.txt

@@ -1,13 +1,12 @@
-cic-base~=0.1.2a66
+cic-base~=0.1.2a77
 alembic==1.4.2
 confini~=0.3.6rc3
 uwsgi==2.0.19.1
 moolb~=0.1.0
-cic-eth-registry~=0.5.4a13
+cic-eth-registry~=0.5.4a16
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainlib~=0.0.2a10
-chainsyncer[sql]~=0.0.2a1
+chainsyncer[sql]~=0.0.2a2

apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py

@@ -15,7 +15,6 @@ from cic_eth_registry import CICRegistry
 from chainlib.chain import ChainSpec
 from chainlib.eth.tx import unpack
 from chainlib.connection import RPCConnection
-from chainsyncer.error import SyncDone
 from hexathon import strip_0x
 from chainqueue.db.enum import (
     StatusEnum,
@@ -153,10 +152,7 @@ class DispatchSyncer:
 def main(): 
     syncer = DispatchSyncer(chain_spec)
     conn = RPCConnection.connect(chain_spec, 'default')
-    try:
-        syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
-    except SyncDone as e:
-        sys.stderr.write("dispatcher done at block {}\n".format(e))
+    syncer.loop(conn, float(config.get('DISPATCHER_LOOP_INTERVAL')))
 
     sys.exit(0)
 

apps/cic-eth/cic_eth/runnable/daemons/tracker.py

@@ -26,7 +26,7 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend import SyncerBackend
+from chainsyncer.backend.sql import SQLBackend
 from chainsyncer.driver import (
         HeadSyncer,
         HistorySyncer,
@@ -88,18 +88,18 @@ def main():
 
     syncers = []
 
-    #if SyncerBackend.first(chain_spec):
-    #    backend = SyncerBackend.initial(chain_spec, block_offset)
-    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
+    #if SQLBackend.first(chain_spec):
+    #    backend = SQLBackend.initial(chain_spec, block_offset)
+    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
+    syncer_backends.append(SQLBackend.live(chain_spec, block_offset+1))
 
     for syncer_backend in syncer_backends:
         try:

apps/cic-eth/cic_eth/sync/retry.py

@@ -4,7 +4,7 @@ import datetime
 
 # external imports
 from chainsyncer.driver import HeadSyncer
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.error import NoBlockForYou
 from chainlib.eth.block import (
         block_by_number,

apps/cic-eth/docker/Dockerfile

@@ -29,7 +29,8 @@ RUN /usr/local/bin/python -m pip install --upgrade pip
 #	python merge_requirements.py | tee merged_requirements.txt 
 #RUN cd cic-base && \
 #	pip install $pip_extra_index_url_flag -r ./merged_requirements.txt 
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
+RUN pip install $pip_extra_index_url_flag   chainsyncer~=0.0.2a2
 
 COPY cic-eth/scripts/ scripts/
 COPY cic-eth/setup.cfg cic-eth/setup.py ./

apps/cic-eth/requirements.txt

@@ -18,7 +18,7 @@ moolb~=0.1.1b2
 eth-address-index~=0.1.1a7
 chainlib~=0.0.2a5
 hexathon~=0.0.1a7
-chainsyncer[sql]~=0.0.2a1
+chainsyncer[sql]~=0.0.2a2
 chainqueue~=0.0.1a7
 pysha3==1.0.2
 coincurve==15.0.0

apps/contract-migration/docker/Dockerfile

@@ -57,9 +57,9 @@ WORKDIR /home/grassroots
 USER grassroots
 
 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
-ARG cic_base_version=0.1.2a67
-ARG cic_eth_version=0.11.0b1
-ARG sarafu_faucet_version=0.0.2a19
+ARG cic_base_version=0.1.2a77
+ARG cic_eth_version=0.11.0b6
+ARG sarafu_faucet_version=0.0.2a28
 ARG cic_contracts_version=0.0.2a2
 RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
 	cic-eth==$cic_eth_version \

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -17,7 +17,7 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
@@ -25,13 +25,13 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer

apps/contract-migration/scripts/eth/import_balance.py

@@ -17,7 +17,7 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
 from chainlib.eth.connection import EthHTTPConnection
 from chainlib.eth.block import (
@@ -25,13 +25,13 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer

apps/contract-migration/scripts/requirements.txt

@@ -1,5 +1,5 @@
-cic-base[full_graph]==0.1.2a67
-sarafu-faucet==0.0.2a20
-cic-eth==0.11.0b3
+cic-base[full_graph]==0.1.2a77
+sarafu-faucet==0.0.2a28
+cic-eth==0.11.0b6
 cic-types==0.1.0a10
-crypto-dev-signer==0.4.14b1
+crypto-dev-signer==0.4.14b2

apps/contract-migration/scripts/verify.py

@@ -32,7 +32,7 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
@@ -40,7 +40,7 @@ from chainlib.eth.gas import (
         balance,
         )
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from cic_types.models.person import (
         Person,

docker-compose.yml

@@ -145,7 +145,7 @@ services:
       - -c
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-trackerd -vv -c /usr/local/etc/cic-cache
+        ./start_tracker.sh -c /usr/local/etc/cic-cache -vv
     volumes:
       - contract-config:/tmp/cic/config/:ro
 
@@ -191,13 +191,13 @@ services:
         context: apps
         dockerfile: cic-cache/docker/Dockerfile
     environment:
-      DATABASE_USER: $DATABASE_USER
-      DATABASE_HOST: $DATABASE_HOST
-      DATABASE_PORT: $DATABASE_PORT
-      DATABASE_PASSWORD: $DATABASE_PASSWORD
-      DATABASE_NAME: $DATABASE_NAME_CIC_CACHE
+      DATABASE_USER: ${DATABASE_USER:-grassroots}
+      DATABASE_HOST: ${DATABASE_HOST:-postgres}
+      DATABASE_PORT: ${DATABASE_PORT:-5432}
+        #DATABASE_PASSWORD: ${DATABASE_PASSWORD:-
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
       DATABASE_DEBUG: 1
-      PGPASSWORD: $DATABASE_PASSWORD
+        #PGPASSWORD: $DATABASE_PASSWORD
       SERVER_PORT: 8000
     ports:
       - ${HTTP_PORT_CIC_CACHE:-63313}:8000
@@ -212,9 +212,10 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \
         --http :8000 \
-        --pyargv -vv
+        --pyargv "-vv"
+
 
   cic-eth-tasker:
     # image: grassrootseconomics:cic-eth-service