cic-cache migrated

Correct db names and ports

See merge request grassrootseconomics/cic-internal-integration!25

.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

.gitmodules

@@ -1,6 +0,0 @@
-[submodule "apps/cic-cache"]
-	path = apps/cic-cache
-	url = git@gitlab.com:grassrootseconomics/cic-cache.git
-[submodule "apps/cic-meta"]
-	path = apps/cic-meta
-	url = git@gitlab.com:grassrootseconomics/cic-meta.git

README.md

@@ -2,6 +2,13 @@
 
 ## Getting started 
 
+## Make some keys
+
+```
+docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
+```
+
+
 ### Prepare the repo
 
 This is stuff we need to put in  makefile but for now...

apps/bloxbergValidatorSetup/.gitignore

@@ -1,3 +1,6 @@
 /validator/bloxbergData
 /validator/bloxberg.log
-keys/**/*
+keys/*
+!keys/Bloxberg
+keys/Bloxberg/*
+!keys/Bloxberg/UTC--2021-02-10T16-57-35Z--03512a62-5334-20cc-4e44-71156f33cff6

apps/bloxbergValidatorSetup/Dockerfile

@@ -17,7 +17,7 @@ COPY ./validator/bloxberg.json \
      ./validator/validator.toml \
      /root/
 
-COPY ./keys/ /root/keys/
+COPY keys/ /root/keys/
 
 # RUN chown -R parity:parity $HOME/ && \
 #     chmod -R 775 $HOME/ && \
@@ -25,4 +25,4 @@ COPY ./keys/ /root/keys/
 # USER parity
 
 ENTRYPOINT [ "parity" ]
-CMD [ "--config", "/root/validator.toml", "--keys-path", "/root/keys/" ]
+CMD [ "--config", "/root/validator.toml", "--keys-path", "/root/keys/", "--password", "/root/validator.pwd" ]

apps/bloxbergValidatorSetup/README.md

@@ -4,7 +4,7 @@
 The original bloxberg node config was kind of annoying so I am running it more like vanilla parity. This way you can pass command flags directly to parity.
 ## Make some keys
 ```
-docker build -t bloxie . && docker run -v ${PWD}/keys:/root/keys --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
+docker build -t bloxie . && docker run -v ${PWD}/keys:/root/keys --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys --password /root/validator.pwd
 ```
 
 ## Enter the signer address and passwords in the config files

apps/bloxbergValidatorSetup/keys/Bloxberg/UTC--2021-02-10T16-57-35Z--03512a62-5334-20cc-4e44-71156f33cff6

@@ -0,0 +1 @@
+{"id":"03512a62-5334-20cc-4e44-71156f33cff6","version":3,"crypto":{"cipher":"aes-128-ctr","cipherparams":{"iv":"dc388338c4d4e3203604aeb3d1c6bbfa"},"ciphertext":"8a945775b87089ce94537e011799f3abc1577c5dd1f3fbaebe1cd96dfdfc8b5a","kdf":"pbkdf2","kdfparams":{"c":10240,"dklen":32,"prf":"hmac-sha256","salt":"e8585836540caca01282381f5c1fe128e53b15b40f9d152fbc5a4f82a7967398"},"mac":"a7c7815e84a632ecf6d8f18c981bea73d50cd2e2a855a3e90477fc84ed14f906"},"address":"4f2a5902158c3969b245247f4154971d393301f2","name":"","meta":"{}"}

apps/bloxbergValidatorSetup/validator/bloxberg.json

@@ -7,7 +7,7 @@
                 "maximumUncleCount": 0,
                 "stepDuration": "5",
                 "validators" : {
-                     "list": ["0x6bd4e51b3730576ddc4049654ef60ed7f7436cb5"]
+                     "list": ["0x4f2a5902158c3969b245247f4154971d393301f2"]
                 }
             }
         }

apps/bloxbergValidatorSetup/validator/validator.toml

@@ -26,7 +26,7 @@ password = ["/root/validator.pwd"]
 
 [mining]
 #CHANGE ENGINE SIGNER TO VALIDATOR ADDRESS
-engine_signer = "0x6bd4e51b3730576ddc4049654ef60ed7f7436cb5"
+engine_signer = "0x4f2a5902158c3969b245247f4154971d393301f2"
 reseal_on_txs = "none"
 force_sealing = true
 min_gas_price = 1000000

apps/cic-eth/cic_eth/eth/util.py

@@ -35,6 +35,10 @@ def unpack_signed_raw_tx(tx_raw_bytes, chain_id):
     if chain_id != 0:
         v = int.from_bytes(d[6], 'big')
         vb = v - (chain_id * 2) - 35
+    while len(d[7]) < 32:
+        d[7] = b'\x00' + d[7]
+    while len(d[8]) < 32:
+        d[8] = b'\x00' + d[8]
     s = b''.join([d[7], d[8], bytes([vb])])
     so = KeyAPI.Signature(signature_bytes=s)
 

apps/cic-eth/cic_eth/queue/tx.py

@@ -557,27 +557,26 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
     session = SessionBase.create_session()
-    q = session.query(
+    q_outer = session.query(
             TxCache.sender,
             func.min(Otx.nonce).label('nonce'),
             )
-    q = q.join(TxCache)
+    q_outer = q_outer.join(TxCache)
-    q = q.join(Lock, isouter=True)
+    q_outer = q_outer.join(Lock, isouter=True)
-    q = q.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
+    q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
     if status >= StatusEnum.SENT:
         raise ValueError('not a valid non-final tx value: {}'.format(s))
-    q = q.filter(Otx.status==status)
+    q_outer = q_outer.filter(Otx.status==status.value)
 
     if recipient != None:
-        q = q.filter(TxCache.recipient==recipient)
+        q_outer = q_outer.filter(TxCache.recipient==recipient)
 
-    q = q.group_by(TxCache.sender)
+    q_outer = q_outer.group_by(TxCache.sender)
 
     txs = {}
 
-    results = q.all()
+    for r in q_outer.all():
-    for r in results:
         q = session.query(Otx)
         q = q.join(TxCache)
         q = q.filter(TxCache.sender==r.sender)
@@ -587,7 +586,6 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
             q = q.filter(TxCache.date_checked<before)
        
         q = q.order_by(TxCache.date_created.desc())
-
         o = q.first()
 
         # TODO: audit; should this be possible if a row is found in the initial query? If not, at a minimum log error.
@@ -602,7 +600,6 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
         q = q.filter(TxCache.otx_id==o.id)
         o = q.first()
 
-        logg.debug('oooo {}'.format(o))
         o.date_checked = datetime.datetime.now()
         session.add(o)
         session.commit()

apps/cic-eth/cic_eth/runnable/dispatcher.py

@@ -68,7 +68,7 @@ app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get(
 queue = args.q
 
 dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
+SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
 
 re_websocket = re.compile('^wss?://')

apps/cic-eth/docker/Dockerfile

@@ -41,3 +41,8 @@ COPY cic-eth/tests/ tests/
 COPY cic-eth/config/ /usr/local/etc/cic-eth/
 COPY cic-eth/cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
 COPY cic-eth/crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
+
+RUN apt-get install -y git && \
+	git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+	mkdir -p /usr/local/share/cic/solidity && \
+	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi

apps/cic-notify/docker/Dockerfile

@@ -14,7 +14,9 @@ RUN pip install -r $root_requirement_file $pip_extra_index_url_flag
 COPY cic-notify/setup.cfg \
 	 cic-notify/setup.py \
      ./
+
 COPY cic-notify/cic_notify/ ./cic_notify/
+
 COPY cic-notify/requirements.txt \
 	 cic-notify/test_requirements.txt \
      ./

apps/cic-notify/docker/Dockerfile.service

@@ -1,13 +0,0 @@
-FROM grassrootseconomics:cic-notify
-
-#FROM python:3.8.6-alpine
-
-#RUN apk update && \
-#	apk add gnupg libpq
-
-#COPY --from=0 /usr/local/ /usr/local/
-#COPY --from=0 /root/ /root/
-
-#RUN apk add bash
-
-WORKDIR /root

apps/contract-migration/reset.sh

@@ -19,7 +19,7 @@ echo \n
 # pushd /usr/src
 
 init_level_file=${CIC_DATA_DIR}/.init
-echo -n 0 > $init_level_file
+echo -n 1 > $init_level_file
 
 # Abort on any error (including if wait-for-it fails).
 set -e
@@ -68,6 +68,6 @@ cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all
 set +a
 set +e
 
-echo -n 1 > $init_level_file
+echo -n 2 > $init_level_file
 
 exec "$@"

apps/contract-migration/seed_cic_eth.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
 
 # defaults
+initlevel=`cat ${CIC_DATA_DIR}/.init`
+echo $inilevel
+if [ $initlevel -lt 2 ]; then
+	>&2 echo "initlevel too low $initlevel"
+	exit 1 
+fi
 source ${CIC_DATA_DIR}/.env
 source ${CIC_DATA_DIR}/.env_all
 DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
@@ -136,7 +142,7 @@ token_description_two=0x54686973206973207468652053617261667520746f6b656e00000000
 >&2 echo "add keystore account $keystore_file to accounts index writers"
 >&2 eth-accounts-index-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_ACCOUNTS_INDEX_ADDRESS --writer $DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER -w $debug
 
-echo -n 2 > $init_level_file
+echo -n 0 > $init_level_file
 
 set +a
 set +e

docker-compose.yml

@@ -35,11 +35,11 @@ services:
       context: apps/bloxbergValidatorSetup
     restart: unless-stopped
     ports:
-      - ${DEV_ETH_PORT_HTTP:-8545}:8545
+      - ${DEV_ETH_PORT_HTTP:-63545}:8545
-      - ${DEV_ETH_PORT_WS-8546}:8546
+      - ${DEV_ETH_PORT_WS-63546}:8546
       - 30303
     volumes:
-      #- ./keys:/root/keys # stores the signing key locally
+      - ./apps/bloxbergValidatorSetup/keys:/root/keys # stores the signing key locally
       - bloxberg-data:/root/.local/share/io.parity.ethereum/
 
   # See contents of /initdb/create_db.sql for app user, password and databases
@@ -49,7 +49,7 @@ services:
       POSTGRES_HOST_AUTH_METHOD: trust # for postgres user access w/o password. Obvioulsy not safe but allows easy elevated debugging.  
       # PGDATA: /tmp/cic/postgres
     ports:
-      - ${DEV_POSTGRES_PORT:-5432}:5432
+      - ${DEV_POSTGRES_PORT:-63432}:5432
     volumes:
       - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql 
       - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
@@ -58,7 +58,7 @@ services:
   redis:
     image: redis:6.0.9-alpine
     ports:
-      - ${DEV_REDIS_PORT:-6379}:6379
+      - ${DEV_REDIS_PORT:-63379}:6379
     command: "--loglevel verbose"
 
   bee:
@@ -68,8 +68,8 @@ services:
       BEE_NETWORK_ID: ${BEE_NETWORK_ID:-313}
       BEE_PASSWORD: ${BEE_PASSWORD:-password}
     ports:
-      - ${DEV_BEE_PORT:-1633}:1633
+      - ${DEV_BEE_PORT:-63633}:1633
-      - ${DEV_BEE_PORT_DEBUG:-1635}:1635
+      - ${DEV_BEE_PORT_DEBUG:-63635}:1635
     command: "start --swap-enable=false --standalone"
     volumes:
       - bee-data:/tmp/cic/bee
@@ -117,8 +117,14 @@ services:
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     command: ["./seed_cic_eth.sh"]
+    deploy:
+      restart_policy:
+        condition: on-failure
     depends_on:
       - eth
+      - postgres
+      - redis
+      - cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
 
@@ -227,7 +233,7 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
@@ -263,7 +269,7 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
@@ -299,7 +305,7 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
@@ -309,6 +315,7 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
+      DATABASE_DEBUG: ${DATABASE_DEBUG:-true}
 
     depends_on:
       - eth
@@ -337,7 +344,7 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
@@ -347,6 +354,7 @@ services:
       CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
       TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS
+      CIC_TX_RETRY_DELAY: 15
     depends_on:
       - eth
       - postgres
@@ -373,11 +381,13 @@ services:
       CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
       CELERY_BROKER_URL: $CELERY_BROKER_URL
       CELERY_RESULT_URL: $CELERY_RESULT_URL
-      SERVER_PORT: 80
+      SERVER_PORT: 8000
     depends_on:
       - eth
       - postgres
       - redis
+    ports:
+      - ${HTTP_PORT_CIC_ETH:-63314}:8000
     deploy:
       restart_policy:
         condition: on-failure
@@ -403,7 +413,9 @@ services:
 
 
   cic-notify-tasker:
-    image: grassrootseconomics:cic-notify-service
+    build:
+      context: apps/
+      dockerfile: cic-notify/docker/Dockerfile
     environment:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
@@ -438,8 +450,8 @@ services:
       DATABASE_USER: ${DATABASE_USER:-grassroots}
       DATABASE_HOST: ${DATABASE_HOST:-postgres}
       DATABASE_PORT: ${DATABASE_PORT:-5432}
-      SERVER_HOST: ${SERVER_HOST:-localhost}
+      SERVER_HOST: localhost
-      SERVER_PORT: ${SERVER_HOST:-80}
+      SERVER_PORT: 8000
       DATABASE_SCHEMA_SQL_PATH: ""
       PGP_EXPORTS_DIR: /tmp/src/cic-meta/tests/
       PGP_PRIVATEKEY_FILE: privatekeys.asc
@@ -448,7 +460,7 @@ services:
       PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc
       PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc
     ports:
-      - ${HTTP_PORT_CIC_META:-63380}:${SERVER_PORT:-80}
+      - ${HTTP_PORT_CIC_META:-63380}:8000
     depends_on:
       - postgres
     deploy:
@@ -471,9 +483,9 @@ services:
       DATABASE_NAME: cic_ussd
       DATABASE_ENGINE: postgresql
       DATABASE_DRIVER: psycopg2
-      SERVER_PORT: 9000
+      SERVER_PORT: 8000
     ports:
-      - 9000:8082
+      - ${HTTP_PORT_CIC_USSD:-63315}:8000
     depends_on:
       - postgres
       - redis