WIP rehabilitate cic-cache

This commit is contained in:
nolash 2021-10-16 08:23:32 +02:00
parent d3576c8ec7
commit 9a154a8046
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
6 changed files with 241 additions and 85 deletions

View File

@ -9,6 +9,7 @@ import celery
import confini
# local imports
import cic_cache.cli
from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase
from cic_cache.tasks.tx import *
@ -16,25 +17,34 @@ from cic_cache.tasks.tx import *
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
config_dir = os.path.join('/usr/local/etc/cic-cache')
argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
# process args
arg_flags = cic_cache.cli.argflag_std_base
local_arg_flags = cic_cache.cli.argflag_local_task
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
if args.vv:
logging.getLogger().setLevel(logging.DEBUG)
elif args.v:
logging.getLogger().setLevel(logging.INFO)
# process config
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
config = confini.Config(args.c, args.env_prefix)
config.process()
## process config
#config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
#argparser = argparse.ArgumentParser()
#argparser.add_argument('-c', type=str, default=config_dir, help='config file')
#argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
#argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
#argparser.add_argument('-v', action='store_true', help='be verbose')
#argparser.add_argument('-vv', action='store_true', help='be more verbose')
#
#args = argparser.parse_args()
#
#if args.vv:
# logging.getLogger().setLevel(logging.DEBUG)
#elif args.v:
# logging.getLogger().setLevel(logging.INFO)
#
#config = confini.Config(args.c, args.env_prefix)
#config.process()
# connect to database
dsn = dsn_from_config(config)

View File

@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
# process args
arg_flags = cic_cache.cli.argflag_std_read
arg_flags = cic_cache.cli.argflag_std_base
local_arg_flags = cic_cache.cli.argflag_local_sync
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)

View File

@ -1,19 +1,18 @@
# syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
ARG DOCKER_REGISTRY=registry.gitlab.com/grassrootseconomics
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
COPY requirements.txt .
#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
#RUN pip install $pip_extra_index_url_flag .
#RUN pip install .[server]
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_ARGS=""
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url https://pypi.org/simple \
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
ARG PIP_INDEX_URL="https://pypi.org/simple"
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \
--no-cache \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt
COPY . .
@ -23,7 +22,7 @@ RUN python setup.py install
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
COPY config/ /usr/local/etc/cic-cache/
#COPY config/ /usr/local/etc/cic-cache/
# for db migrations
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/

View File

@ -1,27 +1,16 @@
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e as dev
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
# Copy just the requirements and install....this _might_ give docker a hint on caching but we
# do load these all into setup.py later
# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
#COPY cic-eth/requirements.txt .
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"
ARG PIP_INDEX_URL=https://pypi.org/simple
#ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
#ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
#ARG EXTRA_PIP_ARGS=""
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
# pip install --index-url https://pypi.org/simple \
# --force-reinstall \
# --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
# -r requirements.txt
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \
@ -31,7 +20,6 @@ RUN pip install --index-url $PIP_INDEX_URL \
COPY *requirements.txt ./
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \
@ -40,15 +28,7 @@ RUN pip install --index-url $PIP_INDEX_URL \
-r requirements.txt \
-r services_requirements.txt \
-r admin_requirements.txt
# always install the latest signer
#RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
# pip install --index-url https://pypi.org/simple \
# --extra-index-url $GITLAB_PYTHON_REGISTRY \
# --extra-index-url $EXTRA_INDEX_URL \
# $EXTRA_PIP_ARGS \
# crypto-dev-signer
COPY . .
RUN python setup.py install

View File

@ -1,17 +1,14 @@
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e as dev
ARG DOCKER_REGISTRY=registry.gitlab.com/grassrootseconomics
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-55da5f4e
WORKDIR /root
COPY requirements.txt .
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"
ARG pip_trusted_host=pypi.org
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--force-reinstall \

View File

@ -85,12 +85,11 @@ services:
build:
context: apps/cic-signer
dockerfile: Dockerfile
target: dev
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
@ -110,18 +109,17 @@ services:
- contract-config:/tmp/cic/config/:ro
command: ["python", "/usr/local/bin/funga-ethd", "-vv"]
# queue handling for outgoing transactions and incoming transactions
cic-eth-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest}
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
@ -167,12 +165,11 @@ services:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
@ -192,6 +189,7 @@ services:
CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth}
SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-5}
SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1}
SYNCER_OFFSET: ${SYNCER_OFFSET:-0}
TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
restart: unless-stopped
depends_on:
@ -215,12 +213,11 @@ services:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
@ -260,12 +257,11 @@ services:
build:
context: apps/cic-eth
dockerfile: docker/Dockerfile
target: dev
args:
DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: $EXTRA_PIP_INDEX_URL
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
@ -300,3 +296,177 @@ services:
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
./start_retry.sh -vv
cic-cache-tracker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_cache}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
CELERY_DEBUG: ${CELERY_DEBUG:-1}
CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth}
SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-5}
SYNCER_OFFSET: ${SYNCER_OFFSET:-0}
SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1}
TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"}
restart: unless-stopped
depends_on:
- evm
- postgres
- redis
volumes:
- contract-config:/tmp/cic/config/:ro
command:
- /bin/bash
- -c
- |
set -a
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
./start_tracker.sh -vv
cic-cache-tasker:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
CHAIN_SPEC: ${CHAIN_SPEC:-"evm:byzantium:8996:bloxberg"}
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_cache}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis}
CELERY_DEBUG: ${CELERY_DEBUG:-1}
CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth}
restart: unless-stopped
depends_on:
- evm
- postgres
- redis
volumes:
- contract-config:/tmp/cic/config/:ro
command:
- /bin/bash
- -c
- |
set -a
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
/usr/local/bin/cic-cache-taskerd -vv
# "/usr/local/bin/uwsgi" \
# --wsgi-file /root/cic_cache/runnable/daemons/server.py \
# --http :8000 \
# --pyargv "-vv"
cic-cache-server:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest}
build:
context: apps/cic-cache
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_cache}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
SERVER_PORT: 8000
restart: on-failure
ports:
- ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000
depends_on:
- postgres
- cic-cache-tracker
command:
- |
set -a
if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi
set +a
#/usr/local/bin/uwsgi
volumes:
- contract-config:/tmp/cic/config/:ro
# metadata replacement server for swarm
cic-meta-server:
image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-meta:${TAG:-latest}
hostname: meta
build:
context: apps/cic-meta
dockerfile: docker/Dockerfile
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
DATABASE_PORT: ${DATABASE_PORT:-5432}
DATABASE_NAME: ${DATABASE_NAME:-cic_meta}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala}
DATABASE_USER: ${DATABASE_USER:-grassroots}
DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
DATABASE_POOL_SIZE: 0
DATABASE_SCHEMA_SQL_PATH: ""
SERVER_HOST: localhost
SERVER_PORT: 8000
PGP_EXPORTS_DIR: /root/tests/
PGP_PRIVATEKEY_FILE: privatekeys.asc
PGP_PASSPHRASE: merman
PGP_PUBLICKEY_TRUSTED_FILE: publickeys.asc
PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc
PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc
SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql
restart: on-failure
ports:
- ${DEV_CIC_META_SERVER_PORT:-63380}:8000
depends_on:
- postgres
volumes:
- ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp