From 94d3e61d0cf36e95beef3150cfb8ace696fd0e4e Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 7 Mar 2021 12:25:36 -0800 Subject: [PATCH 01/20] revert the new requirements for ussd --- apps/cic-ussd/requirements.txt | 37 ++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 53b68551..f2887130 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,21 +1,46 @@ -cic_base[full-graph]~=0.1.1a19 alembic==1.4.2 +amqp==2.6.1 +attrs==20.2.0 bcrypt==3.2.0 +betterpath==0.2.2 +billiard==3.6.3.0 celery==4.4.7 -chainlib~=0.0.1a20 -cic-eth~=0.10.0a40 -cic-notify~=0.4.0a2 -cic-types==0.1.0a8 -confini~=0.3.6rc3 +cffi==1.14.3 +cic-eth~=0.10.0a22 +cic-notify==0.3.1 +click==7.1.2 +confini~=0.3.6a1 cryptography==3.2.1 faker==4.17.1 +iniconfig==1.1.1 +kombu==4.6.11 +Mako==1.1.3 +MarkupSafe==1.1.1 +mirakuru==2.3.0 +more-itertools==8.5.0 +packaging==20.4 phonenumbers==8.12.12 +pluggy==0.13.1 +port-for==0.4 +psutil==5.7.3 psycopg2==2.8.6 +py==1.9.0 +pycparser==2.20 +pyparsing==2.4.7 +python-dateutil==2.8.1 +python-editor==1.0.4 python-gnupg==0.4.6 python-i18n==0.3.9 +pytz==2020.1 +PyYAML==5.3.1 redis==3.5.3 semver==2.13.0 +six==1.15.0 SQLAlchemy==1.3.20 tinydb==4.2.0 +toml==0.10.1 transitions==0.8.4 uWSGI==2.0.19.1 +vcversioner==2.16.0.0 +vine==1.3.0 +zope.interface==5.1.2 From bb90ceea0b68110b3331fd6d051953c1e5348591 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 7 Mar 2021 12:54:41 -0800 Subject: [PATCH 02/20] cic types --- apps/cic-ussd/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index f2887130..00dc2bc1 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,3 +1,4 @@ +cic-types==0.1.0a8 alembic==1.4.2 amqp==2.6.1 attrs==20.2.0 From 29da44bb9f016355a451d31396425ceacc19fccb Mon Sep 17 00:00:00 2001 From: PhilipWafula Date: Tue, 9 Mar 2021 19:05:01 +0300 Subject: [PATCH 03/20] Add barebone session resumption feature --- apps/cic-ussd/cic_ussd/operations.py | 42 ++++++++++++++++++++-------- apps/cic-ussd/cic_ussd/processor.py | 30 ++++++++++++++++++-- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/apps/cic-ussd/cic_ussd/operations.py b/apps/cic-ussd/cic_ussd/operations.py index f02b460b..b7b7e0ef 100644 --- a/apps/cic-ussd/cic_ussd/operations.py +++ b/apps/cic-ussd/cic_ussd/operations.py @@ -14,7 +14,7 @@ from cic_ussd.db.models.user import User from cic_ussd.db.models.ussd_session import UssdSession from cic_ussd.db.models.task_tracker import TaskTracker from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.processor import custom_display_text, process_request +from cic_ussd.processor import custom_display_text, process_request, retrieve_most_recent_ussd_session from cic_ussd.redis import InMemoryStore from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession from cic_ussd.validator import check_known_user, validate_response_type @@ -60,7 +60,8 @@ def create_ussd_session( phone: str, service_code: str, user_input: str, - current_menu: str) -> InMemoryUssdSession: + current_menu: str, + session_data: Optional[dict] = None) -> InMemoryUssdSession: """ Creates a new ussd session :param external_session_id: Session id value provided by AT @@ -73,6 +74,8 @@ def create_ussd_session( :type user_input: str :param current_menu: Menu name that is currently being displayed on the ussd session :type current_menu: str + :param session_data: Any additional data that was persisted during the user's interaction with the system. + :type session_data: dict. :return: ussd session object :rtype: Session """ @@ -81,7 +84,8 @@ def create_ussd_session( msisdn=phone, user_input=user_input, state=current_menu, - service_code=service_code + service_code=service_code, + session_data=session_data ) return session @@ -126,7 +130,9 @@ def create_or_update_session( phone=phone, service_code=service_code, user_input=user_input, - current_menu=current_menu) + current_menu=current_menu, + session_data=session_data + ) return ussd_session @@ -338,14 +344,26 @@ def process_menu_interaction_requests(chain_str: str, user_input=user_input ) - # create or update the ussd session as appropriate - ussd_session = create_or_update_session( - external_session_id=external_session_id, - phone=phone_number, - service_code=service_code, - user_input=user_input, - current_menu=current_menu.get('name') - ) + last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number) + + if last_ussd_session: + # create or update the ussd session as appropriate + ussd_session = create_or_update_session( + external_session_id=external_session_id, + phone=phone_number, + service_code=service_code, + user_input=user_input, + current_menu=current_menu.get('name'), + session_data=last_ussd_session.session_data + ) + else: + ussd_session = create_or_update_session( + external_session_id=external_session_id, + phone=phone_number, + service_code=service_code, + user_input=user_input, + current_menu=current_menu.get('name') + ) # define appropriate response response = custom_display_text( diff --git a/apps/cic-ussd/cic_ussd/processor.py b/apps/cic-ussd/cic_ussd/processor.py index 5087fb18..32e69a3e 100644 --- a/apps/cic-ussd/cic_ussd/processor.py +++ b/apps/cic-ussd/cic_ussd/processor.py @@ -6,7 +6,7 @@ from typing import Optional # third party imports import celery -from cic_types.models.person import Person +from sqlalchemy import desc from tinydb.table import Document # local imports @@ -315,6 +315,16 @@ def process_start_menu(display_key: str, user: User): ) +def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession: + # get last ussd session based on user phone number + last_ussd_session = UssdSession.session\ + .query(UssdSession)\ + .filter_by(msisdn=phone_number)\ + .order_by(desc(UssdSession.created))\ + .first() + return last_ussd_session + + def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document: """This function assesses a request based on the user from the request comes, the session_id and the user's input. It determines whether the request translates to a return to an existing session by checking whether the @@ -337,7 +347,23 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] = return UssdMenu.find_by_name(name=successive_state) else: if user.has_valid_pin(): - return UssdMenu.find_by_name(name='start') + last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number) + + key = create_cached_data_key( + identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), + salt='cic.person' + ) + user_metadata = get_cached_data(key=key) + + if last_ussd_session: + # get last state + last_state = last_ussd_session.state + logg.debug(f'LAST USSD SESSION STATE: {last_state}') + # if last state is account_creation_prompt and metadata exists, show start menu + if last_state == 'account_creation_prompt' and user_metadata is not None: + return UssdMenu.find_by_name(name='start') + else: + return UssdMenu.find_by_name(name=last_state) else: if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name: return UssdMenu.find_by_name(name='exit_pin_blocked') From 16d88d389b8e1cf9f62cfcd741ada3e724e1efe4 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Tue, 9 Mar 2021 16:54:33 -0800 Subject: [PATCH 04/20] move envlist to dockercontainer --- apps/contract-migration/docker/Dockerfile | 3 +- apps/contract-migration/envlist | 61 +++++++++++++++++++++++ apps/contract-migration/reset.sh | 2 +- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 apps/contract-migration/envlist diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 9b589c04..0de65735 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -127,5 +127,6 @@ COPY contract-migration/from_env.sh from_env.sh COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore +COPY contract-migration/envlist . -ENTRYPOINT [ "/bin/bash" ] +# ENTRYPOINT [ "/bin/bash" ] diff --git a/apps/contract-migration/envlist b/apps/contract-migration/envlist new file mode 100644 index 00000000..8613215b --- /dev/null +++ b/apps/contract-migration/envlist @@ -0,0 +1,61 @@ +SYNCER_LOOP_INTERVAL +SSL_ENABLE_CLIENT +SSL_CERT_FILE +SSL_KEY_FILE +SSL_PASSWORD +SSL_CA_FILE +BANCOR_DIR +REDIS_HOST +REDIS_PORT +REDIS_DB +PGP_EXPORTS_DIR +PGP_PRIVATEKEY_FILE +PGP_PASSPHRASE +DATABASE_USER +DATABASE_PASSWORD +DATABASE_NAME +DATABASE_HOST +DATABASE_PORT +DATABASE_ENGINE +DATABASE_DRIVER +DATABASE_DEBUG +TASKS_AFRICASTALKING +TASKS_SMS_DB +TASKS_LOG +TASKS_TRACE_QUEUE_STATUS +TASKS_TRANSFER_CALLBACKS +DEV_MNEMONIC +DEV_ETH_RESERVE_ADDRESS +DEV_ETH_ACCOUNTS_INDEX_ADDRESS +DEV_ETH_RESERVE_AMOUNT +DEV_ETH_ACCOUNT_BANCOR_DEPLOYER +DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER +DEV_ETH_ACCOUNT_GAS_PROVIDER +DEV_ETH_ACCOUNT_RESERVE_OWNER +DEV_ETH_ACCOUNT_RESERVE_MINTER +DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_OWNER +DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER +DEV_ETH_ACCOUNT_SARAFU_OWNER +DEV_ETH_ACCOUNT_SARAFU_GIFTER +DEV_ETH_ACCOUNT_APPROVAL_ESCROW_OWNER +DEV_ETH_ACCOUNT_SINGLE_SHOT_FAUCET_OWNER +DEV_ETH_SARAFU_TOKEN_NAME +DEV_ETH_SARAFU_TOKEN_SYMBOL +DEV_ETH_SARAFU_TOKEN_DECIMALS +DEV_ETH_SARAFU_TOKEN_ADDRESS +DEV_PGP_PUBLICKEYS_ACTIVE_FILE +DEV_PGP_PUBLICKEYS_TRUSTED_FILE +DEV_PGP_PUBLICKEYS_ENCRYPT_FILE +CIC_REGISTRY_ADDRESS +CIC_APPROVAL_ESCROW_ADDRESS +CIC_TOKEN_INDEX_ADDRESS +CIC_ACCOUNTS_INDEX_ADDRESS +CIC_DECLARATOR_ADDRESS +CIC_CHAIN_SPEC +ETH_PROVIDER +ETH_ABI_DIR +SIGNER_SOCKET_PATH +SIGNER_SECRET +CELERY_BROKER_URL +CELERY_RESULT_URL +META_PROVIDER diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index d76d2441..8768921f 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -64,7 +64,7 @@ export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER EOF -cat $CIC_DATA_DIR/envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all +cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all # popd set +a From 3cdf7b9965a6587592f13111e0c438453cc415eb Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Wed, 10 Mar 2021 03:48:17 +0000 Subject: [PATCH 05/20] Update ci_templates/.cic-template.yml --- ci_templates/.cic-template.yml | 59 ++++++++++++---------------------- 1 file changed, 20 insertions(+), 39 deletions(-) diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index 57ea3a02..67b38fb1 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -1,54 +1,35 @@ -image: docker:19.03.13 +image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] variables: - # docker host - DOCKER_HOST: tcp://docker:2376 - # container, thanks to volume mount from config.toml - DOCKER_TLS_CERTDIR: "/certs" - # These are usually specified by the entrypoint, however the - # Kubernetes executor doesn't run entrypoints - # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125 - DOCKER_TLS_VERIFY: 1 - DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" - # We are building these from the apps dir to easily share the requirements file there. - # It would be nicer to build from the app dir context. TODO figure out a nice way to do this in local DOCKER_TLS_VERIFY - CONTEXT: apps/ - -services: - - docker:19.03.13-dind - -before_script: - - docker info + KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h" + CONTEXT: $CI_PROJECT_DIR/apps/ .py_build_merge_request: stage: build - before_script: - - cd $CONTEXT variables: - CI_DEBUG_TRACE: "true" - IMAGE_TAG: $APP_NAME:$CI_COMMIT_SHORT_SHA + - CI_DEBUG_TRACE: "true" script: - - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH . + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" + - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --no-push rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: always .py_build_push: - stage: build - before_script: - - cd $CONTEXT - - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" $CI_REGISTRY --password-stdin - variables: - CI_DEBUG_TRACE: "true" - IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA - LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest - script: - - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)" - - docker build -t $IMAGE_TAG -f $DOCKERFILE_PATH . - - docker push $IMAGE_TAG - - docker tag $IMAGE_TAG $LATEST_TAG - - docker push $LATEST_TAG - rules: + stage: build + variables: + IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA + LATEST_TAG: $CI_REGISTRY_IMAGE/$APP_NAME:latest + script: + - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)" + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" + - /kaniko/executor --context $CONTEXT --dockerfile $CI_PROJECT_DIR/docker/Dockerfile $KANIKO_CACHE_ARGS --destination $IMAGE_TAG + - /kaniko/executor --context $CONTEXT --dockerfile $CI_PROJECT_DIR/docker/Dockerfile $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE:latest + rules: - if: $CI_COMMIT_BRANCH == "master" when: always From f6a7956fdfafede7b53430537a4228f737982fc1 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Wed, 10 Mar 2021 03:53:58 +0000 Subject: [PATCH 06/20] Update .cic-template.yml --- ci_templates/.cic-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index 67b38fb1..ecc2e161 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -27,8 +27,8 @@ variables: - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)" - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" - - /kaniko/executor --context $CONTEXT --dockerfile $CI_PROJECT_DIR/docker/Dockerfile $KANIKO_CACHE_ARGS --destination $IMAGE_TAG - - /kaniko/executor --context $CONTEXT --dockerfile $CI_PROJECT_DIR/docker/Dockerfile $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE:latest + - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG + - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest rules: - if: $CI_COMMIT_BRANCH == "master" when: always From f136504988151e0c976eb5f7db2f890912c02885 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Fri, 12 Mar 2021 16:48:39 +0000 Subject: [PATCH 07/20] use the ubuntu slim image --- apps/cic-notify/cic_notify/version.py | 7 ++----- apps/cic-notify/requirements.txt | 4 ++-- apps/cic-ussd/docker/Dockerfile | 9 ++++----- apps/cic-ussd/requirements.txt | 10 +++++----- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/apps/cic-notify/cic_notify/version.py b/apps/cic-notify/cic_notify/version.py index 6c1fe32d..97975986 100644 --- a/apps/cic-notify/cic_notify/version.py +++ b/apps/cic-notify/cic_notify/version.py @@ -10,7 +10,7 @@ from cic_notify.error import PleaseCommitFirstError logg = logging.getLogger() -version = (0, 4, 0, 'alpha.2') +version = (0, 4, 0, 'alpha.3') version_object = semver.VersionInfo( major=version[0], @@ -24,9 +24,6 @@ version_string = str(version_object) def git_hash(): import subprocess - git_diff = subprocess.run(['git', 'diff'], capture_output=True) - if len(git_diff.stdout) > 0: - raise PleaseCommitFirstError() git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True) git_hash_brief = git_hash.stdout.decode('utf-8')[:8] @@ -35,7 +32,7 @@ def git_hash(): try: version_git = git_hash() - version_string += '.build.{}'.format(version_git) + version_string += '+build.{}'.format(version_git) except FileNotFoundError: time_string_pair = str(time.time()).split('.') version_string += '+build.{}{:<09d}'.format( diff --git a/apps/cic-notify/requirements.txt b/apps/cic-notify/requirements.txt index 0ab7977c..0b6f3615 100644 --- a/apps/cic-notify/requirements.txt +++ b/apps/cic-notify/requirements.txt @@ -1,5 +1,5 @@ -celery~=4.4.7 -confini~=0.3.6a1 alembic~=1.4.2 +celery~=4.4.7 +confini~=0.3.6rc3 redis~=3.5.3 semver==2.13.0 \ No newline at end of file diff --git a/apps/cic-ussd/docker/Dockerfile b/apps/cic-ussd/docker/Dockerfile index 6dbdd269..8257348e 100644 --- a/apps/cic-ussd/docker/Dockerfile +++ b/apps/cic-ussd/docker/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.8.5-alpine +# FROM python:3.8.5-alpine +FROM python:3.8.6-slim-buster # set working directory WORKDIR /usr/src @@ -6,10 +7,8 @@ WORKDIR /usr/src # add args for installing from self-hosted packages ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433' -# add alpine sys packages -RUN apk update && \ - apk add git linux-headers postgresql-dev gnupg bash -RUN apk add --update musl-dev gcc libffi-dev +RUN apt-get update && \ + apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git # create secrets directory RUN mkdir -vp pgp/keys diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 00dc2bc1..f22e7246 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,4 +1,3 @@ -cic-types==0.1.0a8 alembic==1.4.2 amqp==2.6.1 attrs==20.2.0 @@ -7,10 +6,11 @@ betterpath==0.2.2 billiard==3.6.3.0 celery==4.4.7 cffi==1.14.3 -cic-eth~=0.10.0a22 -cic-notify==0.3.1 +cic-eth~=0.10.0a41 +cic-notify~=0.4.0a3 +cic-types~=0.1.0a8 click==7.1.2 -confini~=0.3.6a1 +confini~=0.3.6rc3 cryptography==3.2.1 faker==4.17.1 iniconfig==1.1.1 @@ -44,4 +44,4 @@ transitions==0.8.4 uWSGI==2.0.19.1 vcversioner==2.16.0.0 vine==1.3.0 -zope.interface==5.1.2 +zope.interface==5.1.2 \ No newline at end of file From 842cbadf00a7ca028e8e3ed891056057da5c9bfe Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sat, 13 Mar 2021 18:50:36 +0000 Subject: [PATCH 08/20] contract migration image build improvements --- apps/contract-migration/docker/Dockerfile | 149 +++++++++++----------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 0de65735..e2b628e1 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -1,14 +1,21 @@ -#FROM ethereum/solc:0.6.12 -FROM ethereum/solc:0.8.0 +FROM python:3.8.6-slim-buster as compile-image -# The solc image messes up the alpine environment, so we have to go all over again -FROM alpine -COPY --from=0 /usr/bin/solc /usr/bin/solc +RUN apt-get update +RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo -RUN apk update && \ - apk add make git - -WORKDIR /usr/src +RUN apt-get install -y software-properties-common +RUN add-apt-repository ppa:ethereum/ethereum +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C52189C923F6CA9 +RUN apt-get update +RUN apt-get install solc +RUN pip install --upgrade pip + +WORKDIR /root +RUN mkdir -vp /usr/local/etc/cic + +COPY contract-migration/nvm.sh . +ENV CONFINI_DIR /usr/local/etc/cic/ +RUN mkdir -vp $CONFINI_DIR ARG cic_config_commit=35c69ba75f00c8147150acf325565d5391cf25bf ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/ @@ -16,11 +23,8 @@ RUN echo Install confini schema files && \ git clone --depth 1 $cic_config_url cic-config && \ cd cic-config && \ git fetch --depth 1 origin $cic_config_commit && \ - git checkout $cic_config_commit && \ - mkdir -vp /usr/local/etc/cic && \ - cp -v *.ini /usr/local/etc/cic/ -ENV CONFINI_DIR /usr/local/etc/cic - + git checkout $cic_config_commit && \ + cp -v *.ini $CONFINI_DIR ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501 ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/ @@ -31,30 +35,6 @@ RUN echo Install ABI collection for solidity interfaces used across all componen git checkout $cic_contracts_commit && \ make install -#COPY ./Makefile ./cic-contracts/Makefile -#COPY ./*.sol ./cic-contracts/ - -#RUN cd cic-contracts && \ -# make -B && make install -B - -FROM python:3.8.6-slim-buster - -COPY --from=1 /usr/local/share/cic/ /usr/local/share/cic/ -COPY --from=1 /usr/local/etc/ /usr/local/etc/ - -LABEL authors="Louis Holbrook 0826EDA1702D1E87C6E2875121D2E7BB88C2A746" -LABEL spdx-license-identifier="GPL-3.0-or-later" -LABEL description="Base layer for buiding development images for the cic component suite" - -RUN apt-get update && \ - apt-get install -y git gcc g++ libpq-dev && \ - apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client - - -RUN echo installing nodejs tooling - -COPY contract-migration/nvm.sh /root/ - # Install nvm with node and npm # https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker ENV NVM_DIR /root/.nvm @@ -65,59 +45,80 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | && . $NVM_DIR/nvm.sh \ && nvm install $NODE_VERSION \ && nvm alias default $NODE_VERSION \ - && nvm use $NODE_VERSION \ + && nvm use $NODE_VERSION # So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve # owner of these files is "1001" by default - wtf - && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION" +# && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION" ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH +# RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 +ARG cic_base_version=0.1.1a23 +ARG cic_registry_version=0.5.3a24 +RUN pip install --user --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version cic-base[full_graph]==$cic_base_version + +# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/ +# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity +# RUN echo Compile and install bancor protocol contracts && \ +# git clone --depth 1 $cic_bancor_url cic-bancor && \ +# cd cic-bancor + +# RUN cd cic-bancor/python && \ +# pip install --extra-index-url $pip_extra_index_url . # This is a temporary solution for building the Bancor contracts using the bancor protocol repository truffle setup # We should instead flatten the files ourselves and build them with solc in the first image layer in this file -ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094 -ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/ -ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity -RUN echo Compile and install bancor protocol contracts && \ - git clone --depth 1 $cic_bancor_url cic-bancor && \ - cd cic-bancor && \ - git fetch --depth 1 origin $cic_bancor_commit && \ - git checkout $cic_bancor_commit && \ - # Apparently the git version here doesn't have set-url as a command. *sigh* - #if [ ! -z $cic_bancor_contracts_url ]; then - # git submodule set-url bancor $cic_bancor_contracts_url - #fi - git submodule init && \ - git submodule update -RUN cd root && \ - . $NVM_DIR/nvm.sh &&\ - nvm install $BANCOR_NODE_VERSION && \ - nvm use $BANCOR_NODE_VERSION && \ - cd - && \ - cd cic-bancor/bancor && \ - npm install --python=/usr/bin/python2 && \ - node_modules/truffle/build/cli.bundled.js compile && \ - mkdir -vp /usr/local/share/cic/bancor/solidity/build && \ - cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/ -RUN cd cic-bancor/python && \ - pip install --extra-index-url $pip_extra_index_url . +# ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094 +# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/ +# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity +# RUN echo Compile and install bancor protocol contracts && \ +# git clone --depth 1 $cic_bancor_url cic-bancor && \ +# cd cic-bancor && \ +# git fetch --depth 1 origin $cic_bancor_commit && \ +# git checkout $cic_bancor_commit && \ +# # Apparently the git version here doesn't have set-url as a command. *sigh* +# #if [ ! -z $cic_bancor_contracts_url ]; then +# # git submodule set-url bancor $cic_bancor_contracts_url +# #fi +# git submodule init && \ +# git submodule update +# RUN cd root && \ +# . $NVM_DIR/nvm.sh &&\ +# nvm install $BANCOR_NODE_VERSION && \ +# nvm use $BANCOR_NODE_VERSION && \ +# cd - && \ +# cd cic-bancor/bancor && \ +# npm install --python=/usr/bin/python2 && \ +# node_modules/truffle/build/cli.bundled.js compile && \ +# mkdir -vp /usr/local/share/cic/bancor/solidity/build && \ +# cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/ +# RUN cd cic-bancor/python && \ +# pip install --extra-index-url $pip_extra_index_url . +FROM python:3.8.6-slim-buster as runtime-image -RUN apt-get install -y cargo -ARG cic_base_version=0.1.1a23 -RUN pip install --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version +COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ +COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ -ARG cic_registry_version=0.5.3a24 -RUN pip install --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version +RUN mkdir -p /tmp/cic/config +RUN chmod a+rw /tmp/cic/config -WORKDIR /root +ENV HOME /home/grassroots +RUN useradd --create-home grassroots +WORKDIR $HOME +# COPY python dependencies to user dir +COPY --from=compile-image /root/.local .local +RUN chown -R grassroots:grassroots $HOME/.local/ +ENV PATH=$HOME/.local/bin:$PATH + +USER grassroots COPY contract-migration/testdata/pgp testdata/pgp COPY contract-migration/wait-for-it.sh . -RUN chmod +x ./wait-for-it.sh +# RUN chmod +x ./wait-for-it.sh # COPY contract-migration/.env_config_template .env_config_template # COPY contract-migration/.env_dockercompose_template .env_dockercompose_template @@ -129,4 +130,4 @@ COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore COPY contract-migration/envlist . -# ENTRYPOINT [ "/bin/bash" ] +ENTRYPOINT [ ] From 4927d92215e68c9e6ec8a7f082e00569aa86d487 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 14 Mar 2021 20:21:07 -0700 Subject: [PATCH 09/20] seperate module install --- apps/contract-migration/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index e2b628e1..f78dfacb 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -58,7 +58,8 @@ ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 ARG cic_base_version=0.1.1a23 ARG cic_registry_version=0.5.3a24 -RUN pip install --user --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version cic-base[full_graph]==$cic_base_version +RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version +RUN pip install --user --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version # ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/ # ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity From 097a80e8de44ec81171841fba4987e9dbff933b8 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Mon, 15 Mar 2021 13:20:18 +0000 Subject: [PATCH 10/20] cic cache db fixes --- apps/cic-cache/.config/database.ini | 2 +- apps/cic-cache/cic_cache/db/migrations/default/alembic.ini | 2 +- apps/cic-cache/db/initdb_files/create_all_db.sql | 5 ----- docker-compose.yml | 1 + 4 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 apps/cic-cache/db/initdb_files/create_all_db.sql diff --git a/apps/cic-cache/.config/database.ini b/apps/cic-cache/.config/database.ini index b8c77685..72175b93 100644 --- a/apps/cic-cache/.config/database.ini +++ b/apps/cic-cache/.config/database.ini @@ -1,5 +1,5 @@ [database] -NAME=cic-eth +NAME=cic_cache USER=postgres PASSWORD= HOST=localhost diff --git a/apps/cic-cache/cic_cache/db/migrations/default/alembic.ini b/apps/cic-cache/cic_cache/db/migrations/default/alembic.ini index 352cf337..d05f4541 100644 --- a/apps/cic-cache/cic_cache/db/migrations/default/alembic.ini +++ b/apps/cic-cache/cic_cache/db/migrations/default/alembic.ini @@ -36,7 +36,7 @@ script_location = . # output_encoding = utf-8 #sqlalchemy.url = driver://user:pass@localhost/dbname -sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-cache +sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic_cache [post_write_hooks] diff --git a/apps/cic-cache/db/initdb_files/create_all_db.sql b/apps/cic-cache/db/initdb_files/create_all_db.sql deleted file mode 100644 index 1282d8b8..00000000 --- a/apps/cic-cache/db/initdb_files/create_all_db.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE DATABASE "cic-cache"; -CREATE DATABASE "cic-eth"; -CREATE DATABASE "cic-notify"; -CREATE DATABASE "cic-meta"; -CREATE DATABASE "cic-signer"; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7bd962bf..ad78f0b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,7 @@ services: volumes: - ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql - ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh + - ./apps/cic-cache/db/psycopg2/db.sql:/docker-entrypoint-initdb.d/3-init-cic-meta.sql - postgres-db:/var/lib/postgresql/data redis: From 432dbe9fee2b6eb30d2c47e94a37aafe48ae0bcd Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Mon, 15 Mar 2021 13:22:39 +0000 Subject: [PATCH 11/20] adding cic cache details --- apps/cic-cache/cic_cache/tasks/__init__.py | 0 apps/cic-cache/setup.cfg | 6 ++-- docker-compose.yml | 40 ++++++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 apps/cic-cache/cic_cache/tasks/__init__.py diff --git a/apps/cic-cache/cic_cache/tasks/__init__.py b/apps/cic-cache/cic_cache/tasks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-cache/setup.cfg b/apps/cic-cache/setup.cfg index 3831269d..07ceec1e 100644 --- a/apps/cic-cache/setup.cfg +++ b/apps/cic-cache/setup.cfg @@ -25,6 +25,7 @@ licence_files = python_requires = >= 3.6 packages = cic_cache + cic_cache.tasks cic_cache.db cic_cache.db.models cic_cache.runnable @@ -33,5 +34,6 @@ scripts = [options.entry_points] console_scripts = - cic-cache-tracker = cic_cache.runnable.tracker:main - cic-cache-server = cic_cache.runnable.server:main + cic-cache-trackerd = cic_cache.runnable.tracker:main + cic-cache-serverd = cic_cache.runnable.server:main + cic-cache-taskerd = cic_cache.runnable.tasker:main diff --git a/docker-compose.yml b/docker-compose.yml index 7bd962bf..5079aa19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,7 +162,43 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - /usr/local/bin/cic-cache-tracker -vv + /usr/local/bin/cic-cache-trackerd -vv + volumes: + - contract-config:/tmp/cic/config/:ro + + cic-cache-tasker: + build: + context: apps + dockerfile: cic-cache/docker/Dockerfile + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: 1 + ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} + CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CELERY_BROKER_URL: redis://redis:6379 + CELERY_RESULT_URL: redis://redis:6379 + deploy: + restart_policy: + condition: on-failure + depends_on: + - redis + - postgres + - eth + command: + - /bin/bash + - -c + - | + if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + /usr/local/bin/cic-cache-taskerd -vv volumes: - contract-config:/tmp/cic/config/:ro @@ -192,7 +228,7 @@ services: - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \ + --wsgi-file /usr/src/cic-cache/cic_cache/runnable/serverd.py \ --http :8000 \ --pyargv -vv From c0dff41b3c7863a19ff24350d6a25b717ce3afc8 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Mon, 15 Mar 2021 07:10:08 -0700 Subject: [PATCH 12/20] fix home dir permissions --- apps/contract-migration/docker/Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index f78dfacb..6d5735e4 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -104,26 +104,17 @@ FROM python:3.8.6-slim-buster as runtime-image COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ -RUN mkdir -p /tmp/cic/config -RUN chmod a+rw /tmp/cic/config ENV HOME /home/grassroots RUN useradd --create-home grassroots WORKDIR $HOME # COPY python dependencies to user dir COPY --from=compile-image /root/.local .local -RUN chown -R grassroots:grassroots $HOME/.local/ ENV PATH=$HOME/.local/bin:$PATH - -USER grassroots +# RUN chown grassroots:grassroots $HOME/.local/ COPY contract-migration/testdata/pgp testdata/pgp COPY contract-migration/wait-for-it.sh . -# RUN chmod +x ./wait-for-it.sh - -# COPY contract-migration/.env_config_template .env_config_template -# COPY contract-migration/.env_dockercompose_template .env_dockercompose_template - COPY contract-migration/reset.sh reset.sh COPY contract-migration/from_env.sh from_env.sh COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh @@ -131,4 +122,14 @@ COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore COPY contract-migration/envlist . +# critically, includes the .local folder from compile-image +RUN chown -R grassroots:grassroots . + +RUN mkdir -p /tmp/cic/config +RUN chown grassroots:grassroots /tmp/cic/config +# A shared output dir for environment configs +RUN chmod a+rwx /tmp/cic/config + +USER grassroots + ENTRYPOINT [ ] From 20a26045eb76f2889a58feedc6674a3853bb424a Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Mon, 15 Mar 2021 14:21:39 +0000 Subject: [PATCH 13/20] Update .cic-template.yml --- ci_templates/.cic-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index ecc2e161..420a9c4f 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -27,7 +27,7 @@ variables: - export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)" - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" - - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG + # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest rules: - if: $CI_COMMIT_BRANCH == "master" From c59f9da0fcfda6e554315fdd56141de936abf2c6 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Mon, 15 Mar 2021 14:31:48 +0000 Subject: [PATCH 14/20] try the multiple dest flags... --- ci_templates/.cic-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index 420a9c4f..28eaf091 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -28,7 +28,7 @@ variables: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json" # - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG - - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest + - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest rules: - if: $CI_COMMIT_BRANCH == "master" when: always From 75262dae5db0159813bdcb1770cf343c313dfb8d Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Tue, 16 Mar 2021 05:34:19 +0000 Subject: [PATCH 15/20] add permissions and move some pip installs --- apps/contract-migration/docker/Dockerfile | 34 +++++++++++++++-------- apps/contract-migration/seed_cic_eth.sh | 3 -- 2 files changed, 22 insertions(+), 15 deletions(-) mode change 100644 => 100755 apps/contract-migration/seed_cic_eth.sh diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 6d5735e4..034bd6c8 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax = docker/dockerfile:1.2 FROM python:3.8.6-slim-buster as compile-image RUN apt-get update @@ -46,8 +47,6 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | && nvm install $NODE_VERSION \ && nvm alias default $NODE_VERSION \ && nvm use $NODE_VERSION -# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve -# owner of these files is "1001" by default - wtf # && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION" ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules @@ -55,11 +54,21 @@ ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH # RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version +RUN useradd --create-home grassroots +WORKDIR /home/grassroots +USER grassroots + ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 ARG cic_base_version=0.1.1a23 ARG cic_registry_version=0.5.3a24 -RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version -RUN pip install --user --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version +ARG cic_eth_version=0.10.0a41 +ARG chainlib_version=0.0.1a21 +ARG cic_contracts_version=0.0.2a2 +RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \ + cic-registry==$cic_registry_version \ + cic-eth==$cic_eth_version \ + chainlib==$chainlib_version \ + cic-contracts==$cic_contracts_version # ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/ # ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity @@ -101,17 +110,17 @@ RUN pip install --user --extra-index-url $pip_extra_index_url cic-registry==$ci FROM python:3.8.6-slim-buster as runtime-image +RUN apt-get update +RUN apt-get install -y --no-install-recommends gnupg + COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ - -ENV HOME /home/grassroots RUN useradd --create-home grassroots -WORKDIR $HOME +WORKDIR /home/grassroots # COPY python dependencies to user dir -COPY --from=compile-image /root/.local .local -ENV PATH=$HOME/.local/bin:$PATH -# RUN chown grassroots:grassroots $HOME/.local/ +COPY --from=compile-image /home/grassroots/.local .local +ENV PATH=/home/grassroots/.local/bin:$PATH COPY contract-migration/testdata/pgp testdata/pgp COPY contract-migration/wait-for-it.sh . @@ -122,8 +131,9 @@ COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore COPY contract-migration/envlist . -# critically, includes the .local folder from compile-image -RUN chown -R grassroots:grassroots . +# RUN chown grassroots:grassroots .local/ +RUN chown grassroots:grassroots ./ +RUN chmod gu+x *.sh RUN mkdir -p /tmp/cic/config RUN chown grassroots:grassroots /tmp/cic/config diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh old mode 100644 new mode 100755 index d59efefa..a36b384a --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -30,9 +30,6 @@ truncate $env_out_file -s 0 set -e set -a -# We need to not install these here... -pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2 - >&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` From 7fe5b6bea3fb7db7cc8942d1e5b6aab87ccbaa9f Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Wed, 17 Mar 2021 07:30:34 -0700 Subject: [PATCH 16/20] add libpq to runtime --- apps/contract-migration/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 034bd6c8..7ce7ee51 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -111,7 +111,7 @@ RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_grap FROM python:3.8.6-slim-buster as runtime-image RUN apt-get update -RUN apt-get install -y --no-install-recommends gnupg +RUN apt-get install -y --no-install-recommends gnupg libpq-dev COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ From e4d1f5b65ca25daf96de3adf4f6b591cc0b0a84f Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Fri, 19 Mar 2021 14:19:40 +0000 Subject: [PATCH 17/20] contract migration refactor --- apps/contract-migration/docker/Dockerfile | 9 ++------ apps/contract-migration/reset.sh | 8 +++++++- apps/contract-migration/run_job.sh | 11 ++++++++++ apps/contract-migration/seed_cic_eth.sh | 2 ++ docker-compose.yml | 25 ++--------------------- 5 files changed, 24 insertions(+), 31 deletions(-) create mode 100644 apps/contract-migration/run_job.sh diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 7ce7ee51..28f0d65f 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -52,8 +52,6 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH -# RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version - RUN useradd --create-home grassroots WORKDIR /home/grassroots USER grassroots @@ -123,16 +121,13 @@ COPY --from=compile-image /home/grassroots/.local .local ENV PATH=/home/grassroots/.local/bin:$PATH COPY contract-migration/testdata/pgp testdata/pgp -COPY contract-migration/wait-for-it.sh . -COPY contract-migration/reset.sh reset.sh -COPY contract-migration/from_env.sh from_env.sh -COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore COPY contract-migration/envlist . +COPY contract-migration/*.sh ./ # RUN chown grassroots:grassroots .local/ -RUN chown grassroots:grassroots ./ +RUN chown grassroots:grassroots ./* RUN chmod gu+x *.sh RUN mkdir -p /tmp/cic/config diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 8768921f..138a70e1 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -19,6 +19,12 @@ echo \n # pushd /usr/src init_level_file=${CIC_DATA_DIR}/.init +if [ ! -f ${CIC_DATA_DIR}/.init ]; then + echo "Creating .init file..." + mkdir -p $CIC_DATA_DIR + touch /tmp/cic/config/.init +# touch $init_level_file +fi echo -n 1 > $init_level_file # Abort on any error (including if wait-for-it fails). @@ -61,7 +67,7 @@ export DEV_ETH_ACCOUNTS_INDEX_ADDRESS=$CIC_ACCOUNTS_INDEX_ADDRESS export BANCOR_REGISTRY_ADDRESS=$BANCOR_REGISTRY_ADDRESS export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER - +export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS EOF cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all diff --git a/apps/contract-migration/run_job.sh b/apps/contract-migration/run_job.sh new file mode 100644 index 00000000..f6c4c076 --- /dev/null +++ b/apps/contract-migration/run_job.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +if [[ $RUN_LEVEL -gt 0 ]] +then + ./reset.sh +fi + +if [[ $RUN_LEVEL -gt 1 ]] +then + ./seed_cic_eth.sh +fi \ No newline at end of file diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index a36b384a..3264ecda 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -26,6 +26,7 @@ env_out_file=${CIC_DATA_DIR}/.env_seed init_level_file=${CIC_DATA_DIR}/.init truncate $env_out_file -s 0 +pip install --extra-index-url https://pip.grassrootseconomics.net:8433 chainlib==0.0.1a22 set -e set -a @@ -125,6 +126,7 @@ export CIC_TOKEN_INDEX_ADDRESS=$CIC_TOKEN_INDEX_ADDRESS >&2 echo "add declarations for sarafu token" token_description_one=`sha256sum sarafu_declaration.json | awk '{ print $1; }'` token_description_two=0x54686973206973207468652053617261667520746f6b656e0000000000000000 +echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo $CIC_DECLARATOR_ADDRESSh" >&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_one >&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_two diff --git a/docker-compose.yml b/docker-compose.yml index 8953d646..25ad1775 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,24 +77,6 @@ services: - bee-data:/tmp/cic/bee contract-migration: - build: - context: apps/ - dockerfile: contract-migration/docker/Dockerfile - environment: - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - ETH_PROVIDER: http://eth:8545 - # And these two are for wait-for-it (could parse this) - ETH_PROVIDER_HOST: eth - ETH_PROVIDER_PORT: 8545 - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} - command: ["./reset.sh"] - depends_on: - - eth - volumes: - - contract-config:/tmp/cic/config - - seed-cic-eth: build: context: apps/ dockerfile: contract-migration/docker/Dockerfile @@ -118,10 +100,8 @@ services: CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - command: ["./seed_cic_eth.sh"] - # deploy: - #restart_policy: - # condition: on-failure + RUN_LEVEL: 2 #0: noop 1: contract migrations 2: seed data + command: ["./run_job.sh"] depends_on: - eth - postgres @@ -130,7 +110,6 @@ services: volumes: - contract-config:/tmp/cic/config - cic-cache-tracker: build: context: apps From e3ab8bcd82b293c174f32a8bbf0915a014679644 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Sat, 20 Mar 2021 06:04:02 +0000 Subject: [PATCH 18/20] Fix breaking seed script --- apps/cic-ussd/requirements.txt | 3 ++- service-configs/.gitkeep | 0 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 service-configs/.gitkeep diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index f22e7246..654f53bd 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,3 +1,4 @@ +africastalking==1.2.3 alembic==1.4.2 amqp==2.6.1 attrs==20.2.0 @@ -6,7 +7,7 @@ betterpath==0.2.2 billiard==3.6.3.0 celery==4.4.7 cffi==1.14.3 -cic-eth~=0.10.0a41 +cic-eth~=0.10.0a46 cic-notify~=0.4.0a3 cic-types~=0.1.0a8 click==7.1.2 diff --git a/service-configs/.gitkeep b/service-configs/.gitkeep deleted file mode 100644 index e69de29b..00000000 From bdd7ca14c28025db746e0b29756be7237779dde4 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 21 Mar 2021 11:32:53 -0700 Subject: [PATCH 19/20] contract migration permissions --- apps/contract-migration/docker/Dockerfile | 1 + docker-compose.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 28f0d65f..7589c6c8 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -128,6 +128,7 @@ COPY contract-migration/*.sh ./ # RUN chown grassroots:grassroots .local/ RUN chown grassroots:grassroots ./* +RUN chown grassroots:grassroots ./.local RUN chmod gu+x *.sh RUN mkdir -p /tmp/cic/config diff --git a/docker-compose.yml b/docker-compose.yml index 25ad1775..98d4f038 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: restart: unless-stopped ports: - ${DEV_ETH_PORT_HTTP:-63545}:8545 - - ${DEV_ETH_PORT_WS-63546}:8546 + - ${DEV_ETH_PORT_WS:-63546}:8546 - 30303 volumes: - ./apps/bloxbergValidatorSetup/keys:/root/keys # stores the signing key locally @@ -80,6 +80,7 @@ services: build: context: apps/ dockerfile: contract-migration/docker/Dockerfile + # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: # ETH_PROVIDER should be broken out into host/port but cic-eth expects this ETH_PROVIDER: http://eth:8545 From 299385f320cc614c1e5ef5f897e61927e1e43a17 Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 21 Mar 2021 17:18:47 -0700 Subject: [PATCH 20/20] recursive own --- apps/contract-migration/docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index 7589c6c8..91c160a1 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -127,8 +127,7 @@ COPY contract-migration/envlist . COPY contract-migration/*.sh ./ # RUN chown grassroots:grassroots .local/ -RUN chown grassroots:grassroots ./* -RUN chown grassroots:grassroots ./.local +RUN chown grassroots:grassroots -R . RUN chmod gu+x *.sh RUN mkdir -p /tmp/cic/config