Compare commits

..

2 Commits

Author SHA1 Message Date
nolash
820accf553 Add tasks module to python package build 2021-03-12 18:47:35 +01:00
361f5f7f8b adding cic cache details 2021-03-12 09:27:27 -08:00
15 changed files with 124 additions and 172 deletions

View File

@@ -1,5 +1,5 @@
[database]
NAME=cic_cache
NAME=cic-eth
USER=postgres
PASSWORD=
HOST=localhost

View File

@@ -36,7 +36,7 @@ script_location = .
# output_encoding = utf-8
#sqlalchemy.url = driver://user:pass@localhost/dbname
sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic_cache
sqlalchemy.url = postgresql+psycopg2://postgres@localhost:5432/cic-cache
[post_write_hooks]

View File

@@ -0,0 +1,5 @@
CREATE DATABASE "cic-cache";
CREATE DATABASE "cic-eth";
CREATE DATABASE "cic-notify";
CREATE DATABASE "cic-meta";
CREATE DATABASE "cic-signer";

View File

@@ -10,7 +10,7 @@ from cic_notify.error import PleaseCommitFirstError
logg = logging.getLogger()
version = (0, 4, 0, 'alpha.3')
version = (0, 4, 0, 'alpha.2')
version_object = semver.VersionInfo(
major=version[0],
@@ -24,6 +24,9 @@ version_string = str(version_object)
def git_hash():
import subprocess
git_diff = subprocess.run(['git', 'diff'], capture_output=True)
if len(git_diff.stdout) > 0:
raise PleaseCommitFirstError()
git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
@@ -32,7 +35,7 @@ def git_hash():
try:
version_git = git_hash()
version_string += '+build.{}'.format(version_git)
version_string += '.build.{}'.format(version_git)
except FileNotFoundError:
time_string_pair = str(time.time()).split('.')
version_string += '+build.{}{:<09d}'.format(

View File

@@ -1,5 +1,5 @@
alembic~=1.4.2
celery~=4.4.7
confini~=0.3.6rc3
confini~=0.3.6a1
alembic~=1.4.2
redis~=3.5.3
semver==2.13.0

View File

@@ -14,7 +14,7 @@ from cic_ussd.db.models.user import User
from cic_ussd.db.models.ussd_session import UssdSession
from cic_ussd.db.models.task_tracker import TaskTracker
from cic_ussd.menu.ussd_menu import UssdMenu
from cic_ussd.processor import custom_display_text, process_request, retrieve_most_recent_ussd_session
from cic_ussd.processor import custom_display_text, process_request
from cic_ussd.redis import InMemoryStore
from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
from cic_ussd.validator import check_known_user, validate_response_type
@@ -60,8 +60,7 @@ def create_ussd_session(
phone: str,
service_code: str,
user_input: str,
current_menu: str,
session_data: Optional[dict] = None) -> InMemoryUssdSession:
current_menu: str) -> InMemoryUssdSession:
"""
Creates a new ussd session
:param external_session_id: Session id value provided by AT
@@ -74,8 +73,6 @@ def create_ussd_session(
:type user_input: str
:param current_menu: Menu name that is currently being displayed on the ussd session
:type current_menu: str
:param session_data: Any additional data that was persisted during the user's interaction with the system.
:type session_data: dict.
:return: ussd session object
:rtype: Session
"""
@@ -84,8 +81,7 @@ def create_ussd_session(
msisdn=phone,
user_input=user_input,
state=current_menu,
service_code=service_code,
session_data=session_data
service_code=service_code
)
return session
@@ -130,9 +126,7 @@ def create_or_update_session(
phone=phone,
service_code=service_code,
user_input=user_input,
current_menu=current_menu,
session_data=session_data
)
current_menu=current_menu)
return ussd_session
@@ -344,26 +338,14 @@ def process_menu_interaction_requests(chain_str: str,
user_input=user_input
)
last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
if last_ussd_session:
# create or update the ussd session as appropriate
ussd_session = create_or_update_session(
external_session_id=external_session_id,
phone=phone_number,
service_code=service_code,
user_input=user_input,
current_menu=current_menu.get('name'),
session_data=last_ussd_session.session_data
)
else:
ussd_session = create_or_update_session(
external_session_id=external_session_id,
phone=phone_number,
service_code=service_code,
user_input=user_input,
current_menu=current_menu.get('name')
)
# create or update the ussd session as appropriate
ussd_session = create_or_update_session(
external_session_id=external_session_id,
phone=phone_number,
service_code=service_code,
user_input=user_input,
current_menu=current_menu.get('name')
)
# define appropriate response
response = custom_display_text(

View File

@@ -6,7 +6,7 @@ from typing import Optional
# third party imports
import celery
from sqlalchemy import desc
from cic_types.models.person import Person
from tinydb.table import Document
# local imports
@@ -315,16 +315,6 @@ def process_start_menu(display_key: str, user: User):
)
def retrieve_most_recent_ussd_session(phone_number: str) -> UssdSession:
# get last ussd session based on user phone number
last_ussd_session = UssdSession.session\
.query(UssdSession)\
.filter_by(msisdn=phone_number)\
.order_by(desc(UssdSession.created))\
.first()
return last_ussd_session
def process_request(user_input: str, user: User, ussd_session: Optional[dict] = None) -> Document:
"""This function assesses a request based on the user from the request comes, the session_id and the user's
input. It determines whether the request translates to a return to an existing session by checking whether the
@@ -347,23 +337,7 @@ def process_request(user_input: str, user: User, ussd_session: Optional[dict] =
return UssdMenu.find_by_name(name=successive_state)
else:
if user.has_valid_pin():
last_ussd_session = retrieve_most_recent_ussd_session(phone_number=user.phone_number)
key = create_cached_data_key(
identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address),
salt='cic.person'
)
user_metadata = get_cached_data(key=key)
if last_ussd_session:
# get last state
last_state = last_ussd_session.state
logg.debug(f'LAST USSD SESSION STATE: {last_state}')
# if last state is account_creation_prompt and metadata exists, show start menu
if last_state == 'account_creation_prompt' and user_metadata is not None:
return UssdMenu.find_by_name(name='start')
else:
return UssdMenu.find_by_name(name=last_state)
return UssdMenu.find_by_name(name='start')
else:
if user.failed_pin_attempts >= 3 and user.get_account_status() == AccountStatus.LOCKED.name:
return UssdMenu.find_by_name(name='exit_pin_blocked')

View File

@@ -1,5 +1,4 @@
# FROM python:3.8.5-alpine
FROM python:3.8.6-slim-buster
FROM python:3.8.5-alpine
# set working directory
WORKDIR /usr/src
@@ -7,8 +6,10 @@ WORKDIR /usr/src
# add args for installing from self-hosted packages
ARG pip_extra_index_url_flag='--extra-index-url https://pip.grassrootseconomics.net:8433'
RUN apt-get update && \
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
# add alpine sys packages
RUN apk update && \
apk add git linux-headers postgresql-dev gnupg bash
RUN apk add --update musl-dev gcc libffi-dev
# create secrets directory
RUN mkdir -vp pgp/keys

View File

@@ -1,3 +1,4 @@
cic-types==0.1.0a8
alembic==1.4.2
amqp==2.6.1
attrs==20.2.0
@@ -6,11 +7,10 @@ betterpath==0.2.2
billiard==3.6.3.0
celery==4.4.7
cffi==1.14.3
cic-eth~=0.10.0a41
cic-notify~=0.4.0a3
cic-types~=0.1.0a8
cic-eth~=0.10.0a22
cic-notify==0.3.1
click==7.1.2
confini~=0.3.6rc3
confini~=0.3.6a1
cryptography==3.2.1
faker==4.17.1
iniconfig==1.1.1
@@ -44,4 +44,4 @@ transitions==0.8.4
uWSGI==2.0.19.1
vcversioner==2.16.0.0
vine==1.3.0
zope.interface==5.1.2
zope.interface==5.1.2

View File

@@ -1,22 +1,14 @@
# syntax = docker/dockerfile:1.2
FROM python:3.8.6-slim-buster as compile-image
#FROM ethereum/solc:0.6.12
FROM ethereum/solc:0.8.0
RUN apt-get update
RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo
# The solc image messes up the alpine environment, so we have to go all over again
FROM alpine
COPY --from=0 /usr/bin/solc /usr/bin/solc
RUN apt-get install -y software-properties-common
RUN add-apt-repository ppa:ethereum/ethereum
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1C52189C923F6CA9
RUN apt-get update
RUN apt-get install solc
RUN pip install --upgrade pip
WORKDIR /root
RUN mkdir -vp /usr/local/etc/cic
COPY contract-migration/nvm.sh .
ENV CONFINI_DIR /usr/local/etc/cic/
RUN mkdir -vp $CONFINI_DIR
RUN apk update && \
apk add make git
WORKDIR /usr/src
ARG cic_config_commit=35c69ba75f00c8147150acf325565d5391cf25bf
ARG cic_config_url=https://gitlab.com/grassrootseconomics/cic-config.git/
@@ -24,8 +16,11 @@ RUN echo Install confini schema files && \
git clone --depth 1 $cic_config_url cic-config && \
cd cic-config && \
git fetch --depth 1 origin $cic_config_commit && \
git checkout $cic_config_commit && \
cp -v *.ini $CONFINI_DIR
git checkout $cic_config_commit && \
mkdir -vp /usr/local/etc/cic && \
cp -v *.ini /usr/local/etc/cic/
ENV CONFINI_DIR /usr/local/etc/cic
ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501
ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/
@@ -36,6 +31,30 @@ RUN echo Install ABI collection for solidity interfaces used across all componen
git checkout $cic_contracts_commit && \
make install
#COPY ./Makefile ./cic-contracts/Makefile
#COPY ./*.sol ./cic-contracts/
#RUN cd cic-contracts && \
# make -B && make install -B
FROM python:3.8.6-slim-buster
COPY --from=1 /usr/local/share/cic/ /usr/local/share/cic/
COPY --from=1 /usr/local/etc/ /usr/local/etc/
LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
LABEL spdx-license-identifier="GPL-3.0-or-later"
LABEL description="Base layer for buiding development images for the cic component suite"
RUN apt-get update && \
apt-get install -y git gcc g++ libpq-dev && \
apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
RUN echo installing nodejs tooling
COPY contract-migration/nvm.sh /root/
# Install nvm with node and npm
# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
ENV NVM_DIR /root/.nvm
@@ -46,84 +65,63 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh |
&& . $NVM_DIR/nvm.sh \
&& nvm install $NODE_VERSION \
&& nvm alias default $NODE_VERSION \
&& nvm use $NODE_VERSION
# && chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
&& nvm use $NODE_VERSION \
# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
# owner of these files is "1001" by default - wtf
&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
# RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
RUN useradd --create-home grassroots
WORKDIR /home/grassroots
USER grassroots
ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
ARG cic_base_version=0.1.1a23
ARG cic_registry_version=0.5.3a24
ARG cic_eth_version=0.10.0a41
ARG chainlib_version=0.0.1a21
ARG cic_contracts_version=0.0.2a2
RUN pip install --user --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version \
cic-registry==$cic_registry_version \
cic-eth==$cic_eth_version \
chainlib==$chainlib_version \
cic-contracts==$cic_contracts_version
# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
# RUN echo Compile and install bancor protocol contracts && \
# git clone --depth 1 $cic_bancor_url cic-bancor && \
# cd cic-bancor
# RUN cd cic-bancor/python && \
# pip install --extra-index-url $pip_extra_index_url .
# This is a temporary solution for building the Bancor contracts using the bancor protocol repository truffle setup
# We should instead flatten the files ourselves and build them with solc in the first image layer in this file
# ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094
# ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
# ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
# RUN echo Compile and install bancor protocol contracts && \
# git clone --depth 1 $cic_bancor_url cic-bancor && \
# cd cic-bancor && \
# git fetch --depth 1 origin $cic_bancor_commit && \
# git checkout $cic_bancor_commit && \
# # Apparently the git version here doesn't have set-url as a command. *sigh*
# #if [ ! -z $cic_bancor_contracts_url ]; then
# # git submodule set-url bancor $cic_bancor_contracts_url
# #fi
# git submodule init && \
# git submodule update
# RUN cd root && \
# . $NVM_DIR/nvm.sh &&\
# nvm install $BANCOR_NODE_VERSION && \
# nvm use $BANCOR_NODE_VERSION && \
# cd - && \
# cd cic-bancor/bancor && \
# npm install --python=/usr/bin/python2 && \
# node_modules/truffle/build/cli.bundled.js compile && \
# mkdir -vp /usr/local/share/cic/bancor/solidity/build && \
# cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/
# RUN cd cic-bancor/python && \
# pip install --extra-index-url $pip_extra_index_url .
ARG cic_bancor_commit=a04c7ae6882ea515938d852cc861d59a35070094
ARG cic_bancor_url=https://gitlab.com/grassrootseconomics/cic-bancor.git/
ARG cic_bancor_contracts_url=https://github.com/bancorprotocol/contracts-solidity
RUN echo Compile and install bancor protocol contracts && \
git clone --depth 1 $cic_bancor_url cic-bancor && \
cd cic-bancor && \
git fetch --depth 1 origin $cic_bancor_commit && \
git checkout $cic_bancor_commit && \
# Apparently the git version here doesn't have set-url as a command. *sigh*
#if [ ! -z $cic_bancor_contracts_url ]; then
# git submodule set-url bancor $cic_bancor_contracts_url
#fi
git submodule init && \
git submodule update
RUN cd root && \
. $NVM_DIR/nvm.sh &&\
nvm install $BANCOR_NODE_VERSION && \
nvm use $BANCOR_NODE_VERSION && \
cd - && \
cd cic-bancor/bancor && \
npm install --python=/usr/bin/python2 && \
node_modules/truffle/build/cli.bundled.js compile && \
mkdir -vp /usr/local/share/cic/bancor/solidity/build && \
cp -vR solidity/build/contracts /usr/local/share/cic/bancor/solidity/build/
RUN cd cic-bancor/python && \
pip install --extra-index-url $pip_extra_index_url .
FROM python:3.8.6-slim-buster as runtime-image
RUN apt-get update
RUN apt-get install -y --no-install-recommends gnupg libpq-dev
RUN apt-get install -y cargo
ARG cic_base_version=0.1.1a23
RUN pip install --extra-index-url $pip_extra_index_url cic-base[full_graph]==$cic_base_version
COPY --from=compile-image /usr/local/bin/ /usr/local/bin/
COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/
ARG cic_registry_version=0.5.3a24
RUN pip install --extra-index-url $pip_extra_index_url cic-registry==$cic_registry_version
RUN useradd --create-home grassroots
WORKDIR /home/grassroots
# COPY python dependencies to user dir
COPY --from=compile-image /home/grassroots/.local .local
ENV PATH=/home/grassroots/.local/bin:$PATH
WORKDIR /root
COPY contract-migration/testdata/pgp testdata/pgp
COPY contract-migration/wait-for-it.sh .
RUN chmod +x ./wait-for-it.sh
# COPY contract-migration/.env_config_template .env_config_template
# COPY contract-migration/.env_dockercompose_template .env_dockercompose_template
COPY contract-migration/reset.sh reset.sh
COPY contract-migration/from_env.sh from_env.sh
COPY contract-migration/seed_cic_eth.sh seed_cic_eth.sh
@@ -131,15 +129,4 @@ COPY contract-migration/sarafu_declaration.json sarafu_declaration.json
COPY contract-migration/keystore keystore
COPY contract-migration/envlist .
# RUN chown grassroots:grassroots .local/
RUN chown grassroots:grassroots ./
RUN chmod gu+x *.sh
RUN mkdir -p /tmp/cic/config
RUN chown grassroots:grassroots /tmp/cic/config
# A shared output dir for environment configs
RUN chmod a+rwx /tmp/cic/config
USER grassroots
ENTRYPOINT [ ]
# ENTRYPOINT [ "/bin/bash" ]

View File

@@ -61,7 +61,7 @@ export DEV_ETH_ACCOUNTS_INDEX_ADDRESS=$CIC_ACCOUNTS_INDEX_ADDRESS
export BANCOR_REGISTRY_ADDRESS=$BANCOR_REGISTRY_ADDRESS
export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
export CIC_DECLARATOR_ADDRESS=$CIC_DECLARATOR_ADDRESS
EOF
cat ./envlist | bash from_env.sh > $CIC_DATA_DIR/.env_all

5
apps/contract-migration/seed_cic_eth.sh Executable file → Normal file
View File

@@ -26,11 +26,13 @@ env_out_file=${CIC_DATA_DIR}/.env_seed
init_level_file=${CIC_DATA_DIR}/.init
truncate $env_out_file -s 0
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 chainlib==0.0.1a22
set -e
set -a
# We need to not install these here...
pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a41 chainlib==0.0.1a21 cic-contracts==0.0.2a2
>&2 echo "create account for gas gifter"
old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
@@ -126,7 +128,6 @@ export CIC_TOKEN_INDEX_ADDRESS=$CIC_TOKEN_INDEX_ADDRESS
>&2 echo "add declarations for sarafu token"
token_description_one=`sha256sum sarafu_declaration.json | awk '{ print $1; }'`
token_description_two=0x54686973206973207468652053617261667520746f6b656e0000000000000000
echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo $CIC_DECLARATOR_ADDRESSh"
>&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_one
>&2 eth-address-declarator-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_DECLARATOR_ADDRESS -w $debug $DEV_ETH_SARAFU_TOKEN_ADDRESS $token_description_two

View File

@@ -27,8 +27,8 @@ variables:
- export IMAGE_TAG="$IMAGE_TAG_BASE-$(date +%F.%H%M%S)"
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
# - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $IMAGE_TAG
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --destination $CI_REGISTRY_IMAGE/$APP_NAME:latest
rules:
- if: $CI_COMMIT_BRANCH == "master"
when: always

View File

@@ -54,7 +54,6 @@ services:
volumes:
- ./scripts/initdb/create_db.sql:/docker-entrypoint-initdb.d/1-create_all_db.sql
- ./apps/cic-meta/scripts/initdb/postgresql.sh:/docker-entrypoint-initdb.d/2-init-cic-meta.sh
- ./apps/cic-cache/db/psycopg2/db.sql:/docker-entrypoint-initdb.d/3-init-cic-meta.sql
- postgres-db:/var/lib/postgresql/data
redis:

0
service-configs/.gitkeep Normal file
View File