From b855211eed802733dd3a2a14c974e5ada8272633 Mon Sep 17 00:00:00 2001 From: Blair V Date: Thu, 16 Sep 2021 13:06:53 +0300 Subject: [PATCH 01/13] chore: fix docker compose restarts --- README.md | 2 +- docker-compose.yml | 53 ++++++++++---------------------------- dump.sh => scripts/dump.sh | 0 3 files changed, 15 insertions(+), 40 deletions(-) rename dump.sh => scripts/dump.sh (100%) mode change 100644 => 100755 diff --git a/README.md b/README.md index a41e0026..b6fae088 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ docker-compose down stop cluster and delete data ``` -docker-compose down -v +docker-compose down -v --remove-orphans ``` rebuild an images diff --git a/docker-compose.yml b/docker-compose.yml index 563e8621..1df84bdf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -103,6 +103,7 @@ services: TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS TOKEN_TYPE: $TOKEN_TYPE SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} + restart: on-failure command: ["./run_job.sh"] #command: ["./reset.sh"] depends_on: @@ -139,9 +140,7 @@ services: CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 CELERY_RESULT_URL: redis://redis:6379 - deploy: - restart_policy: - condition: on-failure + restart: on-failure depends_on: - redis - postgres @@ -180,9 +179,7 @@ services: CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 CELERY_RESULT_URL: redis://redis:6379 - deploy: - restart_policy: - condition: on-failure + restart: unless-stopped depends_on: - redis - postgres @@ -214,13 +211,11 @@ services: DATABASE_DEBUG: 1 #PGPASSWORD: $DATABASE_PASSWORD SERVER_PORT: 8000 + restart: on-failure ports: - ${HTTP_PORT_CIC_CACHE:-63313}:8000 depends_on: - postgres - deploy: - restart_policy: - condition: on-failure command: - /bin/bash - -c @@ -268,14 +263,12 @@ services: ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} + restart: unless-stopped depends_on: - eth - postgres - redis - cic-eth-signer - deploy: - restart_policy: - condition: on-failure volumes: - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro @@ -324,13 +317,11 @@ services: ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} + restart: on-failure depends_on: - eth - postgres - redis - deploy: - restart_policy: - condition: on-failure volumes: - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro @@ -369,13 +360,11 @@ services: CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS + restart: on-failure depends_on: - eth - postgres - redis - deploy: - restart_policy: - condition: on-failure volumes: - contract-config:/tmp/cic/config/:ro command: @@ -415,13 +404,11 @@ services: TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS DATABASE_DEBUG: ${DATABASE_DEBUG:-false} #DATABASE_DEBUG: 1 + restart: on-failure depends_on: - eth - postgres - redis - deploy: - restart_policy: - condition: on-failure volumes: - contract-config:/tmp/cic/config/:ro command: @@ -463,13 +450,11 @@ services: CIC_TX_RETRY_DELAY: 60 BATCH_SIZE: ${RETRIER_BATCH_SIZE:-50} #DATABASE_DEBUG: 1 + restart: on-failure depends_on: - eth - postgres - redis - deploy: - restart_policy: - condition: on-failure volumes: - contract-config:/tmp/cic/config/:ro command: @@ -502,12 +487,10 @@ services: AFRICASTALKING_API_USERNAME: $AFRICASTALKING_API_USERNAME AFRICASTALKING_API_KEY: $AFRICASTALKING_API_KEY AFRICASTALKING_API_SENDER_ID: $AFRICASTALKING_API_SENDER_ID + restart: unless-stopped depends_on: - postgres - redis - deploy: - restart_policy: - condition: on-failure command: "/root/start_tasker.sh -q cic-notify -vv" @@ -536,13 +519,11 @@ services: PGP_PUBLICKEY_ACTIVE_FILE: publickeys.asc PGP_PUBLICKEY_ENCRYPT_FILE: publickeys.asc SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql + restart: on-failure ports: - ${HTTP_PORT_CIC_META:-63380}:8000 depends_on: - postgres - deploy: - restart_policy: - condition: on-failure volumes: - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp # command: "/root/start_server.sh -vv" @@ -568,6 +549,7 @@ services: PGP_PASSPHRASE: merman SERVER_PORT: 9000 CIC_META_URL: ${CIC_META_URL:-http://meta:8000} + restart: on-failure ports: - ${HTTP_PORT_CIC_USER_USSD_SERVER:-63315}:9000 depends_on: @@ -575,9 +557,6 @@ services: - redis volumes: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ - deploy: - restart_policy: - condition: on-failure command: "/root/start_cic_user_ussd_server.sh -vv" cic-user-server: @@ -597,13 +576,11 @@ services: DATABASE_ENGINE: postgresql DATABASE_DRIVER: psycopg2 DATABASE_POOL_SIZE: 0 + restart: on-failure ports: - ${HTTP_PORT_CIC_USER_SERVER:-63415}:9500 depends_on: - postgres - deploy: - restart_policy: - condition: on-failure command: "/root/start_cic_user_server.sh -vv" cic-user-tasker: @@ -627,12 +604,10 @@ services: CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis} PGP_PASSPHRASE: merman CIC_META_URL: ${CIC_META_URL:-http://meta:8000} + restart: unless-stopped depends_on: - postgres - redis volumes: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ - deploy: - restart_policy: - condition: on-failure command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" diff --git a/dump.sh b/scripts/dump.sh old mode 100644 new mode 100755 similarity index 100% rename from dump.sh rename to scripts/dump.sh From d65455fc298a22aaa9a9d0b63ad07c66aafe9858 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Fri, 17 Sep 2021 06:54:58 +0000 Subject: [PATCH 02/13] Update signer to fill in missing sign to wire symbol --- apps/cic-eth/cic_eth/cli/rpc.py | 4 ++-- apps/cic-eth/cic_eth/runnable/daemons/tasker.py | 1 - apps/contract-migration/config.sh | 2 +- apps/contract-migration/envlist | 1 + apps/contract-migration/requirements.txt | 4 ++-- apps/contract-migration/reset.sh | 5 +++-- apps/contract-migration/run_job.sh | 2 +- apps/data-seeding/requirements.txt | 4 ++-- docker-compose.yml | 7 ++++--- scripts/dump.sh | 0 10 files changed, 16 insertions(+), 14 deletions(-) mode change 100755 => 100644 scripts/dump.sh diff --git a/apps/cic-eth/cic_eth/cli/rpc.py b/apps/cic-eth/cic_eth/cli/rpc.py index 25e28f9b..c695402f 100644 --- a/apps/cic-eth/cic_eth/cli/rpc.py +++ b/apps/cic-eth/cic_eth/cli/rpc.py @@ -35,14 +35,14 @@ class RPC: def from_config(config, use_signer=False, default_label='default', signer_label='signer'): chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) - RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label) + RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label) if use_signer: RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label) RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label) RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label) RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) - rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) logg.info('set up rpc: {}'.format(rpc)) return rpc diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index fab470f3..52e8e1d9 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -69,7 +69,6 @@ from cic_eth.registry import ( ) from cic_eth.task import BaseTask - logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index 3831ce7e..858eaf85 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -36,7 +36,7 @@ export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOY # Migration variable processing -confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset +confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset cat ${DEV_DATA_DIR}/env_reset diff --git a/apps/contract-migration/envlist b/apps/contract-migration/envlist index 8613215b..d68d9f44 100644 --- a/apps/contract-migration/envlist +++ b/apps/contract-migration/envlist @@ -56,6 +56,7 @@ ETH_PROVIDER ETH_ABI_DIR SIGNER_SOCKET_PATH SIGNER_SECRET +SIGNER_PROVIDER CELERY_BROKER_URL CELERY_RESULT_URL META_PROVIDER diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index b04c6d90..0e62e29a 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,5 +1,5 @@ cic-eth[tools]==0.12.4a8 -chainlib-eth>=0.0.9a9,<0.1.0 +chainlib-eth>=0.0.9a14,<0.1.0 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 eth-accounts-index>=0.1.2a2,<0.2.0 @@ -9,4 +9,4 @@ erc20-transfer-authorization>=0.3.5a2,<0.4.0 erc20-faucet>=0.3.2a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 confini>=0.4.2rc3,<1.0.0 -crypto-dev-signer>=0.4.15a4,<=0.4.15 +crypto-dev-signer>=0.4.15a7,<=0.4.15 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index b9cd99d5..728356ce 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -3,8 +3,8 @@ set -a . ${DEV_DATA_DIR}/env_reset -WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60} +WAIT_FOR_TIMEOUT=${WAIT_FOR_TIMEOUT:-60} set -e @@ -109,7 +109,8 @@ giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERV #echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL -confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset + +confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config set +a diff --git a/apps/contract-migration/run_job.sh b/apps/contract-migration/run_job.sh index fcb62d7e..0df34361 100644 --- a/apps/contract-migration/run_job.sh +++ b/apps/contract-migration/run_job.sh @@ -1,7 +1,7 @@ #! /bin/bash >&2 echo -e "\033[;96mRUNNING\033[;39m configurations" -. ./config.sh +./config.sh if [ $? -ne "0" ]; then >&2 echo -e "\033[;31mFAILED\033[;39m configurations" exit 1; diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index c6931fbd..4c445443 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,10 +1,10 @@ sarafu-faucet~=0.0.7a1 cic-eth[tools]~=0.12.4a8 cic-types~=0.1.0a14 -crypto-dev-signer>=0.4.15a1,<=0.4.15 +crypto-dev-signer>=0.4.15a4,<=0.4.15 faker==4.17.1 chainsyncer~=0.0.6a3 -chainlib-eth~=0.0.9a7 +chainlib-eth~=0.0.9a13 eth-address-index~=0.2.3a4 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3 diff --git a/docker-compose.yml b/docker-compose.yml index 1df84bdf..c6cfd6eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,8 +94,8 @@ services: RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE - CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-SARAFU} TOKEN_NAME: ${TOKEN_NAME:-Sarafu Token} + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT @@ -257,8 +257,9 @@ services: CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} - SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} - SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} + #SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} + SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000} + SIGNER_PROVIDER: http://cic-eth-signer:8000 SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} diff --git a/scripts/dump.sh b/scripts/dump.sh old mode 100755 new mode 100644 From 6019143ba1151e794f1c42fcc0b27b57ecd33d3b Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Fri, 17 Sep 2021 11:15:43 +0000 Subject: [PATCH 03/13] Philip/ussd demurrage --- apps/cic-ussd/cic_ussd/account/balance.py | 31 +++++++++++ apps/cic-ussd/cic_ussd/processor/menu.py | 54 +++++++++++++------ apps/cic-ussd/cic_ussd/runnable/client.py | 6 +-- apps/cic-ussd/docker/Dockerfile | 7 +++ apps/cic-ussd/requirements.txt | 3 ++ .../tests/cic_ussd/account/test_balance.py | 11 +++- .../tests/cic_ussd/account/test_statement.py | 6 ++- .../cic_ussd/account/test_transaction.py | 9 ++-- .../tests/cic_ussd/processor/test_menu.py | 35 +++++++++--- .../logic/test_transaction_logic.py | 1 + .../cic_ussd/tasks/test_callback_handler.py | 1 + .../tasks/test_notifications_tasks.py | 3 +- .../cic_ussd/tasks/test_processor_tasks.py | 9 ++-- .../tests/fixtures/patches/account.py | 16 ++++++ apps/cic-ussd/var/lib/locale/ussd.en.yml | 22 +++++--- apps/cic-ussd/var/lib/locale/ussd.sw.yml | 22 +++++--- 16 files changed, 184 insertions(+), 52 deletions(-) diff --git a/apps/cic-ussd/cic_ussd/account/balance.py b/apps/cic-ussd/cic_ussd/account/balance.py index 904cf737..e8563a9f 100644 --- a/apps/cic-ussd/cic_ussd/account/balance.py +++ b/apps/cic-ussd/cic_ussd/account/balance.py @@ -1,10 +1,12 @@ # standard imports + import json import logging from typing import Optional # third-party imports from cic_eth.api import Api +from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi # local imports from cic_ussd.account.transaction import from_wei @@ -73,6 +75,24 @@ def calculate_available_balance(balances: dict) -> float: return from_wei(value=available_balance) +def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str): + """ + :param balance: + :type balance: + :param chain_str: + :type chain_str: + :param timestamp: + :type timestamp: + :param token_symbol: + :type token_symbol: + :return: + :rtype: + """ + logg.debug(f'retrieving adjusted balance on chain: {chain_str}') + demurrage_api = DemurrageApi(chain_str=chain_str) + return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result + + def get_cached_available_balance(blockchain_address: str) -> float: """This function attempts to retrieve balance data from the redis cache. :param blockchain_address: Ethereum address of an account. @@ -88,3 +108,14 @@ def get_cached_available_balance(blockchain_address: str) -> float: return calculate_available_balance(json.loads(cached_balances)) else: raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}') + + +def get_cached_adjusted_balance(identifier: bytes): + """ + :param identifier: + :type identifier: + :return: + :rtype: + """ + key = cache_data_key(identifier, ':cic.adjusted_balance') + return get_cached_data(key) diff --git a/apps/cic-ussd/cic_ussd/processor/menu.py b/apps/cic-ussd/cic_ussd/processor/menu.py index 9896c9e1..042f8680 100644 --- a/apps/cic-ussd/cic_ussd/processor/menu.py +++ b/apps/cic-ussd/cic_ussd/processor/menu.py @@ -1,13 +1,17 @@ # standard imports import json import logging +from datetime import datetime, timedelta # external imports import i18n.config -from sqlalchemy.orm.session import Session # local imports -from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance +from cic_ussd.account.balance import (calculate_available_balance, + get_adjusted_balance, + get_balances, + get_cached_adjusted_balance, + get_cached_available_balance) from cic_ussd.account.chain import Chain from cic_ussd.account.metadata import get_cached_preferred_language from cic_ussd.account.statement import ( @@ -16,14 +20,15 @@ from cic_ussd.account.statement import ( query_statement, statement_transaction_set ) -from cic_ussd.account.transaction import from_wei, to_wei from cic_ussd.account.tokens import get_default_token_symbol +from cic_ussd.account.transaction import from_wei, to_wei from cic_ussd.cache import cache_data_key, cache_data from cic_ussd.db.models.account import Account from cic_ussd.metadata import PersonMetadata from cic_ussd.phone_number import Support -from cic_ussd.processor.util import latest_input, parse_person_metadata +from cic_ussd.processor.util import parse_person_metadata from cic_ussd.translation import translation_for +from sqlalchemy.orm.session import Session logg = logging.getLogger(__name__) @@ -43,21 +48,26 @@ class MenuProcessor: :rtype: """ available_balance = get_cached_available_balance(self.account.blockchain_address) - logg.debug('Requires call to retrieve tax and bonus amounts') - tax = '' - bonus = '' + adjusted_balance = get_cached_adjusted_balance(self.identifier) token_symbol = get_default_token_symbol() preferred_language = get_cached_preferred_language(self.account.blockchain_address) if not preferred_language: preferred_language = i18n.config.get('fallback') - return translation_for( - key=self.display_key, - preferred_language=preferred_language, - available_balance=available_balance, - tax=tax, - bonus=bonus, - token_symbol=token_symbol - ) + with_available_balance = f'{self.display_key}.available_balance' + with_fees = f'{self.display_key}.with_fees' + if not adjusted_balance: + return translation_for(key=with_available_balance, + preferred_language=preferred_language, + available_balance=available_balance, + token_symbol=token_symbol) + adjusted_balance = json.loads(adjusted_balance) + tax_wei = to_wei(int(available_balance)) - int(adjusted_balance) + tax = from_wei(int(tax_wei)) + return translation_for(key=with_fees, + preferred_language=preferred_language, + available_balance=available_balance, + tax=tax, + token_symbol=token_symbol) def account_statement(self) -> str: """ @@ -67,7 +77,7 @@ class MenuProcessor: cached_statement = get_cached_statement(self.account.blockchain_address) statement = json.loads(cached_statement) statement_transactions = parse_statement_transactions(statement) - transaction_sets = [statement_transactions[tx:tx+3] for tx in range(0, len(statement_transactions), 3)] + transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)] preferred_language = get_cached_preferred_language(self.account.blockchain_address) if not preferred_language: preferred_language = i18n.config.get('fallback') @@ -149,12 +159,22 @@ class MenuProcessor: :return: :rtype: """ + chain_str = Chain.spec.__str__() token_symbol = get_default_token_symbol() blockchain_address = self.account.blockchain_address - balances = get_balances(blockchain_address, Chain.spec.__str__(), token_symbol, False)[0] + balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0] key = cache_data_key(self.identifier, ':cic.balances') cache_data(key, json.dumps(balances)) available_balance = calculate_available_balance(balances) + now = datetime.now() + if (now - self.account.created).days >= 30: + if available_balance <= 0: + logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.') + else: + timestamp = int((now - timedelta(30)).timestamp()) + adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol) + key = cache_data_key(self.identifier, ':cic.adjusted_balance') + cache_data(key, json.dumps(adjusted_balance)) query_statement(blockchain_address) diff --git a/apps/cic-ussd/cic_ussd/runnable/client.py b/apps/cic-ussd/cic_ussd/runnable/client.py index 5ea98e8e..37a62898 100644 --- a/apps/cic-ussd/cic_ussd/runnable/client.py +++ b/apps/cic-ussd/cic_ussd/runnable/client.py @@ -63,7 +63,7 @@ elif ssl == 0: else: ssl = True - +valid_service_codes = config.get('USSD_SERVICE_CODE').split(",") def main(): # TODO: improve url building @@ -79,9 +79,9 @@ def main(): session = uuid.uuid4().hex data = { 'sessionId': session, - 'serviceCode': config.get('APP_SERVICE_CODE'), + 'serviceCode': valid_service_codes[0], 'phoneNumber': args.phone, - 'text': config.get('APP_SERVICE_CODE'), + 'text': "", } state = "_BEGIN" diff --git a/apps/cic-ussd/docker/Dockerfile b/apps/cic-ussd/docker/Dockerfile index 8de82801..82120dc0 100644 --- a/apps/cic-ussd/docker/Dockerfile +++ b/apps/cic-ussd/docker/Dockerfile @@ -10,6 +10,13 @@ RUN mkdir -vp data ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" + +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY \ + --extra-index-url $EXTRA_INDEX_URL \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + COPY requirements.txt . RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 551f9ab3..3d274059 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,6 +1,9 @@ alembic==1.4.2 +attrs==21.2.0 +billiard==3.6.4.0 bcrypt==3.2.0 celery==4.4.7 +cffi==1.14.6 cic-eth[services]~=0.12.4a7 cic-notify~=0.4.0a10 cic-types~=0.1.0a14 diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_balance.py b/apps/cic-ussd/tests/cic_ussd/account/test_balance.py index 554f7fb6..fcfb8383 100644 --- a/apps/cic-ussd/tests/cic_ussd/account/test_balance.py +++ b/apps/cic-ussd/tests/cic_ussd/account/test_balance.py @@ -43,10 +43,13 @@ def test_sync_get_balances(activated_account, (5000000, 89000000, 67000000, 27.00) ]) def test_calculate_available_balance(activated_account, + available_balance, balance_incoming, balance_network, balance_outgoing, - available_balance): + cache_balances, + cache_default_token_data, + load_chain_spec): balances = { 'address': activated_account.blockchain_address, 'converters': [], @@ -57,7 +60,11 @@ def test_calculate_available_balance(activated_account, assert calculate_available_balance(balances) == available_balance -def test_get_cached_available_balance(activated_account, cache_balances, balances): +def test_get_cached_available_balance(activated_account, + balances, + cache_balances, + cache_default_token_data, + load_chain_spec): cached_available_balance = get_cached_available_balance(activated_account.blockchain_address) available_balance = calculate_available_balance(balances[0]) assert cached_available_balance == available_balance diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py index 86d79b2e..0e05c383 100644 --- a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py +++ b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py @@ -27,6 +27,8 @@ def test_filter_statement_transactions(transactions_list): def test_generate(activated_account, + cache_default_token_data, + cache_statement, cache_preferences, celery_session_worker, init_cache, @@ -60,7 +62,7 @@ def test_get_cached_statement(activated_account, cache_statement, statement): assert cached_statement[0].get('blockchain_address') == statement[0].get('blockchain_address') -def test_parse_statement_transactions(statement): +def test_parse_statement_transactions(cache_default_token_data, statement): parsed_transactions = parse_statement_transactions(statement) parsed_transaction = parsed_transactions[0] parsed_transaction.startswith('Sent') @@ -76,7 +78,7 @@ def test_query_statement(blockchain_address, limit, load_chain_spec, activated_a assert mock_transaction_list_query.get('limit') == limit -def test_statement_transaction_set(preferences, set_locale_files, statement): +def test_statement_transaction_set(cache_default_token_data, load_chain_spec, preferences, set_locale_files, statement): parsed_transactions = parse_statement_transactions(statement) preferred_language = preferences.get('preferred_language') transaction_set = statement_transaction_set(preferred_language, parsed_transactions) diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py b/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py index de8ac516..45d7a105 100644 --- a/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py +++ b/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py @@ -36,19 +36,19 @@ def test_aux_transaction_data(preferences, set_locale_files, transactions_list): check_aux_data('helpers.sent', 'helpers.to', preferred_language, sender_tx_aux_data) -@pytest.mark.parametrize("wei, expected_result", [ +@pytest.mark.parametrize("value, expected_result", [ (50000000, Decimal('50.00')), (100000, Decimal('0.10')) ]) -def test_from_wei(wei, expected_result): - assert from_wei(wei) == expected_result +def test_from_wei(cache_default_token_data, expected_result, value): + assert from_wei(value) == expected_result @pytest.mark.parametrize("value, expected_result", [ (50, 50000000), (0.10, 100000) ]) -def test_to_wei(value, expected_result): +def test_to_wei(cache_default_token_data, expected_result, value): assert to_wei(value) == expected_result @@ -96,6 +96,7 @@ def test_validate_transaction_account(activated_account, init_database, transact @pytest.mark.parametrize("amount", [50, 0.10]) def test_outgoing_transaction_processor(activated_account, amount, + cache_default_token_data, celery_session_worker, load_config, load_chain_spec, diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py index 3e9a698e..422c4829 100644 --- a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py @@ -1,5 +1,6 @@ # standard imports import json +import datetime # external imports from chainlib.hash import strip_0x @@ -14,6 +15,7 @@ from cic_ussd.account.statement import ( ) from cic_ussd.account.tokens import get_default_token_symbol from cic_ussd.account.transaction import from_wei, to_wei +from cic_ussd.cache import cache_data, cache_data_key from cic_ussd.menu.ussd_menu import UssdMenu from cic_ussd.metadata import PersonMetadata from cic_ussd.phone_number import Support @@ -38,24 +40,34 @@ def test_menu_processor(activated_account, load_chain_spec, load_support_phone, load_ussd_menu, + mock_get_adjusted_balance, mock_sync_balance_api_query, mock_transaction_list_query, valid_recipient): preferred_language = get_cached_preferred_language(activated_account.blockchain_address) available_balance = get_cached_available_balance(activated_account.blockchain_address) token_symbol = get_default_token_symbol() - - tax = '' - bonus = '' - display_key = 'ussd.kenya.account_balances' + with_available_balance = 'ussd.kenya.account_balances.available_balance' + with_fees = 'ussd.kenya.account_balances.with_fees' ussd_menu = UssdMenu.find_by_name('account_balances') name = ussd_menu.get('name') - resp = response(activated_account, display_key, name, init_database, generic_ussd_session) - assert resp == translation_for(display_key, + resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session) + assert resp == translation_for(with_available_balance, preferred_language, available_balance=available_balance, + token_symbol=token_symbol) + + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + key = cache_data_key(identifier, ':cic.adjusted_balance') + adjusted_balance = 45931650.64654012 + cache_data(key, json.dumps(adjusted_balance)) + resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session) + tax_wei = to_wei(int(available_balance)) - int(adjusted_balance) + tax = from_wei(int(tax_wei)) + assert resp == translation_for(key=with_fees, + preferred_language=preferred_language, + available_balance=available_balance, tax=tax, - bonus=bonus, token_symbol=token_symbol) cached_statement = get_cached_statement(activated_account.blockchain_address) @@ -123,6 +135,15 @@ def test_menu_processor(activated_account, account_balance=available_balance, account_token_name=token_symbol) + display_key = 'ussd.kenya.start' + ussd_menu = UssdMenu.find_by_name('start') + name = ussd_menu.get('name') + older_timestamp = (activated_account.created - datetime.timedelta(days=35)) + activated_account.created = older_timestamp + init_database.flush() + response(activated_account, display_key, name, init_database, generic_ussd_session) + assert mock_get_adjusted_balance['timestamp'] == int((datetime.datetime.now() - datetime.timedelta(days=30)).timestamp()) + display_key = 'ussd.kenya.transaction_pin_authorization' ussd_menu = UssdMenu.find_by_name('transaction_pin_authorization') name = ussd_menu.get('name') diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py index 6038f687..d7b894fd 100644 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py @@ -49,6 +49,7 @@ def test_is_valid_transaction_amount(activated_account, amount, expected_result, ]) def test_has_sufficient_balance(activated_account, cache_balances, + cache_default_token_data, expected_result, generic_ussd_session, init_database, diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py index f70e220c..b08472b0 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py @@ -121,6 +121,7 @@ def test_statement_callback(activated_account, mocker, transactions_list): def test_transaction_balances_callback(activated_account, balances, cache_balances, + cache_default_token_data, cache_person_metadata, cache_preferences, load_chain_spec, diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py index 8ad7faab..cb554d8b 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py @@ -13,7 +13,8 @@ from cic_ussd.translation import translation_for # tests imports -def test_transaction(celery_session_worker, +def test_transaction(cache_default_token_data, + celery_session_worker, load_support_phone, mock_notifier_api, notification_data, diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py index 999bd4fb..63a5edd7 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py @@ -30,10 +30,11 @@ def test_generate_statement(activated_account, def test_cache_statement(activated_account, + cache_default_token_data, cache_person_metadata, + cache_preferences, celery_session_worker, init_database, - preferences, transaction_result): recipient_transaction, sender_transaction = transaction_actors(transaction_result) identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) @@ -41,7 +42,7 @@ def test_cache_statement(activated_account, cached_statement = get_cached_data(key) assert cached_statement is None s_parse_transaction = celery.signature( - 'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction]) + 'cic_ussd.tasks.processor.parse_transaction', [sender_transaction]) result = s_parse_transaction.apply_async().get() s_cache_statement = celery.signature( 'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address] @@ -61,15 +62,15 @@ def test_cache_statement(activated_account, def test_parse_transaction(activated_account, cache_person_metadata, + cache_preferences, celery_session_worker, init_database, - preferences, transaction_result): recipient_transaction, sender_transaction = transaction_actors(transaction_result) assert sender_transaction.get('metadata_id') is None assert sender_transaction.get('phone_number') is None s_parse_transaction = celery.signature( - 'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction]) + 'cic_ussd.tasks.processor.parse_transaction', [sender_transaction]) result = s_parse_transaction.apply_async().get() assert result.get('metadata_id') == activated_account.standard_metadata_id() assert result.get('phone_number') == activated_account.phone_number diff --git a/apps/cic-ussd/tests/fixtures/patches/account.py b/apps/cic-ussd/tests/fixtures/patches/account.py index 687e192a..d63b556f 100644 --- a/apps/cic-ussd/tests/fixtures/patches/account.py +++ b/apps/cic-ussd/tests/fixtures/patches/account.py @@ -41,6 +41,22 @@ def mock_async_balance_api_query(mocker): return query_args +@pytest.fixture(scope='function') +def mock_get_adjusted_balance(mocker, task_uuid): + query_args = {} + + def get_adjusted_balance(self, token_symbol, balance, timestamp): + sync_res = mocker.patch('celery.result.AsyncResult') + sync_res.id = task_uuid + sync_res.result = 45931650.64654012 + query_args['balance'] = balance + query_args['timestamp'] = timestamp + query_args['token_symbol'] = token_symbol + return sync_res + mocker.patch('cic_eth_aux.erc20_demurrage_token.api.Api.get_adjusted_balance', get_adjusted_balance) + return query_args + + @pytest.fixture(scope='function') def mock_notifier_api(mocker): sms = {} diff --git a/apps/cic-ussd/var/lib/locale/ussd.en.yml b/apps/cic-ussd/var/lib/locale/ussd.en.yml index 2fe70f30..70ae54fd 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.en.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.en.yml @@ -141,12 +141,22 @@ en: 0. Back retry: |- %{retry_pin_entry} - account_balances: |- - CON Your balances are as follows: - balance: %{available_balance} %{token_symbol} - fees: %{tax} %{token_symbol} - rewards: %{bonus} %{token_symbol} - 0. Back + account_balances: + available_balance: |- + CON Your balances are as follows: + balance: %{available_balance} %{token_symbol} + 0. Back + with_fees: |- + CON Your balances are as follows: + balances: %{available_balance} %{token_symbol} + fees: %{tax} %{token_symbol} + 0. Back + with_rewards: |- + CON Your balances are as follows: + balance: %{available_balance} %{token_symbol} + fees: %{tax} %{token_symbol} + rewards: %{bonus} %{token_symbol} + 0. Back first_transaction_set: |- CON %{first_transaction_set} 1. Next diff --git a/apps/cic-ussd/var/lib/locale/ussd.sw.yml b/apps/cic-ussd/var/lib/locale/ussd.sw.yml index 5c001f75..4b74df49 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.sw.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.sw.yml @@ -140,12 +140,22 @@ sw: 0. Nyuma retry: |- %{retry_pin_entry} - account_balances: |- - CON Salio zako ni zifuatazo: - salio: %{available_balance} %{token_symbol} - ushuru: %{tax} %{token_symbol} - tuzo: %{bonus} %{token_symbol} - 0. Nyuma + account_balances: + available_balance: |- + CON Salio zako ni zifuatazo: + salio: %{available_balance} %{token_symbol} + 0. Nyuma + with_fees: |- + CON Salio zako ni zifuatazo: + salio: %{available_balance} %{token_symbol} + ushuru: %{tax} %{token_symbol} + 0. Nyuma + with_rewards: |- + CON Salio zako ni zifuatazo: + salio: %{available_balance} %{token_symbol} + ushuru: %{tax} %{token_symbol} + tuzo: %{bonus} %{token_symbol} + 0. Nyuma first_transaction_set: |- CON %{first_transaction_set} 1. Mbele From fb8db3ffd2c347f0fd56f96ba305fc1b5670a05b Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Sat, 18 Sep 2021 09:07:05 +0000 Subject: [PATCH 04/13] Philip/ussd cli fix --- apps/cic-eth-aux/erc20-demurrage-token/requirements.txt | 5 ++--- apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/tools_requirements.txt | 2 +- apps/cic-ussd/cic_ussd/state_machine/logic/account.py | 5 ++--- apps/cic-ussd/config/client/app.ini | 2 -- apps/cic-ussd/config/client/ussd.ini | 1 + apps/cic-ussd/requirements.txt | 2 +- apps/data-seeding/requirements.txt | 2 +- docker-compose.yml | 9 ++++----- 9 files changed, 13 insertions(+), 17 deletions(-) delete mode 100644 apps/cic-ussd/config/client/app.ini diff --git a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt index 764056bb..04e3428b 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt @@ -1,5 +1,4 @@ celery==4.4.7 erc20-demurrage-token~=0.0.3a1 -cic-eth-registry~=0.5.8a1 -chainlib~=0.0.7a1 -cic_eth~=0.12.2a4 +cic-eth-registry>=0.6.1a2,<0.7.0 +cic-eth[services]~=0.12.4a8 diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 46482052..61fb0011 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.9a11,<0.1.0 +chainlib-eth>=0.0.9a14,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 4a6a7fee..f10c88e9 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,4 +1,4 @@ -crypto-dev-signer>=0.4.15a4,<=0.4.15 +crypto-dev-signer>=0.4.15a7,<=0.4.15 chainqueue>=0.0.5a1,<0.1.0 cic-eth-registry>=0.6.1a2,<0.7.0 redis==3.5.3 diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/account.py b/apps/cic-ussd/cic_ussd/state_machine/logic/account.py index ca08e41c..13bc13c0 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/account.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/account.py @@ -154,15 +154,14 @@ def parse_person_metadata(account: Account, metadata: dict): phone_number = account.phone_number date_registered = int(account.created.replace().timestamp()) blockchain_address = account.blockchain_address - chain_spec = f'{Chain.spec.common_name()}:{Chain.spec.engine()}: {Chain.spec.chain_id()}' + chain_str = Chain.spec.__str__() if isinstance(metadata.get('identities'), dict): identities = metadata.get('identities') else: identities = manage_identity_data( blockchain_address=blockchain_address, - blockchain_type=Chain.spec.engine(), - chain_spec=chain_spec + chain_str=chain_str ) return { diff --git a/apps/cic-ussd/config/client/app.ini b/apps/cic-ussd/config/client/app.ini deleted file mode 100644 index fa041acb..00000000 --- a/apps/cic-ussd/config/client/app.ini +++ /dev/null @@ -1,2 +0,0 @@ -[app] -service_code = *483*46# diff --git a/apps/cic-ussd/config/client/ussd.ini b/apps/cic-ussd/config/client/ussd.ini index 633ad403..1e9d0fd1 100644 --- a/apps/cic-ussd/config/client/ussd.ini +++ b/apps/cic-ussd/config/client/ussd.ini @@ -1,3 +1,4 @@ [ussd] +service_code = *483*46# user = pass = diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 3d274059..66ac133a 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -6,7 +6,7 @@ celery==4.4.7 cffi==1.14.6 cic-eth[services]~=0.12.4a7 cic-notify~=0.4.0a10 -cic-types~=0.1.0a14 +cic-types~=0.1.0a15 confini>=0.4.1a1,<0.5.0 phonenumbers==8.12.12 psycopg2==2.8.6 diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 4c445443..eaa28ed8 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,6 +1,6 @@ sarafu-faucet~=0.0.7a1 cic-eth[tools]~=0.12.4a8 -cic-types~=0.1.0a14 +cic-types~=0.1.0a15 crypto-dev-signer>=0.4.15a4,<=0.4.15 faker==4.17.1 chainsyncer~=0.0.6a3 diff --git a/docker-compose.yml b/docker-compose.yml index c6cfd6eb..4c5b8f5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,10 +94,11 @@ services: RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE - TOKEN_NAME: ${TOKEN_NAME:-Sarafu Token} - TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} + TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token} + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET} TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD + TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS @@ -259,7 +260,6 @@ services: CELERY_DEBUG: ${CELERY_DEBUG:-1} #SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000} - SIGNER_PROVIDER: http://cic-eth-signer:8000 SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} @@ -357,10 +357,9 @@ services: CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} - TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS + TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} restart: on-failure depends_on: - eth From 5128c7828cf51685f353ae947a86317ebc24a40f Mon Sep 17 00:00:00 2001 From: Blair V Date: Mon, 20 Sep 2021 09:59:12 +0300 Subject: [PATCH 05/13] chore: clean old dockerfile and add docs for clearing cache --- README.md | 5 ++++- apps/data-seeding/docker/Dockerfile_ci | 24 ------------------------ 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 apps/data-seeding/docker/Dockerfile_ci diff --git a/README.md b/README.md index b6fae088..9e0d62d0 100644 --- a/README.md +++ b/README.md @@ -34,4 +34,7 @@ rebuild an images docker-compose up --build ``` - +to delete the buildkit cache +``` +docker builder prune --filter type=exec.cachemount +``` diff --git a/apps/data-seeding/docker/Dockerfile_ci b/apps/data-seeding/docker/Dockerfile_ci deleted file mode 100644 index 8169b4db..00000000 --- a/apps/data-seeding/docker/Dockerfile_ci +++ /dev/null @@ -1,24 +0,0 @@ -# syntax = docker/dockerfile:1.2 -FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45 - -WORKDIR /root - -RUN mkdir -vp /usr/local/etc/cic - -COPY package.json \ - package-lock.json \ - . - -RUN npm install - -COPY requirements.txt . - -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -RUN pip install \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL -r requirements.txt - -COPY . . - -ENTRYPOINT [ ] From e89aec76fa94107a1aa228e4580f879ffd0b94b0 Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Tue, 21 Sep 2021 05:33:46 +0000 Subject: [PATCH 06/13] Fixes migrations in cic-cache (ideally) --- apps/cic-cache/cic_cache/cli/rpc.py | 4 ++-- apps/cic-cache/requirements.txt | 7 +++---- apps/contract-migration/config.sh | 7 +++++-- apps/contract-migration/config/config.ini | 10 ---------- apps/contract-migration/reset.sh | 8 ++++++-- apps/contract-migration/seed_cic_eth.sh | 2 +- docker-compose.yml | 18 ++++++++++-------- scripts/dump.sh | 2 +- 8 files changed, 28 insertions(+), 30 deletions(-) mode change 100644 => 100755 scripts/dump.sh diff --git a/apps/cic-cache/cic_cache/cli/rpc.py b/apps/cic-cache/cic_cache/cli/rpc.py index 7f6500ae..b7383242 100644 --- a/apps/cic-cache/cic_cache/cli/rpc.py +++ b/apps/cic-cache/cic_cache/cli/rpc.py @@ -27,11 +27,11 @@ class RPC: @staticmethod def from_config(config): chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) - RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default') if config.get('SIGNER_PROVIDER'): RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer') RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') - rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) logg.info('set up rpc: {}'.format(rpc)) return rpc diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index b2216817..8f47f9b5 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -9,7 +9,6 @@ psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 chainsyncer[sql]>=0.0.6a3,<0.1.0 -erc20-faucet>=0.3.2a1, <0.4.0 -chainlib-eth>=0.0.9a7,<0.1.0 -chainlib>=0.0.9a3,<0.1.0 -eth-address-index>=0.2.3a1,<0.3.0 +erc20-faucet>=0.3.2a2, <0.4.0 +chainlib-eth>=0.0.9a14,<0.1.0 +eth-address-index>=0.2.3a4,<0.3.0 diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index 858eaf85..819b131c 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -35,8 +35,11 @@ export TOKEN_SINK_ADDRESS=${TOKEN_SINK_ADDRESS:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOY # Migration variable processing - -confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset +confini-dump --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset +echo "export CIC_TRUST_ADDRESS=$CIC_TRUST_ADDRESS +export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL +export WALLET_KEY_FILE=$WALLET_KEY_FILE +" >> ${DEV_DATA_DIR}/env_reset cat ${DEV_DATA_DIR}/env_reset diff --git a/apps/contract-migration/config/config.ini b/apps/contract-migration/config/config.ini index 4bce6a78..3c30a030 100644 --- a/apps/contract-migration/config/config.ini +++ b/apps/contract-migration/config/config.ini @@ -1,13 +1,3 @@ -[token] -name = Giftable Token -symbol = GFT -type = giftable_erc20_token -demurrage_level = 196454828847045000000000000000000 -redistribution_period = -supply_limit = -sink_address = - - [dev] eth_account_contract_deployer = eth_account_reserve_minter = diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 728356ce..f4d45f7f 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -110,8 +110,12 @@ giftable-token-minter -s -u $fee_price_arg -w -y $WALLET_KEY_FILE -e $DEV_RESERV #echo "export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL" >> ${DEV_DATA_DIR}/env_reset export CIC_DEFAULT_TOKEN_SYMBOL=$TOKEN_SYMBOL -confini-dump -vv --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config --prefix export > ${DEV_DATA_DIR}/env_reset -confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config +echo "Writing env_reset file ..." + +echo "export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS +export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL +export TOKEN_NAME=$TOKEN_NAME +" >> "${DEV_DATA_DIR}"/env_reset set +a set +e diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index eb1bfd59..4d0fcd73 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -82,7 +82,7 @@ cic-eth-ctl -i $CHAIN_SPEC unlock INIT cic-eth-ctl -i $CHAIN_SPEC unlock SEND cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE -confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config +#confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config set +a set +e diff --git a/docker-compose.yml b/docker-compose.yml index 4c5b8f5f..bcc39136 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,8 +126,8 @@ services: EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} - RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} + RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -217,6 +217,8 @@ services: - ${HTTP_PORT_CIC_CACHE:-63313}:8000 depends_on: - postgres + - cic-cache-tasker + - cic-cache-tracker command: - /bin/bash - -c @@ -240,7 +242,7 @@ services: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -294,7 +296,7 @@ services: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -344,8 +346,8 @@ services: args: EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: - RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} - ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -387,7 +389,7 @@ services: EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 - RPC_HTTP_PROVIDER: http://eth:8545 + RPC_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -431,7 +433,7 @@ services: EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 - RPC_HTTP_PROVIDER: http://eth:8545 + RPC_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} diff --git a/scripts/dump.sh b/scripts/dump.sh old mode 100644 new mode 100755 index dd02ea32..814c451f --- a/scripts/dump.sh +++ b/scripts/dump.sh @@ -1 +1 @@ -docker run -t -v cic-internal-integration_contract-config:/tmp/cic/config cic-internal-integration_contract-migration cat /tmp/cic/config/env_reset +docker run -t -v --rm cic-internal-integration_contract-config:/tmp/cic/config busybox cat /tmp/cic/config/env_reset From 7b57f1b4c2a7decae34bea20f9351f44fb416fc1 Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Tue, 21 Sep 2021 06:04:44 +0000 Subject: [PATCH 07/13] Dataseeding container --- .gitignore | 1 + .gitlab-ci.yml | 6 +- apps/data-seeding/cic_ussd/import_balance.py | 2 +- apps/data-seeding/config/cic.ini | 7 +-- apps/data-seeding/create_import_users.py | 9 ++- apps/data-seeding/docker/Dockerfile | 3 +- apps/data-seeding/import_ussd.sh | 64 +++++++++++++++----- apps/data-seeding/requirements.txt | 9 +-- apps/data-seeding/verify.py | 14 +---- ci_templates/.cic-template.yml | 12 +++- docker-compose.yml | 47 +++++++++++++- 11 files changed, 123 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index 3fb25a37..bd35ec75 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ build/ **/.venv .idea **/.vim +**/*secret.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 584d01c1..67ccdc9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,13 +12,13 @@ include: stages: - build - test - - deploy + - deploy -image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest +image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest variables: DOCKER_BUILDKIT: "1" - COMPOSE_DOCKER_CLI_BUILD: "1" + COMPOSE_DOCKER_CLI_BUILD: "1" CI_DEBUG_TRACE: "true" before_script: diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 9f974e70..8fb487a3 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -16,7 +16,7 @@ from crypto_dev_signer.keystore.dict import DictKeystore from import_util import BalanceProcessor, get_celery_worker_status from import_task import ImportTask, MetadataTask -default_config_dir = './config' +default_config_dir = '/usr/local/etc/data-seeding/' logg = logging.getLogger() arg_parser = argparse.ArgumentParser(description='Daemon worker that handles data seeding tasks.') diff --git a/apps/data-seeding/config/cic.ini b/apps/data-seeding/config/cic.ini index 5064df15..2e346dae 100644 --- a/apps/data-seeding/config/cic.ini +++ b/apps/data-seeding/config/cic.ini @@ -1,11 +1,10 @@ [cic] -registry_address = 0x32E860c2A0645d1B7B005273696905F5D6DC5D05 +registry_address = token_index_address = accounts_index_address = declarator_address = approval_escrow_address = -chain_spec = evm:bloxberg:8996 +chain_spec = tx_retry_delay = -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C +trust_address = user_ussd_svc_service_port = - diff --git a/apps/data-seeding/create_import_users.py b/apps/data-seeding/create_import_users.py index aa8e4e59..01d58c09 100644 --- a/apps/data-seeding/create_import_users.py +++ b/apps/data-seeding/create_import_users.py @@ -28,12 +28,11 @@ logg = logging.getLogger() fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) -script_dir = os.path.realpath(os.path.dirname(__file__)) -# config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -config_dir = os.environ.get('CONFINI_DIR', os.path.join(script_dir, 'config')) +default_config_dir = './config' + argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='Config dir') +argparser.add_argument('-c', type=str, default=default_config_dir, help='Config dir') argparser.add_argument('--tag', type=str, action='append', help='Tags to add to record') argparser.add_argument('--gift-threshold', type=int, @@ -53,7 +52,7 @@ elif args.vv: config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX')) config.process() -logg.info('loaded config\n{}'.format(config)) +logg.debug('loaded config\n{}'.format(config)) dt_now = datetime.datetime.utcnow() diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index 4113c2d1..f993e33c 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -13,7 +13,8 @@ COPY package.json \ RUN npm ci --production #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install -COPY requirements.txt . +COPY requirements.txt . +COPY config/ /usr/local/etc/data-seeding ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh index 02773481..1eb7e3f6 100644 --- a/apps/data-seeding/import_ussd.sh +++ b/apps/data-seeding/import_ussd.sh @@ -1,60 +1,94 @@ -#!/usr/bin/env bash +#!/bin/bash -set -e +if [[ -d "$OUT_DIR" ]] +then + echo "found existing IMPORT DIR cleaning up..." + rm -rf "$OUT_DIR" + mkdir -p "$OUT_DIR" +else + echo "IMPORT DIR does not exist creating it." + mkdir -p "$OUT_DIR" +fi + +# using timeout because the timeout flag for celery inspect does not work +timeout 5 celery inspect ping -b "$CELERY_BROKER_URL" +if [[ $? -eq 124 ]] +then + >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?" + exit 1 +fi echo "Creating seed data..." -python create_import_users.py -vv --dir "$IMPORT_DIR" "$ACCOUNT_COUNT" +python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" wait $! + +echo "Check for running celery workers ..." +if [ -f ./cic-ussd-import.pid ]; +then + echo "Found a running worker. Killing ..." + kill -9 $( nohup.out 2> nohup.err < /dev/null & else echo "Running worker with opening balance transactions" - TARGET_TX_COUNT=$((ACCOUNT_COUNT*2)) - python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$IMPORT_DIR" & + TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) + nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" & fi +echo "Target count set to ${TARGET_TX_COUNT}" until [ -f ./cic-import-ussd.pid ] do echo "Polling for celery worker pid file..." sleep 1 done IMPORT_BALANCE_JOB=$(=0.4.15a4,<=0.4.15 +crypto-dev-signer~=0.4.15a7 faker==4.17.1 chainsyncer~=0.0.6a3 -chainlib-eth~=0.0.9a13 +chainlib-eth~=0.0.9a14 eth-address-index~=0.2.3a4 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3 -eth-erc20~=0.1.2a2 +eth-erc20~=0.1.2a3 erc20-faucet~=0.3.2a2 psycopg2==2.8.6 +liveness~=0.0.1a7 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 3db3e919..216b26ef 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -164,20 +164,8 @@ for t in custodial_tests: logg.info('activating custodial module'.format(t)) break -cols = os.get_terminal_size().columns - -def to_terminalwidth(s): - ss = s.ljust(int(cols)-1) - ss += "\r" - return ss - -def default_outfunc(s): - ss = to_terminalwidth(s) - sys.stdout.write(ss) -outfunc = default_outfunc -if logg.isEnabledFor(logging.DEBUG): - outfunc = logg.debug +outfunc = logg.debug def send_ussd_request(address, data_dir): diff --git a/ci_templates/.cic-template.yml b/ci_templates/.cic-template.yml index 5ba47394..2ecba76b 100644 --- a/ci_templates/.cic-template.yml +++ b/ci_templates/.cic-template.yml @@ -1,12 +1,12 @@ -image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] variables: KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=24h" MR_IMAGE_TAG: $CI_REGISTRY_IMAGE/mergerequest/$APP_NAME:$CI_COMMIT_SHORT_SHA .py_build_merge_request: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build script: - mkdir -p /kaniko/.docker @@ -16,6 +16,9 @@ variables: --cache-repo $CI_REGISTRY_IMAGE --destination $MR_IMAGE_TAG .py_build_target_dev: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build variables: IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA @@ -28,6 +31,9 @@ variables: --destination $MR_IMAGE_TAG .py_build_push: + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build variables: IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA diff --git a/docker-compose.yml b/docker-compose.yml index 4c5b8f5f..143f9818 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,8 +94,8 @@ services: RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE - TOKEN_NAME: ${TOKEN_NAME:-Demurrage Token} - TOKEN_SYMBOL: ${TOKEN_SYMBOL:-DET} + TOKEN_NAME: ${TOKEN_NAME:-Giftable Token} + TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} @@ -115,6 +115,49 @@ services: volumes: - contract-config:/tmp/cic/config + + data-seeding: + image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} + pip_extra_args: $PIP_EXTRA_ARGS + environment: + CIC_REGISTRY_ADDRESS: ${CIC_REGISTRY_ADDRESS:-0xea6225212005e86a4490018ded4bf37f3e772161} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: /usr/local/etc/data-seeding + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} + TOKEN_SYMBOL: GFT + KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: bash import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro + cic-cache-tracker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} profiles: From cf64387d81f092834ef610a8778dacdf12bccbee Mon Sep 17 00:00:00 2001 From: PhilipWafula Date: Wed, 22 Sep 2021 11:10:33 +0300 Subject: [PATCH 08/13] Refactors to avoid conversion of zero values to wei. --- apps/cic-ussd/cic_ussd/tasks/notifications.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/cic-ussd/cic_ussd/tasks/notifications.py b/apps/cic-ussd/cic_ussd/tasks/notifications.py index e28f1567..8cab098e 100644 --- a/apps/cic-ussd/cic_ussd/tasks/notifications.py +++ b/apps/cic-ussd/cic_ussd/tasks/notifications.py @@ -24,7 +24,8 @@ def transaction(notification_data: dict): :rtype: """ role = notification_data.get('role') - amount = from_wei(notification_data.get('token_value')) + token_value = notification_data.get('token_value') + amount = token_value if token_value == 0 else from_wei(token_value) balance = notification_data.get('available_balance') phone_number = notification_data.get('phone_number') preferred_language = notification_data.get('preferred_language') From 1a7c4deab617f11cf28fa4e0dea2800984bb584b Mon Sep 17 00:00:00 2001 From: PhilipWafula Date: Mon, 27 Sep 2021 14:27:10 +0300 Subject: [PATCH 09/13] Bumps cic-ussd version following token fix. --- apps/cic-ussd/cic_ussd/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/cic-ussd/cic_ussd/version.py b/apps/cic-ussd/cic_ussd/version.py index 4b0d5e3e..3c7d24bf 100644 --- a/apps/cic-ussd/cic_ussd/version.py +++ b/apps/cic-ussd/cic_ussd/version.py @@ -1,7 +1,7 @@ # standard imports import semver -version = (0, 3, 1, 'alpha.4') +version = (0, 3, 1, 'alpha.5') version_object = semver.VersionInfo( major=version[0], From 3aef2aa65fd3fa87673d10b6a62230c85b1a3670 Mon Sep 17 00:00:00 2001 From: PhilipWafula Date: Mon, 27 Sep 2021 15:26:22 +0300 Subject: [PATCH 10/13] removes profile attrs from docker-compose file. --- docker-compose.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 92beb0f4..737db350 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,8 +60,6 @@ services: contract-migration: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest} - profiles: - - migrations build: context: apps/contract-migration dockerfile: docker/Dockerfile @@ -160,8 +158,6 @@ services: cic-cache-tracker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -200,8 +196,6 @@ services: cic-cache-tasker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -239,8 +233,6 @@ services: cic-cache-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - profiles: - - cache build: context: apps/cic-cache dockerfile: docker/Dockerfile @@ -541,8 +533,6 @@ services: cic-meta-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest} - profiles: - - custodial-meta hostname: meta build: context: apps/cic-meta @@ -575,8 +565,6 @@ services: cic-user-ussd-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd dockerfile: docker/Dockerfile @@ -606,8 +594,6 @@ services: cic-user-server: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd dockerfile: docker/Dockerfile @@ -630,8 +616,6 @@ services: cic-user-tasker: image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - profiles: - - custodial-ussd build: context: apps/cic-ussd/ dockerfile: docker/Dockerfile From 3608fd1fc7151dbebeae26af59ec28c9b51a1fe2 Mon Sep 17 00:00:00 2001 From: Blair V Date: Mon, 27 Sep 2021 16:14:31 +0300 Subject: [PATCH 11/13] chore: gift defaults --- docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 737db350..9d59d79a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ services: environment: RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} ETH_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} DEV_USE_DOCKER_WAIT_SCRIPT: 1 CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} @@ -90,17 +91,17 @@ services: CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data - DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-0} + DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-50000000} DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE TOKEN_NAME: ${TOKEN_NAME:-Giftable Token} TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} + TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token} TOKEN_DECIMALS: $TOKEN_DECIMALS TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS - TOKEN_TYPE: $TOKEN_TYPE SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} restart: on-failure command: ["./run_job.sh"] From f5bfc8ace26b1227664b37b84d50927834da41c5 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Thu, 7 Oct 2021 15:12:35 +0000 Subject: [PATCH 12/13] bug: cic-eth transaction listings --- apps/cic-eth/admin_requirements.txt | 2 +- apps/cic-eth/cic_eth/api/api_task.py | 3 ++ apps/cic-eth/cic_eth/eth/account.py | 26 ++++++++++----- apps/cic-eth/cic_eth/eth/erc20.py | 20 +++++++---- apps/cic-eth/cic_eth/eth/gas.py | 15 ++++++--- apps/cic-eth/cic_eth/eth/meta.py | 7 ++-- apps/cic-eth/cic_eth/eth/nonce.py | 11 ++++--- apps/cic-eth/cic_eth/ext/tx.py | 13 +++++--- apps/cic-eth/cic_eth/queue/query.py | 1 + .../runnable/daemons/filters/callback.py | 13 ++++---- .../runnable/daemons/filters/register.py | 3 +- .../runnable/daemons/filters/transferauth.py | 7 ++-- .../cic_eth/runnable/daemons/tasker.py | 2 +- apps/cic-eth/requirements.txt | 3 +- apps/cic-eth/services_requirements.txt | 2 +- .../tests/filters/test_callback_filter.py | 10 ++++-- .../tests/filters/test_register_filter.py | 11 ++++--- .../tests/filters/test_transferauth_filter.py | 6 ++-- apps/cic-eth/tests/task/api/test_admin.py | 4 +-- apps/cic-eth/tests/task/api/test_app.py | 9 +++++ apps/cic-eth/tests/task/api/test_balance.py | 9 +++-- apps/cic-eth/tests/task/api/test_list.py | 33 ++++++++++++------- apps/cic-eth/tests/task/test_ext_tx.py | 3 +- apps/cic-eth/tools_requirements.txt | 2 +- apps/cic-ussd/cic_ussd/account/balance.py | 2 +- apps/cic-ussd/cic_ussd/account/metadata.py | 3 +- apps/cic-ussd/cic_ussd/account/statement.py | 2 +- apps/cic-ussd/cic_ussd/db/models/account.py | 5 ++- apps/cic-ussd/cic_ussd/processor/menu.py | 2 +- .../cic_ussd/tasks/callback_handler.py | 6 ++-- apps/cic-ussd/cic_ussd/tasks/metadata.py | 15 ++++----- apps/cic-ussd/cic_ussd/tasks/processor.py | 9 ++--- .../tests/cic_ussd/account/test_statement.py | 33 +++++++++---------- .../tests/cic_ussd/processor/test_menu.py | 4 +-- .../cic_ussd/tasks/test_callback_handler.py | 1 + apps/cic-ussd/tests/fixtures/account.py | 12 +++---- apps/cic-ussd/tests/fixtures/transaction.py | 14 ++++---- apps/cic-ussd/tests/helpers/accounts.py | 3 +- 38 files changed, 195 insertions(+), 131 deletions(-) diff --git a/apps/cic-eth/admin_requirements.txt b/apps/cic-eth/admin_requirements.txt index 03f27205..4e9b1a16 100644 --- a/apps/cic-eth/admin_requirements.txt +++ b/apps/cic-eth/admin_requirements.txt @@ -1,5 +1,5 @@ SQLAlchemy==1.3.20 -cic-eth-registry>=0.6.1a2,<0.7.0 +cic-eth-registry>=0.6.1a3,<0.7.0 hexathon~=0.0.1a8 chainqueue>=0.0.4a6,<0.1.0 eth-erc20>=0.1.2a2,<0.2.0 diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 79501d30..1ae14123 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -9,6 +9,7 @@ import logging # external imports import celery from chainlib.chain import ChainSpec +from hexathon import strip_0x # local imports from cic_eth.api.base import ApiBase @@ -254,6 +255,8 @@ class Api(ApiBase): :returns: uuid of root task :rtype: celery.Task """ + #from_address = strip_0x(from_address) + #to_address = strip_0x(to_address) s_check = celery.signature( 'cic_eth.admin.ctrl.check_lock', [ diff --git a/apps/cic-eth/cic_eth/eth/account.py b/apps/cic-eth/cic_eth/eth/account.py index 42bccec5..9f4916c7 100644 --- a/apps/cic-eth/cic_eth/eth/account.py +++ b/apps/cic-eth/cic_eth/eth/account.py @@ -13,7 +13,7 @@ from chainlib.eth.sign import ( new_account, sign_message, ) -from chainlib.eth.address import to_checksum_address +from chainlib.eth.address import to_checksum_address, is_address from chainlib.eth.tx import TxFormat from chainlib.chain import ChainSpec from chainlib.error import JSONRPCException @@ -31,6 +31,7 @@ from cic_eth.eth.gas import ( from cic_eth.db.models.nonce import Nonce from cic_eth.db.models.base import SessionBase from cic_eth.db.models.role import AccountRole +from cic_eth.encode import tx_normalize from cic_eth.error import ( RoleMissingError, SignerError, @@ -49,6 +50,7 @@ from cic_eth.queue.tx import ( from cic_eth.encode import ( unpack_normal, ZERO_ADDRESS_NORMAL, + tx_normalize, ) logg = logging.getLogger() @@ -84,7 +86,7 @@ def create(self, password, chain_spec_dict): # TODO: It seems infeasible that a can be None in any case, verify if a == None: raise SignerError('create account') - + a = tx_normalize.wallet_address(a) logg.debug('created account {}'.format(a)) # Initialize nonce provider record for account @@ -175,6 +177,9 @@ def gift(self, account_address, chain_spec_dict): """ chain_spec = ChainSpec.from_dict(chain_spec_dict) + if is_address(account_address): + account_address = tx_normalize.wallet_address(account_address) + logg.debug('gift account address {} to index'.format(account_address)) queue = self.request.delivery_info.get('routing_key') @@ -248,8 +253,9 @@ def have(self, account, chain_spec_dict): @celery_app.task(bind=True, base=CriticalSQLAlchemyTask) def set_role(self, tag, address, chain_spec_dict): - if not to_checksum_address(address): - raise ValueError('invalid checksum address {}'.format(address)) + if not is_address(address): + raise ValueError('invalid address {}'.format(address)) + address = tx_normalize.wallet_address(address) session = SessionBase.create_session() role = AccountRole.set(tag, address, session=session) session.add(role) @@ -298,13 +304,15 @@ def cache_gift_data( tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx = unpack_normal(tx_signed_raw_bytes, chain_spec) tx_data = Faucet.parse_give_to_request(tx['data']) + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx['to']) session = self.create_session() tx_dict = { 'hash': tx['hash'], - 'from': tx['from'], - 'to': tx['to'], + 'from': sender_address, + 'to': recipient_address, 'source_token': ZERO_ADDRESS_NORMAL, 'destination_token': ZERO_ADDRESS_NORMAL, 'from_value': 0, @@ -338,12 +346,14 @@ def cache_account_data( tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex)) tx = unpack_normal(tx_signed_raw_bytes, chain_spec) tx_data = AccountsIndex.parse_add_request(tx['data']) + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx['to']) session = SessionBase.create_session() tx_dict = { 'hash': tx['hash'], - 'from': tx['from'], - 'to': tx['to'], + 'from': sender_address, + 'to': recipient_address, 'source_token': ZERO_ADDRESS_NORMAL, 'destination_token': ZERO_ADDRESS_NORMAL, 'from_value': 0, diff --git a/apps/cic-eth/cic_eth/eth/erc20.py b/apps/cic-eth/cic_eth/eth/erc20.py index 1a1448a7..cf433c9e 100644 --- a/apps/cic-eth/cic_eth/eth/erc20.py +++ b/apps/cic-eth/cic_eth/eth/erc20.py @@ -12,7 +12,10 @@ from chainlib.eth.tx import ( ) from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token -from hexathon import strip_0x +from hexathon import ( + strip_0x, + add_0x, + ) from chainqueue.error import NotLocalTxError from eth_erc20 import ERC20 from chainqueue.sql.tx import cache_tx_dict @@ -38,6 +41,7 @@ from cic_eth.task import ( CriticalSQLAlchemyAndSignerTask, ) from cic_eth.eth.nonce import CustodialTaskNonceOracle +from cic_eth.encode import tx_normalize celery_app = celery.current_app logg = logging.getLogger() @@ -62,7 +66,8 @@ def balance(tokens, holder_address, chain_spec_dict): for t in tokens: address = t['address'] - token = ERC20Token(chain_spec, rpc, address) + logg.debug('address {} {}'.format(address, holder_address)) + token = ERC20Token(chain_spec, rpc, add_0x(address)) c = ERC20(chain_spec) o = c.balance_of(address, holder_address, sender_address=caller_address) r = rpc.do(o) @@ -371,13 +376,15 @@ def cache_transfer_data( tx = unpack(tx_signed_raw_bytes, chain_spec) tx_data = ERC20.parse_transfer_request(tx['data']) - recipient_address = tx_data[0] + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] + session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, - 'from': tx['from'], + 'from': sender_address, 'to': recipient_address, 'source_token': tx['to'], 'destination_token': tx['to'], @@ -448,13 +455,14 @@ def cache_approve_data( tx = unpack(tx_signed_raw_bytes, chain_spec) tx_data = ERC20.parse_approve_request(tx['data']) - recipient_address = tx_data[0] + sender_address = tx_normalize.wallet_address(tx['from']) + recipient_address = tx_normalize.wallet_address(tx_data[0]) token_value = tx_data[1] session = SessionBase.create_session() tx_dict = { 'hash': tx_hash_hex, - 'from': tx['from'], + 'from': sender_address, 'to': recipient_address, 'source_token': tx['to'], 'destination_token': tx['to'], diff --git a/apps/cic-eth/cic_eth/eth/gas.py b/apps/cic-eth/cic_eth/eth/gas.py index 40f569af..435949fc 100644 --- a/apps/cic-eth/cic_eth/eth/gas.py +++ b/apps/cic-eth/cic_eth/eth/gas.py @@ -9,7 +9,11 @@ from hexathon import ( ) #from chainlib.eth.constant import ZERO_ADDRESS from chainlib.chain import ChainSpec -from chainlib.eth.address import is_checksum_address +from chainlib.eth.address import ( + is_checksum_address, + to_checksum_address, + is_address + ) from chainlib.connection import RPCConnection from chainqueue.db.enum import StatusBits from chainqueue.sql.tx import cache_tx_dict @@ -74,7 +78,6 @@ class MaxGasOracle: return MAXIMUM_FEE_UNITS -#def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None): """Creates a celery task signature for a check_gas task that adds the task to the outgoing queue to be processed by the dispatcher. @@ -179,8 +182,9 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga :return: Signed raw transaction data list :rtype: param txs, unchanged """ + rpc_format_address = None if address != None: - if not is_checksum_address(address): + if not is_address(address): raise ValueError('invalid address {}'.format(address)) address = tx_normalize.wallet_address(address) address = add_0x(address) @@ -195,7 +199,6 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga txs.append(tx) chain_spec = ChainSpec.from_dict(chain_spec_dict) - logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes)) addresspass = None if len(txs) == 0: @@ -211,13 +214,15 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from'])) addresspass.append(address) + rpc_format_address = add_0x(to_checksum_address(address)) + queue = self.request.delivery_info.get('routing_key') conn = RPCConnection.connect(chain_spec) gas_balance = 0 try: - o = balance(address) + o = balance(rpc_format_address) r = conn.do(o) conn.disconnect() gas_balance = abi_decode_single(ABIContractType.UINT256, r) diff --git a/apps/cic-eth/cic_eth/eth/meta.py b/apps/cic-eth/cic_eth/eth/meta.py index 21ab8295..881fad66 100644 --- a/apps/cic-eth/cic_eth/eth/meta.py +++ b/apps/cic-eth/cic_eth/eth/meta.py @@ -2,8 +2,9 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.status import Status as TxStatus from cic_eth_registry.erc20 import ERC20Token +from hexathon import add_0x -# local imports +# local impor:ts from cic_eth.ext.address import translate_address @@ -44,8 +45,8 @@ class ExtendedTx: destination = source if destination_value == None: destination_value = source_value - st = ERC20Token(self.chain_spec, self.rpc, source) - dt = ERC20Token(self.chain_spec, self.rpc, destination) + st = ERC20Token(self.chain_spec, self.rpc, add_0x(source)) + dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination)) self.source_token = source self.source_token_symbol = st.symbol self.source_token_name = st.name diff --git a/apps/cic-eth/cic_eth/eth/nonce.py b/apps/cic-eth/cic_eth/eth/nonce.py index 412e18ee..6e493ff5 100644 --- a/apps/cic-eth/cic_eth/eth/nonce.py +++ b/apps/cic-eth/cic_eth/eth/nonce.py @@ -3,11 +3,12 @@ import logging # external imports import celery -from chainlib.eth.address import is_checksum_address +from chainlib.eth.address import is_checksum_address, is_address, strip_0x # local imports from cic_eth.db.models.role import AccountRole from cic_eth.db.models.base import SessionBase +from cic_eth.encode import tx_normalize from cic_eth.task import CriticalSQLAlchemyTask from cic_eth.db.models.nonce import ( Nonce, @@ -42,7 +43,8 @@ class CustodialTaskNonceOracle(): :returns: Nonce :rtype: number """ - r = NonceReservation.release(self.address, self.uuid, session=self.session) + address = tx_normalize.wallet_address(self.address) + r = NonceReservation.release(address, self.uuid, session=self.session) return r[1] @@ -58,17 +60,18 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None): address = chained_input logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input)) else: - if is_checksum_address(signer_address): + if is_address(signer_address): address = signer_address logg.debug('explicit address for reserve nonce {}'.format(signer_address)) else: address = AccountRole.get_address(signer_address, session=session) logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address)) - if not is_checksum_address(address): + if not is_address(address): raise ValueError('invalid result when resolving address for nonce {}'.format(address)) root_id = self.request.root_id + address = tx_normalize.wallet_address(address) r = NonceReservation.next(address, root_id, session=session) logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0])) diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py index 3adefa2e..62f44d2d 100644 --- a/apps/cic-eth/cic_eth/ext/tx.py +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -32,6 +32,7 @@ from potaahto.symbols import snake_and_camel from cic_eth.queue.time import tx_times from cic_eth.task import BaseTask from cic_eth.db.models.base import SessionBase +from cic_eth.encode import tx_normalize celery_app = celery.current_app logg = logging.getLogger() @@ -134,7 +135,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_address = transfer_data[0] tx_token_value = transfer_data[1] - if address == tx_address: + if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address): status = StatusEnum.SENT try: o = receipt(tx['hash']) @@ -152,8 +153,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): times = tx_times(tx['hash'], chain_spec) tx_r = { 'hash': tx['hash'], - 'sender': tx['from'], - 'recipient': tx_address, + 'sender': tx_normalize.wallet_address(tx['from']), + 'recipient': tx_normalize.wallet_address(tx_address), 'source_value': tx_token_value, 'destination_value': tx_token_value, 'source_token': tx['to'], @@ -164,10 +165,10 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_r['date_created'] = times['queue'] else: tx_r['date_created'] = times['network'] - txs[tx['hash']] = tx_r + txs[strip_0x(tx['hash'])] = tx_r break - return txs + return txs # TODO: Surely it must be possible to optimize this @@ -230,6 +231,8 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr except UnknownContractError: logg.error('verify failed on tx {}, skipping'.format(tx['hash'])) continue + tx['recipient'] = tx_normalize.wallet_address(tx['recipient']) + tx['sender'] = tx_normalize.wallet_address(tx['sender']) txs.append(tx) return txs diff --git a/apps/cic-eth/cic_eth/queue/query.py b/apps/cic-eth/cic_eth/queue/query.py index aa0b2504..3d941599 100644 --- a/apps/cic-eth/cic_eth/queue/query.py +++ b/apps/cic-eth/cic_eth/queue/query.py @@ -58,6 +58,7 @@ def get_tx_local(chain_spec, tx_hash, session=None): @celery_app.task(base=CriticalSQLAlchemyTask) def get_account_tx(chain_spec_dict, address, as_sender=True, as_recipient=True, counterpart=None): + address = tx_normalize.wallet_address(address) chain_spec = ChainSpec.from_dict(chain_spec_dict) return get_account_tx_local(chain_spec, address, as_sender=as_sender, as_recipient=as_recipient, counterpart=counterpart) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py index 4852ab65..c834d053 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py @@ -21,6 +21,7 @@ from erc20_faucet import Faucet # local imports from .base import SyncFilter from cic_eth.eth.meta import ExtendedTx +from cic_eth.encode import tx_normalize logg = logging.getLogger().getChild(__name__) @@ -42,9 +43,9 @@ class CallbackFilter(SyncFilter): return (None, None) r = ERC20.parse_transfer_request(tx.payload) transfer_data = {} - transfer_data['to'] = r[0] + transfer_data['to'] = tx_normalize.wallet_address(r[0]) transfer_data['value'] = r[1] - transfer_data['from'] = tx.outputs[0] + transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0]) transfer_data['token_address'] = tx.inputs[0] return ('transfer', transfer_data) @@ -54,8 +55,8 @@ class CallbackFilter(SyncFilter): return (None, None) r = ERC20.parse_transfer_from_request(tx.payload) transfer_data = {} - transfer_data['from'] = r[0] - transfer_data['to'] = r[1] + transfer_data['from'] = tx_normalize.wallet_address(r[0]) + transfer_data['to'] = tx_normalize.wallet_address(r[1]) transfer_data['value'] = r[2] transfer_data['token_address'] = tx.inputs[0] return ('transferfrom', transfer_data) @@ -66,9 +67,9 @@ class CallbackFilter(SyncFilter): return (None, None) r = Faucet.parse_give_to_request(tx.payload) transfer_data = {} - transfer_data['to'] = r[0] + transfer_data['to'] = tx_normalize.wallet_address(r[0]) transfer_data['value'] = tx.value - transfer_data['from'] = tx.outputs[0] + transfer_data['from'] = tx_normalize.wallet_address(tx.outputs[0]) #transfer_data['token_address'] = tx.inputs[0] faucet_contract = tx.inputs[0] diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py index 54b8ad58..03cd774e 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/register.py @@ -12,7 +12,8 @@ from hexathon import ( # local imports from .base import SyncFilter -logg = logging.getLogger(__name__) +#logg = logging.getLogger(__name__) +logg = logging.getLogger() account_registry_add_log_hash = '0x9cc987676e7d63379f176ea50df0ae8d2d9d1141d1231d4ce15b5965f73c9430' diff --git a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py index 3ba77791..7288d349 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/filters/transferauth.py @@ -17,6 +17,7 @@ from cic_eth_registry import CICRegistry from erc20_transfer_authorization import TransferAuthorization # local imports +from cic_eth.encode import tx_normalize from .base import SyncFilter @@ -52,9 +53,9 @@ class TransferAuthFilter(SyncFilter): r = TransferAuthorization.parse_create_request_request(tx.payload) - sender = r[0] - recipient = r[1] - token = r[2] + sender = tx_normalize.wallet_address(r[0]) + recipient = tx_normalize.wallet_address(r[1]) + token = tx_normalize.executable_address(r[2]) value = r[3] token_data = { diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 52e8e1d9..d9d7a7f8 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -206,7 +206,7 @@ def main(): BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) - default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address) + default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address)) default_token.load(conn) BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 61fb0011..542580c9 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,4 @@ celery==4.4.7 -chainlib-eth>=0.0.9a14,<0.1.0 +chainlib-eth>=0.0.9rc2,<0.1.0 semver==2.13.0 +crypto-dev-signer>=0.4.15rc2,<0.5.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index 7f129268..64b23722 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -8,7 +8,7 @@ pycryptodome==3.10.1 liveness~=0.0.1a7 eth-address-index>=0.2.3a4,<0.3.0 eth-accounts-index>=0.1.2a3,<0.2.0 -cic-eth-registry>=0.6.1a2,<0.7.0 +cic-eth-registry>=0.6.1a3,<0.7.0 erc20-faucet>=0.3.2a2,<0.4.0 erc20-transfer-authorization>=0.3.5a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 diff --git a/apps/cic-eth/tests/filters/test_callback_filter.py b/apps/cic-eth/tests/filters/test_callback_filter.py index 695155cb..4b99de12 100644 --- a/apps/cic-eth/tests/filters/test_callback_filter.py +++ b/apps/cic-eth/tests/filters/test_callback_filter.py @@ -18,7 +18,10 @@ from eth_erc20 import ERC20 from sarafu_faucet import MinterFaucet from eth_accounts_index.registry import AccountRegistry from potaahto.symbols import snake_and_camel -from hexathon import add_0x +from hexathon import ( + add_0x, + strip_0x, + ) # local imports from cic_eth.runnable.daemons.filters.callback import CallbackFilter @@ -160,7 +163,7 @@ def test_faucet_gift_to_tx( assert transfer_data['token_address'] == foo_token -def test_callback_filter( +def test_callback_filter_filter( default_chain_spec, init_database, eth_rpc, @@ -213,6 +216,7 @@ def test_callback_filter( def call_back(self, transfer_type, result): self.results[transfer_type] = result + logg.debug('result {}'.format(result)) return self mock = CallbackMock() @@ -221,4 +225,4 @@ def test_callback_filter( fltr.filter(eth_rpc, mockblock, tx, init_database) assert mock.results.get('transfer') != None - assert mock.results['transfer']['destination_token'] == foo_token + assert mock.results['transfer']['destination_token'] == strip_0x(foo_token) diff --git a/apps/cic-eth/tests/filters/test_register_filter.py b/apps/cic-eth/tests/filters/test_register_filter.py index 34ec09c1..ce81b43a 100644 --- a/apps/cic-eth/tests/filters/test_register_filter.py +++ b/apps/cic-eth/tests/filters/test_register_filter.py @@ -17,6 +17,9 @@ from chainlib.eth.block import ( block_by_number, Block, ) +from chainlib.eth.address import ( + to_checksum_address, + ) from erc20_faucet import Faucet from hexathon import ( strip_0x, @@ -25,7 +28,6 @@ from hexathon import ( # local imports from cic_eth.runnable.daemons.filters.register import RegistrationFilter -from cic_eth.encode import tx_normalize from cic_eth.queue.query import get_account_tx_local logg = logging.getLogger() @@ -70,12 +72,13 @@ def test_register_filter( tx = Tx(tx_src, block=block, rcpt=rcpt) tx.apply_receipt(rcpt) - fltr = RegistrationFilter(default_chain_spec, add_0x(os.urandom(20).hex()), queue=None) + fltr = RegistrationFilter(default_chain_spec, to_checksum_address(os.urandom(20).hex()), queue=None) t = fltr.filter(eth_rpc, block, tx, db_session=init_database) assert t == None - fltr = RegistrationFilter(default_chain_spec, account_registry, queue=None) + fltr = RegistrationFilter(default_chain_spec, to_checksum_address(account_registry), queue=None) t = fltr.filter(eth_rpc, block, tx, db_session=init_database) + logg.debug('t {}'.format(t)) t.get_leaf() assert t.successful() @@ -89,4 +92,4 @@ def test_register_filter( gift_tx = unpack(tx_raw_signed_bytes, default_chain_spec) gift = Faucet.parse_give_to_request(gift_tx['data']) - assert gift[0] == agent_roles['ALICE'] + assert add_0x(gift[0]) == agent_roles['ALICE'] diff --git a/apps/cic-eth/tests/filters/test_transferauth_filter.py b/apps/cic-eth/tests/filters/test_transferauth_filter.py index 4d2266c1..ac84139a 100644 --- a/apps/cic-eth/tests/filters/test_transferauth_filter.py +++ b/apps/cic-eth/tests/filters/test_transferauth_filter.py @@ -19,6 +19,7 @@ from chainqueue.sql.query import get_account_tx # local imports from cic_eth.runnable.daemons.filters.transferauth import TransferAuthFilter +from cic_eth.encode import tx_normalize def test_filter_transferauth( @@ -66,7 +67,8 @@ def test_filter_transferauth( t.get_leaf() assert t.successful() - approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + #approve_txs = get_account_tx(default_chain_spec.asdict(), agent_roles['ALICE'], as_sender=True, session=init_database) + approve_txs = get_account_tx(default_chain_spec.asdict(), tx_normalize.wallet_address(agent_roles['ALICE']), as_sender=True, session=init_database) ks = list(approve_txs.keys()) assert len(ks) == 1 @@ -76,4 +78,4 @@ def test_filter_transferauth( c = ERC20(default_chain_spec) approve = c.parse_approve_request(approve_tx['data']) - assert approve[0] == agent_roles['BOB'] + assert approve[0] == strip_0x(agent_roles['BOB']) diff --git a/apps/cic-eth/tests/task/api/test_admin.py b/apps/cic-eth/tests/task/api/test_admin.py index 71ac61f4..363fb8d7 100644 --- a/apps/cic-eth/tests/task/api/test_admin.py +++ b/apps/cic-eth/tests/task/api/test_admin.py @@ -110,8 +110,8 @@ def test_tag_account( t = api.tag_account('bar', agent_roles['CAROL'], default_chain_spec) t.get() - assert AccountRole.get_address('foo', init_database) == agent_roles['ALICE'] - assert AccountRole.get_address('bar', init_database) == agent_roles['CAROL'] + assert AccountRole.get_address('foo', init_database) == tx_normalize.wallet_address(agent_roles['ALICE']) + assert AccountRole.get_address('bar', init_database) == tx_normalize.wallet_address(agent_roles['CAROL']) def test_tx( diff --git a/apps/cic-eth/tests/task/api/test_app.py b/apps/cic-eth/tests/task/api/test_app.py index 1e081584..acc6161d 100644 --- a/apps/cic-eth/tests/task/api/test_app.py +++ b/apps/cic-eth/tests/task/api/test_app.py @@ -10,6 +10,7 @@ from cic_eth_registry.erc20 import ERC20Token from chainlib.chain import ChainSpec from eth_accounts_index import AccountsIndex from chainlib.eth.tx import ( + receipt, transaction, ) from chainqueue.sql.state import ( @@ -29,6 +30,7 @@ def test_account_api( init_database, init_eth_rpc, account_registry, + cic_registry, custodial_roles, celery_session_worker, ): @@ -49,6 +51,7 @@ def test_account_api_register( eth_rpc, celery_session_worker, ): + api = Api(str(default_chain_spec), callback_param='accounts', callback_task='cic_eth.callbacks.noop.noop', queue=None) t = api.create_account('') register_tx_hash = t.get_leaf() @@ -69,12 +72,18 @@ def test_account_api_register( r = t.get_leaf() assert t.successful() + o = receipt(register_tx_hash) + r = eth_rpc.do(o) + assert r['status'] == 1 + o = transaction(register_tx_hash) tx_src = eth_rpc.do(o) c = AccountsIndex(default_chain_spec) address = c.parse_add_request(tx_src['data']) + logg.debug('address {} '.format(address)) o = c.have(account_registry, address[0], sender_address=custodial_roles['CONTRACT_DEPLOYER']) + logg.debug('o {}'.format(o)) r = eth_rpc.do(o) assert c.parse_have(r) diff --git a/apps/cic-eth/tests/task/api/test_balance.py b/apps/cic-eth/tests/task/api/test_balance.py index 650c2b10..0a72a62c 100644 --- a/apps/cic-eth/tests/task/api/test_balance.py +++ b/apps/cic-eth/tests/task/api/test_balance.py @@ -3,18 +3,22 @@ import os import logging # external imports +import pytest from chainlib.eth.address import to_checksum_address +from hexathon import add_0x # local imports from cic_eth.api.api_task import Api logg = logging.getLogger() + def test_balance_simple_api( default_chain_spec, init_database, cic_registry, foo_token, + register_lookups, register_tokens, api, celery_session_worker, @@ -22,7 +26,7 @@ def test_balance_simple_api( chain_str = str(default_chain_spec) - a = to_checksum_address('0x' + os.urandom(20).hex()) + a = add_0x(to_checksum_address(os.urandom(20).hex())) t = api.balance(a, 'FOO', include_pending=False) r = t.get_leaf() assert t.successful() @@ -36,6 +40,7 @@ def test_balance_complex_api( init_database, cic_registry, foo_token, + register_lookups, register_tokens, api, celery_session_worker, @@ -43,7 +48,7 @@ def test_balance_complex_api( chain_str = str(default_chain_spec) - a = to_checksum_address('0x' + os.urandom(20).hex()) + a = add_0x(to_checksum_address(os.urandom(20).hex())) t = api.balance(a, 'FOO', include_pending=True) r = t.get_leaf() assert t.successful() diff --git a/apps/cic-eth/tests/task/api/test_list.py b/apps/cic-eth/tests/task/api/test_list.py index 22c07ae8..7f0bdf0f 100644 --- a/apps/cic-eth/tests/task/api/test_list.py +++ b/apps/cic-eth/tests/task/api/test_list.py @@ -6,6 +6,7 @@ import pytest from chainlib.eth.nonce import RPCNonceOracle from eth_erc20 import ERC20 from chainlib.eth.tx import receipt +from hexathon import strip_0x # local imports from cic_eth.api.api_task import Api @@ -23,7 +24,6 @@ from cic_eth.pytest.mock.filter import ( logg = logging.getLogger() -@pytest.mark.xfail() def test_list_tx( default_chain_spec, init_database, @@ -34,8 +34,10 @@ def test_list_tx( agent_roles, foo_token, register_tokens, + register_lookups, init_eth_tester, celery_session_worker, + init_celery_tasks, ): tx_hashes = [] @@ -63,13 +65,16 @@ def test_list_tx( o = receipt(tx_hash_hex) r = eth_rpc.do(o) assert r['status'] == 1 + a = r['block_number'] - block_filter.add(a.to_bytes(4, 'big')) + ab = a.to_bytes(4, 'big') + block_filter.add(ab) - a = r['block_number'] + r['transaction_index'] - tx_filter.add(a.to_bytes(4, 'big')) + bb = r['transaction_index'].to_bytes(4, 'big') + cb = ab + bb + tx_filter.add(cb) - tx_hashes.append(tx_hash_hex) + tx_hashes.append(strip_0x(tx_hash_hex)) # external tx two Nonce.next(agent_roles['ALICE'], 'foo', session=init_database) @@ -83,26 +88,29 @@ def test_list_tx( o = receipt(tx_hash_hex) r = eth_rpc.do(o) assert r['status'] == 1 + a = r['block_number'] - block_filter.add(a.to_bytes(4, 'big')) + ab = a.to_bytes(4, 'big') + block_filter.add(ab) - a = r['block_number'] + r['transaction_index'] - tx_filter.add(a.to_bytes(4, 'big')) + bb = r['transaction_index'].to_bytes(4, 'big') + cb = ab + bb + tx_filter.add(cb) - tx_hashes.append(tx_hash_hex) + tx_hashes.append(strip_0x(tx_hash_hex)) init_eth_tester.mine_blocks(28) # custodial tx 1 api = Api(str(default_chain_spec), queue=None) - t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') #, 'blinky') + t = api.transfer(agent_roles['ALICE'], agent_roles['CAROL'], 64, 'FOO') r = t.get_leaf() assert t.successful() tx_hashes.append(r) # custodial tx 2 api = Api(str(default_chain_spec), queue=None) - t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') #, 'blinky') + t = api.transfer(agent_roles['ALICE'], agent_roles['DAVE'], 16, 'FOO') r = t.get_leaf() assert t.successful() tx_hashes.append(r) @@ -117,7 +125,8 @@ def test_list_tx( assert len(r) == 3 logg.debug('rrrr {}'.format(r)) + logg.debug('testing against hashes {}'.format(tx_hashes)) for tx in r: logg.debug('have tx {}'.format(tx)) - tx_hashes.remove(tx['hash']) + tx_hashes.remove(strip_0x(tx['hash'])) assert len(tx_hashes) == 1 diff --git a/apps/cic-eth/tests/task/test_ext_tx.py b/apps/cic-eth/tests/task/test_ext_tx.py index 2740bf20..dba1562a 100644 --- a/apps/cic-eth/tests/task/test_ext_tx.py +++ b/apps/cic-eth/tests/task/test_ext_tx.py @@ -10,6 +10,7 @@ from chainlib.eth.tx import ( ) from eth_erc20 import ERC20 from chainlib.eth.nonce import RPCNonceOracle +from hexathon import add_0x # local imports from cic_eth.db.models.nonce import ( @@ -91,5 +92,5 @@ def test_filter_process( assert len(r) == 2 for tx_hash in r.keys(): - tx_hashes.remove(tx_hash) + tx_hashes.remove(add_0x(tx_hash)) assert len(tx_hashes) == 0 diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index f10c88e9..401bdc8b 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,6 +1,6 @@ crypto-dev-signer>=0.4.15a7,<=0.4.15 chainqueue>=0.0.5a1,<0.1.0 -cic-eth-registry>=0.6.1a2,<0.7.0 +cic-eth-registry>=0.6.1a3,<0.7.0 redis==3.5.3 hexathon~=0.0.1a8 pycryptodome==3.10.1 diff --git a/apps/cic-ussd/cic_ussd/account/balance.py b/apps/cic-ussd/cic_ussd/account/balance.py index e8563a9f..838f8bbb 100644 --- a/apps/cic-ussd/cic_ussd/account/balance.py +++ b/apps/cic-ussd/cic_ussd/account/balance.py @@ -101,7 +101,7 @@ def get_cached_available_balance(blockchain_address: str) -> float: :return: Operational balance of an account. :rtype: float """ - identifier = bytes.fromhex(blockchain_address[2:]) + identifier = bytes.fromhex(blockchain_address) key = cache_data_key(identifier, salt=':cic.balances') cached_balances = get_cached_data(key=key) if cached_balances: diff --git a/apps/cic-ussd/cic_ussd/account/metadata.py b/apps/cic-ussd/cic_ussd/account/metadata.py index d2f8581b..e32b17de 100644 --- a/apps/cic-ussd/cic_ussd/account/metadata.py +++ b/apps/cic-ussd/cic_ussd/account/metadata.py @@ -4,7 +4,6 @@ import logging from typing import Optional # external imports -from chainlib.hash import strip_0x from cic_types.models.person import Person # local imports @@ -20,7 +19,7 @@ def get_cached_preferred_language(blockchain_address: str) -> Optional[str]: :return: Account's set preferred language | Fallback preferred language. :rtype: str """ - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) preferences_metadata_handler = PreferencesMetadata(identifier) cached_preferences_metadata = preferences_metadata_handler.get_cached_metadata() if cached_preferences_metadata: diff --git a/apps/cic-ussd/cic_ussd/account/statement.py b/apps/cic-ussd/cic_ussd/account/statement.py index 94cde296..584f526c 100644 --- a/apps/cic-ussd/cic_ussd/account/statement.py +++ b/apps/cic-ussd/cic_ussd/account/statement.py @@ -86,7 +86,7 @@ def query_statement(blockchain_address: str, limit: int = 9): :param limit: Number of transactions to be returned. :type limit: int """ - logg.debug(f'retrieving balance for address: {blockchain_address}') + logg.debug(f'retrieving statement for address: {blockchain_address}') chain_str = Chain.spec.__str__() cic_eth_api = Api( chain_str=chain_str, diff --git a/apps/cic-ussd/cic_ussd/db/models/account.py b/apps/cic-ussd/cic_ussd/db/models/account.py index c9d36b6d..2eb9291b 100644 --- a/apps/cic-ussd/cic_ussd/db/models/account.py +++ b/apps/cic-ussd/cic_ussd/db/models/account.py @@ -2,7 +2,6 @@ import json # external imports -from chainlib.hash import strip_0x from cic_eth.api import Api # local imports @@ -101,7 +100,7 @@ class Account(SessionBase): session.add(self) session.flush() SessionBase.release_session(session=session) - return f'Pin reset successful.' + return 'Pin reset successful.' def standard_metadata_id(self) -> str: """This function creates an account's standard metadata identification information that contains an account owner's @@ -109,7 +108,7 @@ class Account(SessionBase): :return: Standard metadata identification information | e164 formatted phone number. :rtype: str """ - identifier = bytes.fromhex(strip_0x(self.blockchain_address)) + identifier = bytes.fromhex(self.blockchain_address) key = cache_data_key(identifier, ':cic.person') account_metadata = get_cached_data(key) if not account_metadata: diff --git a/apps/cic-ussd/cic_ussd/processor/menu.py b/apps/cic-ussd/cic_ussd/processor/menu.py index 042f8680..568409c8 100644 --- a/apps/cic-ussd/cic_ussd/processor/menu.py +++ b/apps/cic-ussd/cic_ussd/processor/menu.py @@ -37,7 +37,7 @@ class MenuProcessor: def __init__(self, account: Account, display_key: str, menu_name: str, session: Session, ussd_session: dict): self.account = account self.display_key = display_key - self.identifier = bytes.fromhex(self.account.blockchain_address[2:]) + self.identifier = bytes.fromhex(self.account.blockchain_address) self.menu_name = menu_name self.session = session self.ussd_session = ussd_session diff --git a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py index f3a80caf..30bce11f 100644 --- a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py +++ b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py @@ -5,7 +5,6 @@ from datetime import timedelta # third-party imports import celery -from chainlib.hash import strip_0x # local imports from cic_ussd.account.balance import get_balances, calculate_available_balance @@ -55,6 +54,7 @@ def account_creation_callback(self, result: str, url: str, status_code: int): session.add(account) session.commit() session.close() + logg.debug(f'recorded account with identifier: {result}') queue = self.request.delivery_info.get('routing_key') s_phone_pointer = celery.signature( @@ -86,7 +86,7 @@ def balances_callback(result: list, param: str, status_code: int): raise ValueError(f'Unexpected status code: {status_code}.') balances = result[0] - identifier = bytes.fromhex(strip_0x(param)) + identifier = bytes.fromhex(param) key = cache_data_key(identifier, ':cic.balances') cache_data(key, json.dumps(balances)) @@ -113,8 +113,10 @@ def statement_callback(self, result, param: str, status_code: int): for transaction in statement_transactions: recipient_transaction, sender_transaction = transaction_actors(transaction) if recipient_transaction.get('blockchain_address') == param: + recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address') generate(param, queue, recipient_transaction) if sender_transaction.get('blockchain_address') == param: + sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address') generate(param, queue, sender_transaction) diff --git a/apps/cic-ussd/cic_ussd/tasks/metadata.py b/apps/cic-ussd/cic_ussd/tasks/metadata.py index ff16970d..bd3b4d1b 100644 --- a/apps/cic-ussd/cic_ussd/tasks/metadata.py +++ b/apps/cic-ussd/cic_ussd/tasks/metadata.py @@ -3,7 +3,6 @@ import logging # third-party imports import celery -from hexathon import strip_0x # local imports from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata @@ -21,7 +20,7 @@ def query_person_metadata(blockchain_address: str): :return: :rtype: """ - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.query() @@ -36,14 +35,14 @@ def create_person_metadata(blockchain_address: str, data: dict): :return: :rtype: """ - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.create(data=data) @celery_app.task def edit_person_metadata(blockchain_address: str, data: dict): - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.edit(data=data) @@ -51,21 +50,21 @@ def edit_person_metadata(blockchain_address: str, data: dict): @celery_app.task(bind=True, base=CriticalMetadataTask) def add_phone_pointer(self, blockchain_address: str, phone_number: str): identifier = phone_number.encode('utf-8') - stripped_address = strip_0x(blockchain_address) + stripped_address = blockchain_address phone_metadata_client = PhonePointerMetadata(identifier=identifier) phone_metadata_client.create(data=stripped_address) @celery_app.task() def add_custom_metadata(blockchain_address: str, data: dict): - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) custom_metadata_client = CustomMetadata(identifier=identifier) custom_metadata_client.create(data=data) @celery_app.task() def add_preferences_metadata(blockchain_address: str, data: dict): - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) preferences_metadata_client = PreferencesMetadata(identifier=identifier) preferences_metadata_client.create(data=data) @@ -76,7 +75,7 @@ def query_preferences_metadata(blockchain_address: str): :param blockchain_address: Blockchain address of an account. :type blockchain_address: str | Ox-hex """ - identifier = bytes.fromhex(strip_0x(blockchain_address)) + identifier = bytes.fromhex(blockchain_address) logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.') person_metadata_client = PreferencesMetadata(identifier=identifier) return person_metadata_client.query() diff --git a/apps/cic-ussd/cic_ussd/tasks/processor.py b/apps/cic-ussd/cic_ussd/tasks/processor.py index 8a188f54..04440ee4 100644 --- a/apps/cic-ussd/cic_ussd/tasks/processor.py +++ b/apps/cic-ussd/cic_ussd/tasks/processor.py @@ -5,7 +5,6 @@ import logging # third-party imports import celery import i18n -from chainlib.hash import strip_0x # local imports from cic_ussd.account.metadata import get_cached_preferred_language @@ -24,17 +23,13 @@ logg = logging.getLogger(__file__) def generate_statement(self, querying_party: str, transaction: dict): """""" queue = self.request.delivery_info.get('routing_key') - - s_preferences = celery.signature( - 'cic_ussd.tasks.metadata.query_preferences_metadata', [querying_party], queue=queue - ) s_parse_transaction = celery.signature( 'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue ) s_cache_statement = celery.signature( 'cic_ussd.tasks.processor.cache_statement', [querying_party], queue=queue ) - celery.chain(s_preferences, s_parse_transaction, s_cache_statement).apply_async() + celery.chain(s_parse_transaction, s_cache_statement).apply_async() @celery_app.task @@ -53,7 +48,7 @@ def cache_statement(parsed_transaction: dict, querying_party: str): statement_transactions = json.loads(cached_statement) statement_transactions.append(parsed_transaction) data = json.dumps(statement_transactions) - identifier = bytes.fromhex(strip_0x(querying_party)) + identifier = bytes.fromhex(querying_party) key = cache_data_key(identifier, ':cic.statement') cache_data(key, data) diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py index 0e05c383..82468fec 100644 --- a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py +++ b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py @@ -28,7 +28,6 @@ def test_filter_statement_transactions(transactions_list): def test_generate(activated_account, cache_default_token_data, - cache_statement, cache_preferences, celery_session_worker, init_cache, @@ -37,22 +36,22 @@ def test_generate(activated_account, preferences, preferences_metadata_url, transactions_list): - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri('GET', preferences_metadata_url, status_code=200, reason='OK', json=preferences) - statement_transactions = filter_statement_transactions(transactions_list) - for transaction in statement_transactions: - querying_party = activated_account.blockchain_address - recipient_transaction, sender_transaction = transaction_actors(transaction) - if recipient_transaction.get('blockchain_address') == querying_party: - generate(querying_party, None, recipient_transaction) - if sender_transaction.get('blockchain_address') == querying_party: - generate(querying_party, None, sender_transaction) - time.sleep(2) - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - key = cache_data_key(identifier, ':cic.statement') - statement = get_cached_data(key) - statement = json.loads(statement) - assert len(statement) == 1 + statement_transactions = filter_statement_transactions(transactions_list) + for transaction in statement_transactions: + querying_party = activated_account.blockchain_address + recipient_transaction, sender_transaction = transaction_actors(transaction) + if recipient_transaction.get('blockchain_address') == querying_party: + recipient_transaction['alt_blockchain_address'] = sender_transaction.get('blockchain_address') + generate(querying_party, None, recipient_transaction) + if sender_transaction.get('blockchain_address') == querying_party: + sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address') + generate(querying_party, None, sender_transaction) + time.sleep(2) + identifier = bytes.fromhex(activated_account.blockchain_address) + key = cache_data_key(identifier, ':cic.statement') + statement = get_cached_data(key) + statement = json.loads(statement) + assert len(statement) == 1 def test_get_cached_statement(activated_account, cache_statement, statement): diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py index 422c4829..a343647b 100644 --- a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py @@ -57,7 +57,7 @@ def test_menu_processor(activated_account, available_balance=available_balance, token_symbol=token_symbol) - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + identifier = bytes.fromhex(activated_account.blockchain_address) key = cache_data_key(identifier, ':cic.adjusted_balance') adjusted_balance = 45931650.64654012 cache_data(key, json.dumps(adjusted_balance)) @@ -108,7 +108,7 @@ def test_menu_processor(activated_account, display_key = 'ussd.kenya.display_user_metadata' ussd_menu = UssdMenu.find_by_name('display_user_metadata') name = ussd_menu.get('name') - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + identifier = bytes.fromhex(activated_account.blockchain_address) person_metadata = PersonMetadata(identifier) cached_person_metadata = person_metadata.get_cached_metadata() resp = response(activated_account, display_key, name, init_database, generic_ussd_session) diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py index b08472b0..fa017add 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py @@ -114,6 +114,7 @@ def test_statement_callback(activated_account, mocker, transactions_list): s_statement_callback.apply_async().get() statement_transactions = filter_statement_transactions(transactions_list) recipient_transaction, sender_transaction = transaction_actors(statement_transactions[0]) + sender_transaction['alt_blockchain_address'] = recipient_transaction.get('blockchain_address') mock_statement_generate.assert_called_with( (activated_account.blockchain_address, sender_transaction), {}, queue='cic-ussd') diff --git a/apps/cic-ussd/tests/fixtures/account.py b/apps/cic-ussd/tests/fixtures/account.py index e2703877..48034a03 100644 --- a/apps/cic-ussd/tests/fixtures/account.py +++ b/apps/cic-ussd/tests/fixtures/account.py @@ -54,7 +54,7 @@ def cache_account_creation_data(init_cache, account_creation_data): @pytest.fixture(scope='function') def cache_balances(activated_account, balances, init_cache): - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + identifier = bytes.fromhex(activated_account.blockchain_address) balances = json.dumps(balances[0]) key = cache_data_key(identifier, ':cic.balances') cache_data(key, balances) @@ -70,7 +70,7 @@ def cache_default_token_data(default_token_data, init_cache, load_chain_spec): @pytest.fixture(scope='function') def cache_person_metadata(activated_account, init_cache, person_metadata): - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + identifier = bytes.fromhex(activated_account.blockchain_address) person = json.dumps(person_metadata) key = cache_data_key(identifier, ':cic.person') cache_data(key, person) @@ -78,7 +78,7 @@ def cache_person_metadata(activated_account, init_cache, person_metadata): @pytest.fixture(scope='function') def cache_preferences(activated_account, init_cache, preferences): - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + identifier = bytes.fromhex(activated_account.blockchain_address) preferences = json.dumps(preferences) key = cache_data_key(identifier, ':cic.preferences') cache_data(key, preferences) @@ -86,10 +86,10 @@ def cache_preferences(activated_account, init_cache, preferences): @pytest.fixture(scope='function') def cache_statement(activated_account, init_cache, statement): - identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - preferences = json.dumps(statement) + identifier = bytes.fromhex(activated_account.blockchain_address) + statement = json.dumps(statement) key = cache_data_key(identifier, ':cic.statement') - cache_data(key, preferences) + cache_data(key, statement) @pytest.fixture(scope='function') diff --git a/apps/cic-ussd/tests/fixtures/transaction.py b/apps/cic-ussd/tests/fixtures/transaction.py index 78953da4..dbdfe901 100644 --- a/apps/cic-ussd/tests/fixtures/transaction.py +++ b/apps/cic-ussd/tests/fixtures/transaction.py @@ -7,6 +7,7 @@ import pytest # local import from cic_ussd.account.balance import get_cached_available_balance + # tests imports @@ -103,8 +104,8 @@ def transactions_list(activated_account, valid_recipient): 'destination_token': '0x0000000000000000000000000000000000000000', 'block_number': 80, 'tx_index': 0, - 'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', - 'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A', + 'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A', 'from_value': 0, 'to_value': 0, 'date_created': '2021-07-14T14:13:46.036198', @@ -122,8 +123,8 @@ def transactions_list(activated_account, valid_recipient): 'destination_token': '0x0000000000000000000000000000000000000000', 'block_number': 78, 'tx_index': 0, - 'sender': '0xb41BfEE260693A473254D62b81aE1ADCC9E51AFb', - 'recipient': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'sender': 'b41BfEE260693A473254D62b81aE1ADCC9E51AFb', + 'recipient': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', 'from_value': 1800000000000000, 'to_value': 1800000000000000, 'date_created': '2021-07-14T14:13:35.839638', @@ -142,8 +143,8 @@ def transactions_list(activated_account, valid_recipient): 'destination_token': '0x0000000000000000000000000000000000000000', 'block_number': 79, 'tx_index': 0, - 'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', - 'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A', + 'sender': '367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'recipient': '103d1ed6e370dBa6267045c70d4999384c18a04A', 'from_value': 0, 'to_value': 0, 'date_created': '2021-07-14T14:13:35.638355', @@ -152,4 +153,3 @@ def transactions_list(activated_account, valid_recipient): 'timestamp': 1626272015, 'hash': '0x32ca3dd3bef06463b452f4d32f5f563d083cb4759219eed90f3d2a9c1791c5fc'} ] - diff --git a/apps/cic-ussd/tests/helpers/accounts.py b/apps/cic-ussd/tests/helpers/accounts.py index 93f175a7..8c432719 100644 --- a/apps/cic-ussd/tests/helpers/accounts.py +++ b/apps/cic-ussd/tests/helpers/accounts.py @@ -4,7 +4,6 @@ import random import uuid # external imports -from chainlib.eth.address import to_checksum_address from faker import Faker from faker_e164.providers import E164Provider @@ -21,7 +20,7 @@ def phone_number() -> str: def blockchain_address() -> str: - return to_checksum_address('0x' + os.urandom(20).hex()) + return os.urandom(20).hex().lower() def session_id() -> str: From 9ee42b908d1040166f6c632e6ee3c6438d281600 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Thu, 7 Oct 2021 21:05:58 +0000 Subject: [PATCH 13/13] feat: Integrate okota --- apps/contract-migration/requirements.txt | 11 ++++++---- apps/contract-migration/reset.sh | 26 ++++++++++++++---------- apps/contract-migration/seed_cic_eth.sh | 8 ++++---- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 0e62e29a..8f446a6c 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,12 +1,15 @@ cic-eth[tools]==0.12.4a8 -chainlib-eth>=0.0.9a14,<0.1.0 +chainlib-eth>=0.0.9rc1,<0.1.0 +chainlib==0.0.9rc1,<0.1.0 eth-erc20>=0.1.2a3,<0.2.0 erc20-demurrage-token>=0.0.5a2,<0.1.0 -eth-accounts-index>=0.1.2a2,<0.2.0 -eth-address-index>=0.2.3a4,<0.3.0 +#eth-accounts-index>=0.1.2a2,<0.2.0 +eth-address-index>=0.2.4a1,<0.3.0 cic-eth-registry>=0.6.1a2,<0.7.0 erc20-transfer-authorization>=0.3.5a2,<0.4.0 erc20-faucet>=0.3.2a2,<0.4.0 sarafu-faucet>=0.0.7a2,<0.1.0 confini>=0.4.2rc3,<1.0.0 -crypto-dev-signer>=0.4.15a7,<=0.4.15 +crypto-dev-signer>=0.4.15rc2,<=0.4.15 +eth-token-index>=0.2.4a1,<=0.3.0 +okota>=0.2.4a5,<0.3.0 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index f4d45f7f..fc1bfa20 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -65,22 +65,25 @@ fi echo "giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT" giftable-token-gift $fee_price_arg -p $RPC_PROVIDER -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT ->&2 echo "deploy account index contract" -DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w` ->&2 echo "add deployer address as account index writer" -eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER - ->&2 echo "deploy contract registry contract" -CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w` -eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS -eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS - # Deploy address declarator registry >&2 echo "deploy address declarator contract" declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000 DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv $declarator_description` + +>&2 echo "deploy contract registry contract" +#CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $RPC_PROVIDER -vv -s -u -w` +CIC_REGISTRY_ADDRESS=`okota-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $WALLET_KEY_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry --address-declarator $DEV_DECLARATOR_ADDRESS -p $RPC_PROVIDER -vv -s -u -w` +eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS + eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS +>&2 echo "deploy account index contract" +#DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w` +DEV_ACCOUNT_INDEX_ADDRESS=`okota-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -y $WALLET_KEY_FILE -vv -s -u -w --address-declarator $DEV_DECLARATOR_ADDRESS --token-address $DEV_RESERVE_ADDRESS` +#>&2 echo "add deployer address as account index writer" +#eth-accounts-index-writer $fee_price_arg -s -u -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER +eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS + # Deploy transfer authorization contact >&2 echo "deploy transfer auth contract" DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv` @@ -88,7 +91,8 @@ eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_RE # Deploy token index contract >&2 echo "deploy token index contract" -DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv` +#DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv` +DEV_TOKEN_INDEX_ADDRESS=`okota-token-index-deploy -s -u $fee_price_arg -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -w -vv --address-declarator $DEV_DECLARATOR_ADDRESS` eth-contract-registry-set $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS >&2 echo "add reserve token to token index" eth-token-index-add $fee_price_arg -s -u -w -y $WALLET_KEY_FILE -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index 4d0fcd73..31069adc 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -46,7 +46,7 @@ cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFE DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >&2 echo "add acccounts index writer account as writer on contract" -eth-accounts-index-writer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $account_index_address -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +#eth-accounts-index-writer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER -e $account_index_address -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER # Transfer gas to custodial gas provider adddress _CONFINI_DIR=$CONFINI_DIR @@ -78,9 +78,9 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >&2 erc20-transfer -s -u -y $WALLET_KEY_FILE -i $CHAIN_SPEC -p $RPC_PROVIDER --fee-limit 100000 -e $reserve_address -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${DEV_TOKEN_AMOUNT:0:-1} # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration -cic-eth-ctl -i $CHAIN_SPEC unlock INIT -cic-eth-ctl -i $CHAIN_SPEC unlock SEND -cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE +cic-eth-ctl -vv -i $CHAIN_SPEC unlock INIT +cic-eth-ctl -vv -i $CHAIN_SPEC unlock SEND +cic-eth-ctl -vv -i $CHAIN_SPEC unlock QUEUE #confini-dump --schema-module chainlib.eth.data.config --schema-module cic_eth.data.config --schema-dir ./config