diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index 84c398b5..d8737146 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -1,5 +1,5 @@ chainqueue>=0.0.6a1,<0.1.0 -chainsyncer[sql]>=0.0.6a3,<0.1.0 +chainsyncer[sql]>=0.0.7a3,<0.1.0 alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 redis==3.5.3 diff --git a/apps/cic-meta/docker/Dockerfile b/apps/cic-meta/docker/Dockerfile index 3458cc7b..a4a2d16f 100644 --- a/apps/cic-meta/docker/Dockerfile +++ b/apps/cic-meta/docker/Dockerfile @@ -1,5 +1,4 @@ FROM node:15.3.0-alpine3.10 -#FROM node:lts-alpine3.14 WORKDIR /root @@ -9,7 +8,7 @@ RUN apk add --no-cache postgresql bash COPY package.json package-lock.json ./ RUN --mount=type=cache,mode=0755,target=/root/.npm \ npm set cache /root/.npm && \ - npm cache verify && \ + npm cache verify && \ npm ci --verbose COPY webpack.config.js . diff --git a/apps/cic-notify/docker/Dockerfile b/apps/cic-notify/docker/Dockerfile index ef7167fa..b217d798 100644 --- a/apps/cic-notify/docker/Dockerfile +++ b/apps/cic-notify/docker/Dockerfile @@ -3,6 +3,8 @@ ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 #RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a62 +RUN apt-get install libffi-dev -y + ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433 ARG EXTRA_PIP_ARGS="" @@ -19,8 +21,8 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ COPY . . RUN python setup.py install -COPY docker/*.sh . -RUN chmod +x *.sh +COPY docker/*.sh ./ +RUN chmod +x /root/*.sh # ini files in config directory defines the configurable parameters for the application # they can all be overridden by environment variables diff --git a/apps/cic-ussd/cic_ussd/account/balance.py b/apps/cic-ussd/cic_ussd/account/balance.py index 838f8bbb..ad9e7ba3 100644 --- a/apps/cic-ussd/cic_ussd/account/balance.py +++ b/apps/cic-ussd/cic_ussd/account/balance.py @@ -7,6 +7,7 @@ from typing import Optional # third-party imports from cic_eth.api import Api from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.transaction import from_wei @@ -102,7 +103,7 @@ def get_cached_available_balance(blockchain_address: str) -> float: :rtype: float """ identifier = bytes.fromhex(blockchain_address) - key = cache_data_key(identifier, salt=':cic.balances') + key = cache_data_key(identifier, salt=MetadataPointer.BALANCES) cached_balances = get_cached_data(key=key) if cached_balances: return calculate_available_balance(json.loads(cached_balances)) @@ -117,5 +118,5 @@ def get_cached_adjusted_balance(identifier: bytes): :return: :rtype: """ - key = cache_data_key(identifier, ':cic.adjusted_balance') + key = cache_data_key(identifier, MetadataPointer.BALANCES_ADJUSTED) return get_cached_data(key) diff --git a/apps/cic-ussd/cic_ussd/account/statement.py b/apps/cic-ussd/cic_ussd/account/statement.py index 584f526c..3dff2693 100644 --- a/apps/cic-ussd/cic_ussd/account/statement.py +++ b/apps/cic-ussd/cic_ussd/account/statement.py @@ -7,6 +7,7 @@ from typing import Optional import celery from chainlib.hash import strip_0x from cic_eth.api import Api +from cic_types.condiments import MetadataPointer # local import from cic_ussd.account.chain import Chain @@ -53,7 +54,7 @@ def get_cached_statement(blockchain_address: str) -> bytes: :rtype: str """ identifier = bytes.fromhex(strip_0x(blockchain_address)) - key = cache_data_key(identifier=identifier, salt=':cic.statement') + key = cache_data_key(identifier=identifier, salt=MetadataPointer.STATEMENT) return get_cached_data(key=key) diff --git a/apps/cic-ussd/cic_ussd/account/tokens.py b/apps/cic-ussd/cic_ussd/account/tokens.py index 40685764..f49bd835 100644 --- a/apps/cic-ussd/cic_ussd/account/tokens.py +++ b/apps/cic-ussd/cic_ussd/account/tokens.py @@ -5,6 +5,7 @@ from typing import Dict, Optional # external imports from cic_eth.api import Api +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.chain import Chain @@ -23,7 +24,7 @@ def get_cached_default_token(chain_str: str) -> Optional[str]: :rtype: """ logg.debug(f'Retrieving default token from cache for chain: {chain_str}') - key = cache_data_key(identifier=chain_str.encode('utf-8'), salt=':cic.default_token_data') + key = cache_data_key(identifier=chain_str.encode('utf-8'), salt=MetadataPointer.TOKEN_DEFAULT) return get_cached_data(key=key) diff --git a/apps/cic-ussd/cic_ussd/cache.py b/apps/cic-ussd/cic_ussd/cache.py index 5c45e069..70689826 100644 --- a/apps/cic-ussd/cic_ussd/cache.py +++ b/apps/cic-ussd/cic_ussd/cache.py @@ -2,7 +2,8 @@ import hashlib import logging -# third-party imports +# external imports +from cic_types.condiments import MetadataPointer from redis import Redis logg = logging.getLogger() @@ -38,7 +39,7 @@ def get_cached_data(key: str): return cache.get(name=key) -def cache_data_key(identifier: bytes, salt: str): +def cache_data_key(identifier: bytes, salt: MetadataPointer): """ :param identifier: :type identifier: @@ -49,5 +50,5 @@ def cache_data_key(identifier: bytes, salt: str): """ hash_object = hashlib.new("sha256") hash_object.update(identifier) - hash_object.update(salt.encode(encoding="utf-8")) + hash_object.update(salt.value.encode(encoding="utf-8")) return hash_object.digest().hex() diff --git a/apps/cic-ussd/cic_ussd/db/models/account.py b/apps/cic-ussd/cic_ussd/db/models/account.py index 2eb9291b..fe0fe21c 100644 --- a/apps/cic-ussd/cic_ussd/db/models/account.py +++ b/apps/cic-ussd/cic_ussd/db/models/account.py @@ -3,6 +3,7 @@ import json # external imports from cic_eth.api import Api +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.metadata import get_cached_preferred_language, parse_account_metadata @@ -109,7 +110,7 @@ class Account(SessionBase): :rtype: str """ identifier = bytes.fromhex(self.blockchain_address) - key = cache_data_key(identifier, ':cic.person') + key = cache_data_key(identifier, MetadataPointer.PERSON) account_metadata = get_cached_data(key) if not account_metadata: return self.phone_number diff --git a/apps/cic-ussd/cic_ussd/processor/menu.py b/apps/cic-ussd/cic_ussd/processor/menu.py index 568409c8..24006a6c 100644 --- a/apps/cic-ussd/cic_ussd/processor/menu.py +++ b/apps/cic-ussd/cic_ussd/processor/menu.py @@ -5,6 +5,7 @@ from datetime import datetime, timedelta # external imports import i18n.config +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.balance import (calculate_available_balance, @@ -163,7 +164,7 @@ class MenuProcessor: token_symbol = get_default_token_symbol() blockchain_address = self.account.blockchain_address balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0] - key = cache_data_key(self.identifier, ':cic.balances') + key = cache_data_key(self.identifier, MetadataPointer.BALANCES) cache_data(key, json.dumps(balances)) available_balance = calculate_available_balance(balances) now = datetime.now() @@ -173,7 +174,7 @@ class MenuProcessor: else: timestamp = int((now - timedelta(30)).timestamp()) adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol) - key = cache_data_key(self.identifier, ':cic.adjusted_balance') + key = cache_data_key(self.identifier, MetadataPointer.BALANCES_ADJUSTED) cache_data(key, json.dumps(adjusted_balance)) query_statement(blockchain_address) diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py index f54f3d2f..8e3f4268 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py @@ -12,6 +12,7 @@ import i18n import redis from chainlib.chain import ChainSpec from confini import Config +from cic_types.condiments import MetadataPointer from cic_types.ext.metadata import Metadata from cic_types.ext.metadata.signer import Signer @@ -109,7 +110,7 @@ default_token_data = query_default_token(chain_str) # cache default token for re-usability if default_token_data: - cache_key = cache_data_key(chain_str.encode('utf-8'), ':cic.default_token_data') + cache_key = cache_data_key(chain_str.encode('utf-8'), MetadataPointer.TOKEN_DEFAULT) cache_data(key=cache_key, data=json.dumps(default_token_data)) else: raise InitializationError(f'Default token data for: {chain_str} not found.') diff --git a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py index 30bce11f..9a13bf74 100644 --- a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py +++ b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py @@ -3,8 +3,10 @@ import json import logging from datetime import timedelta -# third-party imports +# external imports import celery +from cic_types.condiments import MetadataPointer + # local imports from cic_ussd.account.balance import get_balances, calculate_available_balance @@ -87,7 +89,7 @@ def balances_callback(result: list, param: str, status_code: int): balances = result[0] identifier = bytes.fromhex(param) - key = cache_data_key(identifier, ':cic.balances') + key = cache_data_key(identifier, MetadataPointer.BALANCES) cache_data(key, json.dumps(balances)) diff --git a/apps/cic-ussd/cic_ussd/tasks/processor.py b/apps/cic-ussd/cic_ussd/tasks/processor.py index 04440ee4..599938ab 100644 --- a/apps/cic-ussd/cic_ussd/tasks/processor.py +++ b/apps/cic-ussd/cic_ussd/tasks/processor.py @@ -2,9 +2,10 @@ import json import logging -# third-party imports +# external imports import celery import i18n +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.metadata import get_cached_preferred_language @@ -49,7 +50,7 @@ def cache_statement(parsed_transaction: dict, querying_party: str): statement_transactions.append(parsed_transaction) data = json.dumps(statement_transactions) identifier = bytes.fromhex(querying_party) - key = cache_data_key(identifier, ':cic.statement') + key = cache_data_key(identifier, MetadataPointer.STATEMENT) cache_data(key, data) diff --git a/apps/cic-ussd/config/test/chain.ini b/apps/cic-ussd/config/test/chain.ini new file mode 100644 index 00000000..6f64b7f1 --- /dev/null +++ b/apps/cic-ussd/config/test/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = 'evm:foo:1:bar' diff --git a/apps/cic-ussd/config/test/cic.ini b/apps/cic-ussd/config/test/cic.ini index 1feb345d..8258b282 100644 --- a/apps/cic-ussd/config/test/cic.ini +++ b/apps/cic-ussd/config/test/cic.ini @@ -1,5 +1,2 @@ [cic] -engine = evm -common_name = bloxberg -network_id = 8996 meta_url = http://test-meta.io diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index ce5bd65d..e4339004 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -6,7 +6,7 @@ celery==4.4.7 cffi==1.14.6 cic-eth~=0.12.5a1 cic-notify~=0.4.0a11 -cic-types~=0.2.0a5 +cic-types~=0.2.0a6 confini>=0.3.6rc4,<0.5.0 phonenumbers==8.12.12 psycopg2==2.8.6 diff --git a/apps/cic-ussd/test_requirements.txt b/apps/cic-ussd/test_requirements.txt index 59af6564..e893b546 100644 --- a/apps/cic-ussd/test_requirements.txt +++ b/apps/cic-ussd/test_requirements.txt @@ -1,3 +1,4 @@ +cic-eth[services]~=0.12.4a13 Faker==8.1.2 faker-e164==0.1.0 pytest==6.2.4 diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py index 82468fec..7fb5a7bc 100644 --- a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py +++ b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py @@ -4,8 +4,7 @@ import time # external imports import pytest -import requests_mock -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.statement import (filter_statement_transactions, @@ -48,7 +47,7 @@ def test_generate(activated_account, generate(querying_party, None, sender_transaction) time.sleep(2) identifier = bytes.fromhex(activated_account.blockchain_address) - key = cache_data_key(identifier, ':cic.statement') + key = cache_data_key(identifier, MetadataPointer.STATEMENT) statement = get_cached_data(key) statement = json.loads(statement) assert len(statement) == 1 diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py index a343647b..f5ec7f19 100644 --- a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py @@ -3,7 +3,7 @@ import json import datetime # external imports -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.balance import get_cached_available_balance @@ -58,7 +58,7 @@ def test_menu_processor(activated_account, token_symbol=token_symbol) identifier = bytes.fromhex(activated_account.blockchain_address) - key = cache_data_key(identifier, ':cic.adjusted_balance') + key = cache_data_key(identifier, MetadataPointer.BALANCES_ADJUSTED) adjusted_balance = 45931650.64654012 cache_data(key, json.dumps(adjusted_balance)) resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session) diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py b/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py index 25ccb8e6..e5bbd52f 100644 --- a/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py @@ -7,6 +7,7 @@ import time import i18n import requests_mock from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.chain import Chain @@ -45,7 +46,7 @@ def test_handle_menu(activated_account, ussd_menu = UssdMenu.find_by_name('initial_language_selection') assert menu_resp.get('name') == ussd_menu.get('name') identifier = bytes.fromhex(strip_0x(pending_account.blockchain_address)) - key = cache_data_key(identifier, ':cic.preferences') + key = cache_data_key(identifier, MetadataPointer.PREFERENCES) cache_data(key, json.dumps(preferences)) time.sleep(2) menu_resp = handle_menu(pending_account, init_database) diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py index fa017add..a0d4676d 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py @@ -1,20 +1,18 @@ # standard imports import json -from decimal import Decimal # external imports import celery import pytest -import requests_mock from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports -from cic_ussd.account.statement import generate, filter_statement_transactions +from cic_ussd.account.statement import filter_statement_transactions from cic_ussd.account.transaction import transaction_actors from cic_ussd.cache import cache_data_key, get_cached_data from cic_ussd.db.models.account import Account from cic_ussd.error import AccountCreationDataNotFound -from cic_ussd.metadata import PreferencesMetadata # test imports @@ -89,7 +87,7 @@ def test_balances_callback(activated_account, balances, celery_session_worker): [balances, activated_account.blockchain_address, status_code]) s_balances_callback.apply_async().get() identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - key = cache_data_key(identifier, ':cic.balances') + key = cache_data_key(identifier, MetadataPointer.BALANCES) cached_balances = get_cached_data(key) cached_balances = json.loads(cached_balances) assert cached_balances == balances[0] diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py index a00c82c7..5bb2b328 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py @@ -1,11 +1,11 @@ # standard imports import json -import os # external imports import celery import requests_mock from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.cache import cache_data_key, get_cached_data @@ -27,7 +27,7 @@ def test_query_person_metadata(activated_account, s_query_person_metadata = celery.signature( 'cic_ussd.tasks.metadata.query_person_metadata', [activated_account.blockchain_address]) s_query_person_metadata.apply().get() - key = cache_data_key(identifier, ':cic.person') + key = cache_data_key(identifier, MetadataPointer.PERSON) cached_person_metadata = get_cached_data(key) cached_person_metadata = json.loads(cached_person_metadata) assert cached_person_metadata == person_metadata @@ -46,7 +46,7 @@ def test_query_preferences_metadata(activated_account, query_preferences_metadata = celery.signature( 'cic_ussd.tasks.metadata.query_preferences_metadata', [activated_account.blockchain_address]) query_preferences_metadata.apply().get() - key = cache_data_key(identifier, ':cic.preferences') + key = cache_data_key(identifier, MetadataPointer.PREFERENCES) cached_preferences_metadata = get_cached_data(key) cached_preferences_metadata = json.loads(cached_preferences_metadata) assert cached_preferences_metadata == preferences diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py index 63a5edd7..fcfdef4a 100644 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py @@ -4,6 +4,7 @@ import json # external imports import celery from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.transaction import transaction_actors @@ -38,7 +39,7 @@ def test_cache_statement(activated_account, transaction_result): recipient_transaction, sender_transaction = transaction_actors(transaction_result) identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) - key = cache_data_key(identifier, ':cic.statement') + key = cache_data_key(identifier, MetadataPointer.STATEMENT) cached_statement = get_cached_data(key) assert cached_statement is None s_parse_transaction = celery.signature( diff --git a/apps/cic-ussd/tests/cic_ussd/test_cache.py b/apps/cic-ussd/tests/cic_ussd/test_cache.py index 6e7625b4..b3fe68cf 100644 --- a/apps/cic-ussd/tests/cic_ussd/test_cache.py +++ b/apps/cic-ussd/tests/cic_ussd/test_cache.py @@ -3,6 +3,7 @@ import hashlib import json # external imports +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.cache import cache_data, cache_data_key, get_cached_data @@ -12,7 +13,7 @@ from cic_ussd.cache import cache_data, cache_data_key, get_cached_data def test_cache_data(init_cache): identifier = 'some_key'.encode() - key = cache_data_key(identifier, ':testing') + key = cache_data_key(identifier, MetadataPointer.PERSON) assert get_cached_data(key) is None cache_data(key, json.dumps('some_value')) assert get_cached_data(key) is not None @@ -20,10 +21,10 @@ def test_cache_data(init_cache): def test_cache_data_key(): identifier = 'some_key'.encode() - key = cache_data_key(identifier, ':testing') + key = cache_data_key(identifier, MetadataPointer.PERSON) hash_object = hashlib.new("sha256") hash_object.update(identifier) - hash_object.update(':testing'.encode(encoding="utf-8")) + hash_object.update(':cic.person'.encode(encoding="utf-8")) assert hash_object.digest().hex() == key diff --git a/apps/cic-ussd/tests/fixtures/account.py b/apps/cic-ussd/tests/fixtures/account.py index 48034a03..5873e0a4 100644 --- a/apps/cic-ussd/tests/fixtures/account.py +++ b/apps/cic-ussd/tests/fixtures/account.py @@ -4,7 +4,7 @@ import random # external accounts import pytest -from chainlib.hash import strip_0x +from cic_types.condiments import MetadataPointer # local imports from cic_ussd.account.chain import Chain @@ -56,7 +56,7 @@ def cache_account_creation_data(init_cache, account_creation_data): def cache_balances(activated_account, balances, init_cache): identifier = bytes.fromhex(activated_account.blockchain_address) balances = json.dumps(balances[0]) - key = cache_data_key(identifier, ':cic.balances') + key = cache_data_key(identifier, MetadataPointer.BALANCES) cache_data(key, balances) @@ -64,7 +64,7 @@ def cache_balances(activated_account, balances, init_cache): def cache_default_token_data(default_token_data, init_cache, load_chain_spec): chain_str = Chain.spec.__str__() data = json.dumps(default_token_data) - key = cache_data_key(chain_str.encode('utf-8'), ':cic.default_token_data') + key = cache_data_key(chain_str.encode('utf-8'), MetadataPointer.TOKEN_DEFAULT) cache_data(key, data) @@ -72,7 +72,7 @@ def cache_default_token_data(default_token_data, init_cache, load_chain_spec): def cache_person_metadata(activated_account, init_cache, person_metadata): identifier = bytes.fromhex(activated_account.blockchain_address) person = json.dumps(person_metadata) - key = cache_data_key(identifier, ':cic.person') + key = cache_data_key(identifier, MetadataPointer.PERSON) cache_data(key, person) @@ -80,7 +80,7 @@ def cache_person_metadata(activated_account, init_cache, person_metadata): def cache_preferences(activated_account, init_cache, preferences): identifier = bytes.fromhex(activated_account.blockchain_address) preferences = json.dumps(preferences) - key = cache_data_key(identifier, ':cic.preferences') + key = cache_data_key(identifier, MetadataPointer.PREFERENCES) cache_data(key, preferences) @@ -88,7 +88,7 @@ def cache_preferences(activated_account, init_cache, preferences): def cache_statement(activated_account, init_cache, statement): identifier = bytes.fromhex(activated_account.blockchain_address) statement = json.dumps(statement) - key = cache_data_key(identifier, ':cic.statement') + key = cache_data_key(identifier, MetadataPointer.STATEMENT) cache_data(key, statement) diff --git a/apps/cic-ussd/tests/fixtures/config.py b/apps/cic-ussd/tests/fixtures/config.py index ed33dd50..eab100d8 100644 --- a/apps/cic-ussd/tests/fixtures/config.py +++ b/apps/cic-ussd/tests/fixtures/config.py @@ -41,11 +41,7 @@ def init_state_machine(load_config): @pytest.fixture(scope='function') def load_chain_spec(load_config): - chain_spec = ChainSpec( - common_name=load_config.get('CIC_COMMON_NAME'), - engine=load_config.get('CIC_ENGINE'), - network_id=load_config.get('CIC_NETWORK_ID') - ) + chain_spec = ChainSpec.from_chain_str(load_config.get('CHAIN_SPEC')) Chain.spec = chain_spec diff --git a/apps/contract-migration/config.sh b/apps/contract-migration/config.sh index 1594405b..ea69a52b 100644 --- a/apps/contract-migration/config.sh +++ b/apps/contract-migration/config.sh @@ -14,6 +14,7 @@ if [ ! -f $WALLET_KEY_FILE ]; then >&2 echo "wallet path '$WALLET_KEY_FILE' does not point to a file" exit 1 fi + export DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-keyfile -z -d $WALLET_KEY_FILE` noncefile=${DEV_DATA_DIR}/nonce_${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index d3dcc0f1..381fa94f 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -2,7 +2,6 @@ ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 - WORKDIR /root RUN touch /etc/apt/sources.list.d/ethereum.list diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index b1ab2f83..9d48803b 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -11,4 +11,4 @@ sarafu-faucet>=0.0.7a2,<0.1.0 confini>=0.4.2rc3,<1.0.0 crypto-dev-signer>=0.4.15rc2,<=0.4.15 eth-token-index>=0.2.4a1,<=0.3.0 -okota>=0.2.4a13,<0.3.0 +okota>=0.2.4a15,<0.3.0 diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index 8fb487a3..6f7f25e0 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -57,8 +57,8 @@ elif args.v: config = Config(args.c, args.env_prefix) config.process() args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), + 'CHAIN_SPEC': getattr(args, 'i'), + 'RPC_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), 'REDIS_HOST': getattr(args, 'redis_host'), 'REDIS_PORT': getattr(args, 'redis_port'), @@ -90,7 +90,7 @@ signer = EIP155Signer(keystore) block_offset = -1 if args.head else args.offset -chain_str = config.get('CIC_CHAIN_SPEC') +chain_str = config.get('CHAIN_SPEC') chain_spec = ChainSpec.from_chain_str(chain_str) ImportTask.chain_spec = chain_spec old_chain_spec_str = args.old_chain_spec @@ -99,16 +99,12 @@ old_chain_spec = ChainSpec.from_chain_str(old_chain_spec_str) MetadataTask.meta_host = config.get('META_HOST') MetadataTask.meta_port = config.get('META_PORT') -txs_dir = os.path.join(args.import_dir, 'txs') -os.makedirs(txs_dir, exist_ok=True) -sys.stdout.write(f'created txs dir: {txs_dir}') - celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) get_celery_worker_status(celery_app) def main(): - conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + conn = EthHTTPConnection(config.get('RPC_PROVIDER')) ImportTask.balance_processor = BalanceProcessor(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'), diff --git a/apps/data-seeding/cic_ussd/import_task.py b/apps/data-seeding/cic_ussd/import_task.py index 00e72f1a..aa702031 100644 --- a/apps/data-seeding/cic_ussd/import_task.py +++ b/apps/data-seeding/cic_ussd/import_task.py @@ -14,7 +14,9 @@ from celery import Task from chainlib.chain import ChainSpec from chainlib.eth.address import to_checksum_address from chainlib.eth.tx import raw, unpack -from cic_types.models.person import Person, generate_metadata_pointer +from cic_types.models.person import Person, identity_tag +from cic_types.processor import generate_metadata_pointer +from cic_types.condiments import MetadataPointer from hexathon import add_0x, strip_0x # local imports @@ -55,7 +57,7 @@ class MetadataTask(ImportTask): def old_address_from_phone(base_path: str, phone_number: str): - pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone') + pid_x = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE) phone_idx_path = os.path.join(f'{base_path}/phone/{pid_x[:2]}/{pid_x[2:4]}/{pid_x}') with open(phone_idx_path, 'r') as f: old_address = f.read() @@ -73,9 +75,13 @@ def generate_person_metadata(self, blockchain_address: str, phone_number: str): person = Person.deserialize(person_metadata) if not person.identities.get('evm'): person.identities['evm'] = {} - sub_chain_str = f'{self.chain_spec.common_name()}:{self.chain_spec.network_id()}' - person.identities['evm'][sub_chain_str] = [add_0x(blockchain_address)] - blockchain_address = strip_0x(blockchain_address) + chain_spec = self.chain_spec.asdict() + arch = chain_spec.get('arch') + fork = chain_spec.get('fork') + tag = identity_tag(chain_spec) + person.identities[arch][fork] = { + tag: [blockchain_address] + } file_path = os.path.join( self.import_dir, 'new', @@ -102,7 +108,7 @@ def generate_preferences_data(self, data: tuple): blockchain_address: str = data[0] preferences = data[1] preferences_dir = os.path.join(self.import_dir, 'preferences') - preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), ':cic.preferences') + preferences_key = generate_metadata_pointer(bytes.fromhex(strip_0x(blockchain_address)), MetadataPointer.PREFERENCES) preferences_filepath = os.path.join(preferences_dir, 'meta', preferences_key) filepath = os.path.join( preferences_dir, @@ -137,7 +143,7 @@ def generate_ussd_data(self, blockchain_address: str, phone_number: str): preferred_language = random.sample(["en", "sw"], 1)[0] preferences = {'preferred_language': preferred_language} with open(ussd_data_file, file_op) as ussd_data_file: - ussd_data_file.write(f'{phone_number}, { 1}, {preferred_language}, {False}\n') + ussd_data_file.write(f'{phone_number}, 1, {preferred_language}, False\n') logg.debug(f'written ussd data for address: {blockchain_address}') return blockchain_address, preferences @@ -163,7 +169,7 @@ def opening_balance_tx(self, blockchain_address: str, phone_number: str, serial: @celery_app.task(bind=True, base=MetadataTask) def resolve_phone(self, phone_number: str): - identifier = generate_metadata_pointer(phone_number.encode('utf-8'), ':cic.phone') + identifier = generate_metadata_pointer(phone_number.encode('utf-8'), MetadataPointer.PHONE) url = parse.urljoin(self.meta_url(), identifier) logg.debug(f'attempt getting phone pointer at: {url} for phone: {phone_number}') r = request.urlopen(url) diff --git a/apps/data-seeding/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py index 72013737..a5ccc286 100644 --- a/apps/data-seeding/cic_ussd/import_users.py +++ b/apps/data-seeding/cic_ussd/import_users.py @@ -17,6 +17,7 @@ from cic_types.models.person import Person from confini import Config # local imports +from common.dirs import initialize_dirs from import_util import get_celery_worker_status default_config_dir = './config' @@ -37,6 +38,7 @@ arg_parser.add_argument('--env-prefix', dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration.') +arg_parser.add_argument('-f', action='store_true', help='force clear previous state') arg_parser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') arg_parser.add_argument('-q', type=str, default='cic-import-ussd', help='celery queue to submit data seeding tasks to.') arg_parser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') @@ -70,21 +72,7 @@ config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug(f'config loaded from {args.c}:\n{config}') -old_account_dir = os.path.join(args.import_dir, 'old') -os.stat(old_account_dir) -logg.debug(f'created old system data dir: {old_account_dir}') - -new_account_dir = os.path.join(args.import_dir, 'new') -os.makedirs(new_account_dir, exist_ok=True) -logg.debug(f'created new system data dir: {new_account_dir}') - -person_metadata_dir = os.path.join(args.import_dir, 'meta') -os.makedirs(person_metadata_dir, exist_ok=True) -logg.debug(f'created person metadata dir: {person_metadata_dir}') - -preferences_dir = os.path.join(args.import_dir, 'preferences') -os.makedirs(os.path.join(preferences_dir, 'meta'), exist_ok=True) -logg.debug(f'created preferences metadata dir: {preferences_dir}') +dirs = initialize_dirs(args.import_dir, force_reset=args.f) valid_service_codes = config.get('USSD_SERVICE_CODE').split(",") @@ -157,7 +145,7 @@ def register_account(person: Person): if __name__ == '__main__': i = 0 j = 0 - for x in os.walk(old_account_dir): + for x in os.walk(dirs['old']): for y in x[2]: if y[len(y) - 5:] != '.json': continue diff --git a/apps/data-seeding/common/dirs.py b/apps/data-seeding/common/dirs.py index 8fa19c9d..1904ca52 100644 --- a/apps/data-seeding/common/dirs.py +++ b/apps/data-seeding/common/dirs.py @@ -13,11 +13,14 @@ def initialize_dirs(user_dir, force_reset=False): dirs['meta'] = os.path.join(user_dir, 'meta') dirs['custom'] = os.path.join(user_dir, 'custom') dirs['phone'] = os.path.join(user_dir, 'phone') + dirs['preferences'] = os.path.join(user_dir, 'preferences') dirs['txs'] = os.path.join(user_dir, 'txs') dirs['keyfile'] = os.path.join(user_dir, 'keystore') dirs['custom_new'] = os.path.join(dirs['custom'], 'new') dirs['custom_meta'] = os.path.join(dirs['custom'], 'meta') dirs['phone_meta'] = os.path.join(dirs['phone'], 'meta') + dirs['preferences_meta'] = os.path.join(dirs['preferences'], 'meta') + dirs['preferences_new'] = os.path.join(dirs['preferences'], 'new') try: os.stat(dirs['old']) diff --git a/apps/data-seeding/config/chain.ini b/apps/data-seeding/config/chain.ini new file mode 100644 index 00000000..9fda0988 --- /dev/null +++ b/apps/data-seeding/config/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = diff --git a/apps/data-seeding/config/cic.ini b/apps/data-seeding/config/cic.ini index 2e346dae..a424459d 100644 --- a/apps/data-seeding/config/cic.ini +++ b/apps/data-seeding/config/cic.ini @@ -1,10 +1,2 @@ [cic] registry_address = -token_index_address = -accounts_index_address = -declarator_address = -approval_escrow_address = -chain_spec = -tx_retry_delay = -trust_address = -user_ussd_svc_service_port = diff --git a/apps/data-seeding/create_import_users.py b/apps/data-seeding/create_import_users.py index 01d58c09..0eae5251 100644 --- a/apps/data-seeding/create_import_users.py +++ b/apps/data-seeding/create_import_users.py @@ -20,7 +20,7 @@ from cic_types.models.person import ( generate_vcard_from_contact_data, get_contact_data_from_vcard, ) -from chainlib.eth.address import to_checksum_address +from chainlib.eth.address import to_checksum_address, strip_0x import phonenumbers logging.basicConfig(level=logging.WARNING) @@ -30,7 +30,6 @@ fake = Faker(['sl', 'en_US', 'no', 'de', 'ro']) default_config_dir = './config' - argparser = argparse.ArgumentParser() argparser.add_argument('-c', type=str, default=default_config_dir, help='Config dir') argparser.add_argument('--tag', type=str, action='append', @@ -54,7 +53,6 @@ config = confini.Config(args.c, os.environ.get('CONFINI_ENV_PREFIX')) config.process() logg.debug('loaded config\n{}'.format(config)) - dt_now = datetime.datetime.utcnow() dt_then = dt_now - datetime.timedelta(weeks=150) ts_now = int(dt_now.timestamp()) @@ -64,7 +62,7 @@ celery_app = celery.Celery(broker=config.get( 'CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) gift_max = args.gift_threshold or 0 -gift_factor = (10**6) +gift_factor = (10 ** 6) categories = [ "food/water", @@ -105,7 +103,6 @@ def genId(addr, typ): def genDate(): - ts = random.randint(ts_then, ts_now) return int(datetime.datetime.fromtimestamp(ts).timestamp()) @@ -148,9 +145,7 @@ def genDob(): def gen(): - old_blockchain_address = '0x' + os.urandom(20).hex() - old_blockchain_checksum_address = to_checksum_address( - old_blockchain_address) + old_blockchain_address = os.urandom(20).hex() gender = random.choice(['female', 'male', 'other']) phone = genPhone() v = genPersonal(phone) @@ -164,9 +159,9 @@ def gen(): p.gender = gender p.identities = { 'evm': { - 'oldchain:1': [ - old_blockchain_checksum_address, - ], + 'foo': { + '1:oldchain': [old_blockchain_address], + }, }, } p.products = [fake.random_element(elements=OrderedDict( @@ -207,7 +202,7 @@ def gen(): # fake.local_latitude() p.location['longitude'] = (random.random() * 360) - 180 - return (old_blockchain_checksum_address, phone, p) + return old_blockchain_address, phone, p def prepareLocalFilePath(datadir, address): @@ -242,7 +237,7 @@ if __name__ == '__main__': except Exception as e: logg.warning('generate failed, trying anew: {}'.format(e)) continue - uid = eth[2:].upper() + uid = strip_0x(eth).upper() print(o) diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index f993e33c..ec6064d1 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -7,20 +7,23 @@ RUN mkdir -vp /usr/local/etc/cic COPY package.json \ package-lock.json \ - . + ./ RUN npm ci --production #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install +COPY common/ cic_ussd/common/ COPY requirements.txt . -COPY config/ /usr/local/etc/data-seeding +COPY config/ config -ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" -ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip pip install \ - --extra-index-url $GITLAB_PYTHON_REGISTRY \ - --extra-index-url $EXTRA_INDEX_URL -r requirements.txt +ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433 +ARG EXTRA_PIP_ARGS="" +ARG PIP_INDEX_URL=https://pypi.org/simple + +RUN pip install --index-url $PIP_INDEX_URL \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ + -r requirements.txt COPY . . diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index c46b3c04..f7a54969 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -47,7 +47,7 @@ argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8 argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing') argparser.add_argument('-c', type=str, help='config override directory') argparser.add_argument('-f', action='store_true', help='force clear previous state') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec') argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='Chain specification string') argparser.add_argument('-r', '--registry', dest='r', type=str, help='Contract registry address') argparser.add_argument('--batch-size', dest='batch_size', default=50, type=int, help='burst size of sending transactions to node') @@ -70,7 +70,7 @@ else: config.process() args_override = { 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), + 'CHAIN_SPEC': getattr(args, 'i'), 'KEYSTORE_FILE_PATH': getattr(args, 'y') } config.dict_override(args_override, 'cli') @@ -78,7 +78,7 @@ config.add(args.user_dir, '_USERDIR', True) #user_dir = args.user_dir -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) chain_str = str(chain_spec) old_chain_spec = ChainSpec.from_chain_str(args.old_chain_spec) @@ -106,7 +106,7 @@ account_registry_address = registry.parse_address_of(r) logg.info('using account registry {}'.format(account_registry_address)) dirs = initialize_dirs(config.get('_USERDIR'), force_reset=args.f) - +dirs['phone'] = os.path.join(config.get('_USERDIR')) def register_eth(i, u): @@ -165,8 +165,8 @@ if __name__ == '__main__': new_address = register_eth(i, u) if u.identities.get('evm') == None: u.identities['evm'] = {} - sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) - u.identities['evm'][sub_chain_str] = [new_address] + sub_chain_str = '{}:{}'.format(chain_spec.network_id(), chain_spec.common_name()) + u.identities['evm']['foo'][sub_chain_str] = [new_address] new_address_clean = strip_0x(new_address) filepath = os.path.join( @@ -188,7 +188,6 @@ if __name__ == '__main__': phone_object = phonenumbers.parse(u.tel) phone = phonenumbers.format_number(phone_object, phonenumbers.PhoneNumberFormat.E164) - logg.debug('>>>>> Using phone {}'.format(phone)) meta_phone_key = generate_metadata_pointer(phone.encode('utf-8'), MetadataPointer.PHONE) meta_phone_filepath = os.path.join(dirs['phone'], 'meta', meta_phone_key) @@ -221,9 +220,9 @@ if __name__ == '__main__': ) os.makedirs(os.path.dirname(filepath), exist_ok=True) - sub_old_chain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) + sub_old_chain_str = '{}:{}'.format(old_chain_spec.network_id(), old_chain_spec.common_name()) f = open(filepath, 'w') - k = u.identities['evm'][sub_old_chain_str][0] + k = u.identities['evm']['foo'][sub_old_chain_str][0] tag_data = {'tags': user_tags[strip_0x(k)]} f.write(json.dumps(tag_data)) f.close() diff --git a/apps/data-seeding/import_ussd.sh b/apps/data-seeding/import_ussd.sh old mode 100644 new mode 100755 index 1eb7e3f6..67f1292e --- a/apps/data-seeding/import_ussd.sh +++ b/apps/data-seeding/import_ussd.sh @@ -2,11 +2,11 @@ if [[ -d "$OUT_DIR" ]] then - echo "found existing IMPORT DIR cleaning up..." + echo -e "\033[;96mfound existing IMPORT DIR cleaning up...\033[;96m" rm -rf "$OUT_DIR" mkdir -p "$OUT_DIR" else - echo "IMPORT DIR does not exist creating it." + echo -e "\033[;96mIMPORT DIR does not exist creating it.\033[;96m" mkdir -p "$OUT_DIR" fi @@ -14,81 +14,81 @@ fi timeout 5 celery inspect ping -b "$CELERY_BROKER_URL" if [[ $? -eq 124 ]] then - >&2 echo "Celery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?" + >&2 echo -e "\033[;96mCelery workers not available. Is the CELERY_BROKER_URL ($CELERY_BROKER_URL) correct?\033[;96m" exit 1 fi -echo "Creating seed data..." +echo -e "\033[;96mCreating seed data...\033[;96m" python create_import_users.py -vv -c "$CONFIG" --dir "$OUT_DIR" "$NUMBER_OF_USERS" wait $! -echo "Check for running celery workers ..." +echo -e "\033[;96mCheck for running celery workers ...\033[;96m" if [ -f ./cic-ussd-import.pid ]; then - echo "Found a running worker. Killing ..." + echo -e "\033[;96mFound a running worker. Killing ...\033[;96m" kill -9 $( nohup.out 2> nohup.err < /dev/null & else - echo "Running worker with opening balance transactions" + echo -e "\033[;96mRunning worker with opening balance transactions\033[;96m" TARGET_TX_COUNT=$((NUMBER_OF_USERS*2)) nohup python cic_ussd/import_balance.py -vv -c "$CONFIG" -p "$ETH_PROVIDER" -r "$CIC_REGISTRY_ADDRESS" --include-balances --token-symbol "$TOKEN_SYMBOL" -y "$KEYSTORE_PATH" "$OUT_DIR" & fi -echo "Target count set to ${TARGET_TX_COUNT}" +echo -e "\033[;96mTarget count set to ${TARGET_TX_COUNT}" until [ -f ./cic-import-ussd.pid ] do - echo "Polling for celery worker pid file..." + echo -e "\033[;96mPolling for celery worker pid file...\033[;96m" sleep 1 done IMPORT_BALANCE_JOB=$(=0.5.1a1,<=0.5.15 faker==4.17.1 -chainsyncer~=0.0.6a3 -chainlib-eth~=0.0.9rc4 +chainsyncer~=0.0.7a3 +chainlib-eth~=0.0.10a10 eth-address-index~=0.2.4a1 eth-contract-registry~=0.6.3a3 eth-accounts-index~=0.1.2a3 diff --git a/apps/data-seeding/verify.py b/apps/data-seeding/verify.py index 56f6e7d0..fd59df45 100644 --- a/apps/data-seeding/verify.py +++ b/apps/data-seeding/verify.py @@ -25,10 +25,9 @@ from chainlib.eth.gas import ( from chainlib.eth.tx import TxFactory from chainlib.hash import keccak256_string_to_hex from chainlib.jsonrpc import JSONRPCRequest -from cic_types.models.person import ( - Person, - generate_metadata_pointer, -) +from cic_types.models.person import Person, identity_tag +from cic_types.condiments import MetadataPointer +from cic_types.processor import generate_metadata_pointer from erc20_faucet import Faucet from eth_erc20 import ERC20 from hexathon.parse import strip_0x, add_0x @@ -74,7 +73,7 @@ all_tests = eth_tests + custodial_tests + metadata_tests + phone_tests argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-c', type=str, help='config override dir') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:foo:1:oldchain', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', default='http://localhost:63315', help='cic-ussd url') @@ -108,7 +107,7 @@ config.process() # override args args_override = { 'CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), + 'RPC_PROVIDER': getattr(args, 'p'), 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), } config.dict_override(args_override, 'cli flag') @@ -307,7 +306,7 @@ class Verifier: def verify_gas(self, address, balance_token=None): - o = balance(address) + o = balance(add_0x(address)) r = self.conn.do(o) logg.debug('wtf {}'.format(r)) actual_balance = int(strip_0x(r), 16) @@ -323,7 +322,7 @@ class Verifier: def verify_metadata(self, address, balance=None): - k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), ':cic.person') + k = generate_metadata_pointer(bytes.fromhex(strip_0x(address)), MetadataPointer.PERSON) url = os.path.join(config.get('_META_PROVIDER'), k) logg.debug('verify metadata url {}'.format(url)) try: @@ -367,7 +366,7 @@ class Verifier: p = Person.deserialize(o) - k = generate_metadata_pointer(p.tel.encode('utf-8'), ':cic.phone') + k = generate_metadata_pointer(p.tel.encode('utf-8'), MetadataPointer.PHONE) url = os.path.join(config.get('_META_PROVIDER'), k) logg.debug('verify metadata phone url {}'.format(url)) try: @@ -427,7 +426,7 @@ class Verifier: def main(): global chain_str, block_offset, user_dir - conn = EthHTTPConnection(config.get('ETH_PROVIDER')) + conn = EthHTTPConnection(config.get('RPC_PROVIDER')) gas_oracle = OverrideGasOracle(conn=conn, limit=8000000) # Get Token registry address @@ -505,10 +504,17 @@ def main(): u = Person.deserialize(o) #logg.debug('data {}'.format(u.identities['evm'])) - subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) - new_address = u.identities['evm'][subchain_str][0] - subchain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) - old_address = u.identities['evm'][subchain_str][0] + new_chain_spec = chain_spec.asdict() + arch = new_chain_spec.get('arch') + fork = new_chain_spec.get('fork') + tag = identity_tag(new_chain_spec) + new_address = u.identities[arch][fork][tag][0] + + old_chainspec = old_chain_spec.asdict() + arch = old_chainspec.get('arch') + fork = old_chainspec.get('fork') + tag = identity_tag(old_chainspec) + old_address = u.identities[arch][fork][tag][0] balance = 0 try: balance = balances[old_address] diff --git a/docker-compose.yml b/docker-compose.yml index 1b47a962..39a257cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,15 @@ version: "3" volumes: - ganache-db: {} postgres-db: {} - bee-data: {} signer-data: {} bloxberg-data: {} contract-config: {} -networks: - default: - name: cic-network services: - eth: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/bloxberg-node:${TAG:-latest} + evm: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/bloxberg-node:${TAG:-latest} build: context: apps/bloxbergValidatorSetup restart: unless-stopped @@ -26,7 +21,6 @@ services: - ./apps/bloxbergValidatorSetup/keys:/root/keys # stores the signing key locally - bloxberg-data:/root/.local/share/io.parity.ethereum/ - # See contents of /initdb/create_db.sql for app user, password and databases postgres: image: postgres:12.5-alpine environment: @@ -40,272 +34,125 @@ services: - postgres-db:/var/lib/postgresql/data redis: - image: redis:6.0.9-alpine + image: ${DEV_DOCKER_REGISTRY:-docker.io}/redis:6.0.9-alpine ports: - ${DEV_REDIS_PORT:-63379}:6379 command: "--loglevel verbose" - bee: - image: ethersphere/bee:0.4.1 - container_name: bee - environment: - BEE_NETWORK_ID: ${BEE_NETWORK_ID:-313} - BEE_PASSWORD: ${BEE_PASSWORD:-password} - ports: - - ${DEV_BEE_PORT:-63633}:1633 - - ${DEV_BEE_PORT_DEBUG:-63635}:1635 - command: "start --swap-enable=false --standalone" - volumes: - - bee-data:/tmp/cic/bee - - contract-migration: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:${TAG:-latest} + + bootstrap: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/contract-migration:${TAG:-latest} build: context: apps/contract-migration dockerfile: docker/Dockerfile args: - pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} - pip_extra_args: $PIP_EXTRA_ARGS - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS - # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest + DOCKER_REGISTRY: $DEV_DOCKER_REGISTRY + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: - RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} - DEV_USE_DOCKER_WAIT_SCRIPT: 1 - CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config} DEV_CONFIG_RESET: $DEV_CONFIG_RESET - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DEV_DATABASE_NAME_CIC_ETH:-cic_eth} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgresql} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_USER: ${DATABASE_USER:-postgres} + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} REDIS_HOST: ${REDIS_HOST:-redis} REDIS_PORT: ${REDIS_PORT:-6379} REDIS_DB: ${REDIS_DB:-0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - RUN_MASK: ${RUN_MASK:-0} # bit flags; 1: contract migrations 2: seed data - DEV_FAUCET_AMOUNT: ${DEV_FAUCET_AMOUNT:-50000000} - DEV_ETH_GAS_PRICE: $DEV_ETH_GAS_PRICE - TOKEN_NAME: ${TOKEN_NAME:-Giftable Token} - TOKEN_SYMBOL: ${TOKEN_SYMBOL:-GFT} - TOKEN_TYPE: ${TOKEN_TYPE:-giftable_erc20_token} + RUN_MASK: ${RUN_MASK:-0} + TOKEN_NAME: $TOKEN_NAME + TOKEN_SYMBOL: $TOKEN_SYMBOL + TOKEN_TYPE: $TOKEN_TYPE TOKEN_DECIMALS: $TOKEN_DECIMALS - TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_DEMURRAGE_REDISTRIBUTION_PERIOD - TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} + TOKEN_REDISTRIBUTION_PERIOD: $TOKEN_REDISTRIBUTION_PERIOD TOKEN_SUPPLY_LIMIT: $TOKEN_SUPPLY_LIMIT TOKEN_DEMURRAGE_LEVEL: $TOKEN_DEMURRAGE_LEVEL TOKEN_SINK_ADDRESS: $TOKEN_SINK_ADDRESS - SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} - restart: on-failure + REDIS_HOST_CALLBACK: ${REDIS_HOST_CALLBACK:-redis} + REDIS_PORT_CALLBACK: ${REDIS_PORT_CALLBACK:-6379} + FAUCET_AMOUNT: ${FAUCET_AMOUNT:-0} command: ["./run_job.sh"] - #command: ["./reset.sh"] depends_on: - - eth + - evm - postgres - redis - - cic-eth-tasker + #- cic-eth-tasker volumes: - contract-config:/tmp/cic/config - data-seeding: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/data-seeding:${TAG:-latest} - build: - context: apps/data-seeding - dockerfile: docker/Dockerfile - args: - pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} - pip_extra_args: $PIP_EXTRA_ARGS - environment: - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - OUT_DIR: out - NUMBER_OF_USERS: 10 - CONFIG: /usr/local/etc/data-seeding - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - TOKEN_SYMBOL: GFT - KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c - USSD_HOST: cic-user-ussd-server - USSD_PORT: 9000 - INCLUDE_BALANCES: y - USSD_SSL: n - DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} - NOTIFY_DATABASE_NAME: cic_notify - REDIS_HOST: redis - REDIS_PORT: 6379 - REDIS_DB: 0 - META_HOST: meta - META_PORT: 8000 - META_URL: http://meta:8000 - USSD_PROVIDER: http://cic-user-ussd-server:9000 - CELERY_QUEUE: cic-import-ussd - EXCLUSIONS: ussd - command: bash import_ussd.sh - volumes: - - contract-config:/tmp/cic/config/:ro - cic-cache-tracker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - build: - context: apps/cic-cache - dockerfile: docker/Dockerfile - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - environment: - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: 1 - CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} - CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} - CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} - CELERY_BROKER_URL: redis://redis:6379 - CELERY_RESULT_URL: redis://redis:6379 - restart: on-failure - depends_on: - - redis - - postgres - - eth - command: - - /bin/bash - - -c - - | - #if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - ./start_tracker.sh -c /usr/local/etc/cic-cache -vv - volumes: - - contract-config:/tmp/cic/config/:ro - - cic-cache-tasker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - build: - context: apps/cic-cache - dockerfile: docker/Dockerfile - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - environment: - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: 1 - DATABASE_POOL_SIZE: 0 - CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CELERY_BROKER_URL: redis://redis:6379 - CELERY_RESULT_URL: redis://redis:6379 - restart: unless-stopped - depends_on: - - redis - - postgres - - eth - command: - - /bin/bash - - -c - - | - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - /usr/local/bin/cic-cache-taskerd -vv - volumes: - - contract-config:/tmp/cic/config/:ro - - cic-cache-server: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-cache:${TAG:-latest} - build: - context: apps/cic-cache - dockerfile: docker/Dockerfile - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - environment: - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PORT: ${DATABASE_PORT:-5432} - #DATABASE_PASSWORD: ${DATABASE_PASSWORD:- - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} - DATABASE_DEBUG: 1 - #PGPASSWORD: $DATABASE_PASSWORD - SERVER_PORT: 8000 - restart: on-failure - ports: - - ${HTTP_PORT_CIC_CACHE:-63313}:8000 - depends_on: - - postgres - - cic-cache-tasker - - cic-cache-tracker - command: - - /bin/bash - - -c - - | - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - "/usr/local/bin/uwsgi" \ - --wsgi-file /root/cic_cache/runnable/daemons/server.py \ - --http :8000 \ - --pyargv "-vv" - - - cic-eth-tasker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} + cic-signer: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/funga-eth:${TAG:-latest} build: - context: apps/cic-eth - dockerfile: docker/Dockerfile - target: dev - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + context: apps/cic-signer + dockerfile: Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_signer} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000} + SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} + depends_on: + - postgres + volumes: + - signer-data:/run/crypto-dev-signer + - contract-config:/tmp/cic/config/:ro + command: ["python", "/usr/local/bin/funga-ethd", "-vv"] + + # queue handling for outgoing transactions and incoming transactions + cic-eth-tasker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} + build: + context: apps/cic-eth + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER - RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} - DATABASE_USER: ${DATABASE_USER:-grassroots} + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_ETH:-cic_eth} DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 REDIS_PORT: 6379 REDIS_HOST: redis - PGPASSWORD: ${DATABASE_PASSWORD:-tralala} - CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} - #SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://cic-eth-signer:8000} - SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-eth-signer:8000} + SIGNER_PROVIDER: ${SIGNER_PROVIDER:-http://cic-signer:8000} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} - ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} - CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} restart: unless-stopped depends_on: - - eth + - evm - postgres - redis - - cic-eth-signer + - cic-signer volumes: - signer-data:/run/crypto-dev-signer - contract-config:/tmp/cic/config/:ro - #command: ["/usr/local/bin/cic-eth-taskerd"] - #command: ["sleep", "3600"] command: - /bin/bash - -c @@ -314,87 +161,42 @@ services: if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi set +a ./start_tasker.sh --aux-all -q cic-eth -vv - - cic-eth-signer: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} + + + cic-eth-tracker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} build: - context: apps/cic-eth - dockerfile: docker/Dockerfile - target: dev - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + context: apps/cic-eth + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER - ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - RPC_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - DATABASE_USER: ${DATABASE_USER:-grassroots} + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_ETH:-cic_eth} DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 - REDIS_PORT: 6379 - REDIS_HOST: redis - PGPASSWORD: ${DATABASE_PASSWORD:-tralala} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} - SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000} - SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-http://0.0.0.0:8000} - SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} - ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} - TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} - CIC_DEFAULT_TOKEN_SYMBOL: ${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} - restart: on-failure - depends_on: - - eth - - postgres - - redis - volumes: - - signer-data:/run/crypto-dev-signer - - contract-config:/tmp/cic/config/:ro - command: ["python", "/usr/local/bin/crypto-dev-daemon", "-c", "/usr/local/etc/crypto-dev-signer", "-vv"] - #command: - # - /bin/bash - # - -c - # - | - # if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - # ./start_tasker.sh --aux-all -q cic-eth -vv - # command: [/bin/sh, "./start_tasker.sh", -q, cic-eth, -vv ] - - cic-eth-tracker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} - build: - context: apps/cic-eth - dockerfile: docker/Dockerfile - target: dev - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - environment: - RPC_PROVIDER: ${RPC_PROVIDER:-http://eth:8545} - DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth} - DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} - CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} + SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-5} + SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} + SYNCER_OFFSET: ${SYNCER_OFFSET:-0} TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} - restart: on-failure + restart: unless-stopped depends_on: - - eth + - evm - postgres - redis volumes: @@ -403,145 +205,264 @@ services: - /bin/bash - -c - | + set -a if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - #./start_tracker.sh -vv -c /usr/local/etc/cic-eth + set +a ./start_tracker.sh -vv - # command: "/root/start_manager.sh head -vv" cic-eth-dispatcher: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} - build: - context: apps/cic-eth - dockerfile: docker/Dockerfile - target: dev - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} + build: + context: apps/cic-eth + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: - ETH_PROVIDER: http://eth:8545 - RPC_PROVIDER: http://eth:8545 - DATABASE_USER: ${DATABASE_USER:-grassroots} + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth} DATABASE_PORT: ${DATABASE_PORT:-5432} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - #BANCOR_DIR: $BANCOR_DIR - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} - CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} - TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS - DATABASE_DEBUG: ${DATABASE_DEBUG:-false} - #DATABASE_DEBUG: 1 - restart: on-failure - depends_on: - - eth - - postgres - - redis - volumes: - - contract-config:/tmp/cic/config/:ro - command: - - /bin/bash - - -c - - | - if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - ./start_dispatcher.sh -q cic-eth -vv - # command: "/root/start_dispatcher.sh -q cic-eth -vv" - - - - cic-eth-retrier: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-eth:${TAG:-latest} - build: - context: apps/cic-eth - dockerfile: docker/Dockerfile - target: dev - args: - EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} - environment: - ETH_PROVIDER: http://eth:8545 - RPC_PROVIDER: http://eth:8545 + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} DATABASE_USER: ${DATABASE_USER:-grassroots} - DATABASE_HOST: ${DATABASE_HOST:-postgres} - DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_eth} - DATABASE_PORT: ${DATABASE_PORT:-5432} DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - #BANCOR_DIR: $BANCOR_DIR + DATABASE_POOL_SIZE: 0 CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} - TASKS_TRANSFER_CALLBACKS: $TASKS_TRANSFER_CALLBACKS - CIC_TX_RETRY_DELAY: 60 - BATCH_SIZE: ${RETRIER_BATCH_SIZE:-50} - #DATABASE_DEBUG: 1 - restart: on-failure + CELERY_DEBUG: ${CELERY_DEBUG:-1} + CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} + DISPATCHER_LOOP_INTERVAL: ${DISPATCHER_LOOP_INTERVAL:-1} + restart: unless-stopped depends_on: - - eth + - evm - postgres - redis volumes: - - contract-config:/tmp/cic/config/:ro + - contract-config:/tmp/cic/config/:ro command: - /bin/bash - -c - | + set -a if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi - ./start_retry.sh -vv - # command: "/root/start_retry.sh -q cic-eth -vv" + set +a + ./start_dispatcher.sh -vv - - cic-notify-tasker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-notify:${TAG:-latest} - build: - context: apps/cic-notify + cic-eth-retrier: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-eth:${TAG:-latest} + build: + context: apps/cic-eth dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS environment: - DATABASE_USER: ${DATABASE_USER:-grassroots} + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_eth} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} - DATABASE_NAME: ${DATABASE_NAME_CIC_NOTIFY:-cic_notify} + DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 - PGPASSWORD: ${DATABASE_PASSWORD:-tralala} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} - CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis} - AFRICASTALKING_API_USERNAME: $AFRICASTALKING_API_USERNAME - AFRICASTALKING_API_KEY: $AFRICASTALKING_API_KEY - AFRICASTALKING_API_SENDER_ID: $AFRICASTALKING_API_SENDER_ID + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} + SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-20} + RETRY_DELAY: ${RETRY_DELAY:-60} + RETRY_BATCH_SIZE: ${RETRY_BATCH_SIZE:-100} restart: unless-stopped depends_on: + - evm - postgres - redis - command: "/root/start_tasker.sh -q cic-notify -vv" + volumes: + - contract-config:/tmp/cic/config/:ro + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./start_retry.sh -vv + + cic-cache-tracker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} + build: + context: apps/cic-cache + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_cache} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} + SYNCER_LOOP_INTERVAL: ${SYNCER_LOOP_INTERVAL:-5} + SYNCER_OFFSET: ${SYNCER_OFFSET:-0} + SYNCER_NO_HISTORY: ${SYNCER_NO_HISTORY:-1} + TASKS_TRANSFER_CALLBACKS: ${TASKS_TRANSFER_CALLBACKS:-"cic-eth:cic_eth.callbacks.noop.noop,cic-ussd:cic_ussd.tasks.callback_handler.transaction_callback"} + restart: unless-stopped + depends_on: + - evm + - postgres + - redis + volumes: + - contract-config:/tmp/cic/config/:ro + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./start_tracker.sh -vv + + + cic-cache-tasker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} + build: + context: apps/cic-cache + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_cache} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + CELERY_DEBUG: ${CELERY_DEBUG:-1} + CELERY_QUEUE: ${CELERY_QUEUE:-cic-eth} + restart: unless-stopped + depends_on: + - evm + - postgres + - redis + volumes: + - contract-config:/tmp/cic/config/:ro + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + /usr/local/bin/cic-cache-taskerd -vv +# "/usr/local/bin/uwsgi" \ +# --wsgi-file /root/cic_cache/runnable/daemons/server.py \ +# --http :8000 \ +# --pyargv "-vv" + + + cic-cache-server: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-cache:${TAG:-latest} + build: + context: apps/cic-cache + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_cache} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + SERVER_PORT: 8000 + restart: on-failure + ports: + - ${DEV_CIC_CACHE_SERVER_PORT:-63313}:8000 + depends_on: + - postgres + - cic-cache-tracker + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + /usr/local/bin/uwsgi \ + --wsgi-file /root/cic_cache/runnable/daemons/server.py \ + --http :8000 \ + --pyargv "-vv" + + volumes: + - contract-config:/tmp/cic/config/:ro + + + # metadata replacement server for swarm cic-meta-server: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-meta:${TAG:-latest} + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-meta:${TAG:-latest} hostname: meta build: context: apps/cic-meta dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} environment: - DATABASE_NAME: ${DATABASE_NAME:-cic_meta} - DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_meta} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + DATABASE_SCHEMA_SQL_PATH: "" SERVER_HOST: localhost SERVER_PORT: 8000 - DATABASE_SCHEMA_SQL_PATH: "" PGP_EXPORTS_DIR: /root/tests/ PGP_PRIVATEKEY_FILE: privatekeys.asc PGP_PASSPHRASE: merman @@ -551,87 +472,198 @@ services: SCHEMA_SQL_PATH: scripts/initdb/server.postgres.sql restart: on-failure ports: - - ${HTTP_PORT_CIC_META:-63380}:8000 + - ${DEV_CIC_META_SERVER_PORT:-63380}:8000 depends_on: - postgres volumes: - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp - # command: "/root/start_server.sh -vv" - cic-user-ussd-server: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - build: - context: apps/cic-ussd - dockerfile: docker/Dockerfile - environment: - APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} - DATABASE_USER: grassroots - DATABASE_HOST: postgres - DATABASE_PORT: 5432 - DATABASE_PASSWORD: tralala - DATABASE_NAME: cic_ussd - DATABASE_ENGINE: postgresql - DATABASE_DRIVER: psycopg2 - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} - CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis} - PGP_PASSPHRASE: merman - SERVER_PORT: 9000 - CIC_META_URL: ${CIC_META_URL:-http://meta:8000} - restart: on-failure - ports: - - ${HTTP_PORT_CIC_USER_USSD_SERVER:-63315}:9000 - depends_on: - - postgres - - redis - volumes: - - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ - command: "/root/start_cic_user_ussd_server.sh -vv" - - cic-user-server: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} - build: - context: apps/cic-ussd - dockerfile: docker/Dockerfile - environment: - APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} - DATABASE_USER: grassroots - DATABASE_HOST: postgres - DATABASE_PORT: 5432 - DATABASE_PASSWORD: tralala - DATABASE_NAME: cic_ussd - DATABASE_ENGINE: postgresql - DATABASE_DRIVER: psycopg2 - DATABASE_POOL_SIZE: 0 - restart: on-failure - ports: - - ${HTTP_PORT_CIC_USER_SERVER:-63415}:9500 - depends_on: - - postgres - command: "/root/start_cic_user_server.sh -vv" cic-user-tasker: - image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/cic-ussd:${TAG:-latest} + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} build: - context: apps/cic-ussd/ + context: apps/cic-ussd dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} environment: - APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} - DATABASE_USER: grassroots - DATABASE_HOST: postgres - DATABASE_PORT: 5432 - DATABASE_PASSWORD: tralala - DATABASE_NAME: cic_ussd - DATABASE_ENGINE: postgresql - DATABASE_DRIVER: psycopg2 + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} DATABASE_POOL_SIZE: 0 CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + REDIS_PORT: 6379 + REDIS_HOST: redis PGP_PASSPHRASE: merman CIC_META_URL: ${CIC_META_URL:-http://meta:8000} restart: unless-stopped depends_on: - postgres - redis + #- cic-meta-server + - cic-eth-tasker + - cic-cache-tasker volumes: - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ command: "/root/start_cic_user_tasker.sh -q cic-ussd -vv" + + + cic-user-server: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} + build: + context: apps/cic-ussd + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + restart: on-failure + ports: + - ${DEV_CIC_USER_SERVER_PORT:-63415}:9500 + depends_on: + - postgres + - redis + command: "/root/start_cic_user_server.sh -vv" + + + cic-user-ussd-server: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-user:${TAG:-latest} + build: + context: apps/cic-ussd + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + REDIS_PORT: 6379 + REDIS_HOST: redis + restart: on-failure + depends_on: + - postgres + - redis + - cic-eth-tasker + - cic-cache-tasker + ports: + - ${DEV_CIC_USER_USSD_SERVER_PORT:-63315}:9500 + volumes: + - ./apps/contract-migration/testdata/pgp/:/usr/src/secrets/ + command: "/root/start_cic_user_ussd_server.sh -vv" + + + cic-notify-tasker: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/cic-notify:${TAG:-latest} + build: + context: apps/cic-notify + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_notify} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + AFRICASTALKING_API_USERNAME: $AFRICASTALKING_API_USERNAME + AFRICASTALKING_API_KEY: $AFRICASTALKING_API_KEY + AFRICASTALKING_API_SENDER_ID: $AFRICASTALKING_API_SENDER_ID + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} + restart: unless-stopped + depends_on: + - postgres + - redis + command: "/root/start_tasker.sh -q cic-notify -vv" + + data-seeding: + image: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics/cic-internal-integration}/data-seeding:${TAG:-latest} + build: + context: apps/data-seeding + dockerfile: docker/Dockerfile + args: + DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics} + PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple} + EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS + environment: + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME:-cic_ussd} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: ${DATABASE_DEBUG:-0} + DATABASE_POOL_SIZE: 0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379} + CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379} + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS + RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545} + OUT_DIR: out + NUMBER_OF_USERS: 10 + CONFIG: config + CHAIN_SPEC: ${CHAIN_SPEC:-evm:byzantium:8996:bloxberg} + TOKEN_SYMBOL: GFT + KEYSTORE_PATH: keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c + USSD_HOST: cic-user-ussd-server + USSD_PORT: 9000 + INCLUDE_BALANCES: y + USSD_SSL: n + NOTIFY_DATABASE_NAME: cic_notify + REDIS_HOST: redis + REDIS_PORT: 6379 + REDIS_DB: 0 + META_HOST: meta + META_PORT: 8000 + META_URL: http://meta:8000 + USSD_PROVIDER: http://cic-user-ussd-server:9000 + CELERY_QUEUE: cic-import-ussd + EXCLUSIONS: ussd + command: + - /bin/bash + - -c + - | + set -a + if [[ -f /tmp/cic/config/env_reset ]]; then source /tmp/cic/config/env_reset; fi + set +a + ./import_ussd.sh + volumes: + - contract-config:/tmp/cic/config/:ro \ No newline at end of file