Merge branch 'philip/ussd-demurrage' into 'master'

Philip/ussd demurrage

See merge request grassrootseconomics/cic-internal-integration!263

apps/cic-ussd/cic_ussd/account/balance.py

@@ -1,10 +1,12 @@
 # standard imports
+
 import json
 import logging
 from typing import Optional
 
 # third-party imports
 from cic_eth.api import Api
+from cic_eth_aux.erc20_demurrage_token.api import Api as DemurrageApi
 
 # local imports
 from cic_ussd.account.transaction import from_wei
@@ -73,6 +75,24 @@ def calculate_available_balance(balances: dict) -> float:
     return from_wei(value=available_balance)
 
 
+def get_adjusted_balance(balance: int, chain_str: str, timestamp: int, token_symbol: str):
+    """
+    :param balance:
+    :type balance:
+    :param chain_str:
+    :type chain_str:
+    :param timestamp:
+    :type timestamp:
+    :param token_symbol:
+    :type token_symbol:
+    :return:
+    :rtype:
+    """
+    logg.debug(f'retrieving adjusted balance on chain: {chain_str}')
+    demurrage_api = DemurrageApi(chain_str=chain_str)
+    return demurrage_api.get_adjusted_balance(token_symbol, balance, timestamp).result
+
+
 def get_cached_available_balance(blockchain_address: str) -> float:
     """This function attempts to retrieve balance data from the redis cache.
     :param blockchain_address: Ethereum address of an account.
@@ -88,3 +108,14 @@ def get_cached_available_balance(blockchain_address: str) -> float:
         return calculate_available_balance(json.loads(cached_balances))
     else:
         raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}')
+
+
+def get_cached_adjusted_balance(identifier: bytes):
+    """
+    :param identifier:
+    :type identifier:
+    :return:
+    :rtype:
+    """
+    key = cache_data_key(identifier, ':cic.adjusted_balance')
+    return get_cached_data(key)

apps/cic-ussd/cic_ussd/processor/menu.py

@@ -1,13 +1,17 @@
 # standard imports
 import json
 import logging
+from datetime import datetime, timedelta
 
 # external imports
 import i18n.config
-from sqlalchemy.orm.session import Session
 
 # local imports
-from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance
+from cic_ussd.account.balance import (calculate_available_balance,
+                                      get_adjusted_balance,
+                                      get_balances,
+                                      get_cached_adjusted_balance,
+                                      get_cached_available_balance)
 from cic_ussd.account.chain import Chain
 from cic_ussd.account.metadata import get_cached_preferred_language
 from cic_ussd.account.statement import (
@@ -16,14 +20,15 @@ from cic_ussd.account.statement import (
     query_statement,
     statement_transaction_set
 )
-from cic_ussd.account.transaction import from_wei, to_wei
 from cic_ussd.account.tokens import get_default_token_symbol
+from cic_ussd.account.transaction import from_wei, to_wei
 from cic_ussd.cache import cache_data_key, cache_data
 from cic_ussd.db.models.account import Account
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
-from cic_ussd.processor.util import latest_input, parse_person_metadata
+from cic_ussd.processor.util import parse_person_metadata
 from cic_ussd.translation import translation_for
+from sqlalchemy.orm.session import Session
 
 logg = logging.getLogger(__name__)
 
@@ -43,21 +48,26 @@ class MenuProcessor:
         :rtype:
         """
         available_balance = get_cached_available_balance(self.account.blockchain_address)
-        logg.debug('Requires call to retrieve tax and bonus amounts')
+        adjusted_balance = get_cached_adjusted_balance(self.identifier)
-        tax = ''
-        bonus = ''
         token_symbol = get_default_token_symbol()
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
-        return translation_for(
+        with_available_balance = f'{self.display_key}.available_balance'
-            key=self.display_key,
+        with_fees = f'{self.display_key}.with_fees'
-            preferred_language=preferred_language,
+        if not adjusted_balance:
-            available_balance=available_balance,
+            return translation_for(key=with_available_balance,
-            tax=tax,
+                                   preferred_language=preferred_language,
-            bonus=bonus,
+                                   available_balance=available_balance,
-            token_symbol=token_symbol
+                                   token_symbol=token_symbol)
-        )
+        adjusted_balance = json.loads(adjusted_balance)
+        tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
+        tax = from_wei(int(tax_wei))
+        return translation_for(key=with_fees,
+                               preferred_language=preferred_language,
+                               available_balance=available_balance,
+                               tax=tax,
+                               token_symbol=token_symbol)
 
     def account_statement(self) -> str:
         """
@@ -67,7 +77,7 @@ class MenuProcessor:
         cached_statement = get_cached_statement(self.account.blockchain_address)
         statement = json.loads(cached_statement)
         statement_transactions = parse_statement_transactions(statement)
-        transaction_sets = [statement_transactions[tx:tx+3] for tx in range(0, len(statement_transactions), 3)]
+        transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)]
         preferred_language = get_cached_preferred_language(self.account.blockchain_address)
         if not preferred_language:
             preferred_language = i18n.config.get('fallback')
@@ -149,12 +159,22 @@ class MenuProcessor:
         :return:
         :rtype:
         """
+        chain_str = Chain.spec.__str__()
         token_symbol = get_default_token_symbol()
         blockchain_address = self.account.blockchain_address
-        balances = get_balances(blockchain_address, Chain.spec.__str__(), token_symbol, False)[0]
+        balances = get_balances(blockchain_address, chain_str, token_symbol, False)[0]
         key = cache_data_key(self.identifier, ':cic.balances')
         cache_data(key, json.dumps(balances))
         available_balance = calculate_available_balance(balances)
+        now = datetime.now()
+        if (now - self.account.created).days >= 30:
+            if available_balance <= 0:
+                logg.info(f'Not retrieving adjusted balance, available balance: {available_balance} is insufficient.')
+            else:
+                timestamp = int((now - timedelta(30)).timestamp())
+                adjusted_balance = get_adjusted_balance(to_wei(int(available_balance)), chain_str, timestamp, token_symbol)
+                key = cache_data_key(self.identifier, ':cic.adjusted_balance')
+                cache_data(key, json.dumps(adjusted_balance))
 
         query_statement(blockchain_address)
 

apps/cic-ussd/cic_ussd/runnable/client.py

@@ -63,7 +63,7 @@ elif ssl == 0:
 else:
     ssl = True
 
-
+valid_service_codes = config.get('USSD_SERVICE_CODE').split(",")
 def main():
 
     # TODO: improve url building 
@@ -79,9 +79,9 @@ def main():
     session = uuid.uuid4().hex
     data = {
             'sessionId': session,
-            'serviceCode': config.get('APP_SERVICE_CODE'),
+            'serviceCode': valid_service_codes[0],
             'phoneNumber': args.phone,
-            'text': config.get('APP_SERVICE_CODE'),
+            'text': "",
         }
 
     state = "_BEGIN"

apps/cic-ussd/docker/Dockerfile

@@ -10,6 +10,13 @@ RUN mkdir -vp data
 
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
+
+RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    pip install --index-url https://pypi.org/simple  \
+	  --extra-index-url $GITLAB_PYTHON_REGISTRY \
+    --extra-index-url $EXTRA_INDEX_URL \
+    cic-eth-aux-erc20-demurrage-token~=0.0.2a6
+
 COPY requirements.txt .
 
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \

apps/cic-ussd/requirements.txt

@@ -1,6 +1,9 @@
 alembic==1.4.2
+attrs==21.2.0
+billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
+cffi==1.14.6
 cic-eth[services]~=0.12.4a7
 cic-notify~=0.4.0a10
 cic-types~=0.1.0a14

apps/cic-ussd/tests/cic_ussd/account/test_balance.py

@@ -43,10 +43,13 @@ def test_sync_get_balances(activated_account,
     (5000000, 89000000, 67000000, 27.00)
 ])
 def test_calculate_available_balance(activated_account,
+                                     available_balance,
                                      balance_incoming,
                                      balance_network,
                                      balance_outgoing,
-                                     available_balance):
+                                     cache_balances,
+                                     cache_default_token_data,
+                                     load_chain_spec):
     balances = {
         'address': activated_account.blockchain_address,
         'converters': [],
@@ -57,7 +60,11 @@ def test_calculate_available_balance(activated_account,
     assert calculate_available_balance(balances) == available_balance
 
 
-def test_get_cached_available_balance(activated_account, cache_balances, balances):
+def test_get_cached_available_balance(activated_account,
+                                      balances,
+                                      cache_balances,
+                                      cache_default_token_data,
+                                      load_chain_spec):
     cached_available_balance = get_cached_available_balance(activated_account.blockchain_address)
     available_balance = calculate_available_balance(balances[0])
     assert cached_available_balance == available_balance

apps/cic-ussd/tests/cic_ussd/account/test_statement.py

@@ -27,6 +27,8 @@ def test_filter_statement_transactions(transactions_list):
 
 
 def test_generate(activated_account,
+                  cache_default_token_data,
+                  cache_statement,
                   cache_preferences,
                   celery_session_worker,
                   init_cache,
@@ -60,7 +62,7 @@ def test_get_cached_statement(activated_account, cache_statement, statement):
     assert cached_statement[0].get('blockchain_address') == statement[0].get('blockchain_address')
 
 
-def test_parse_statement_transactions(statement):
+def test_parse_statement_transactions(cache_default_token_data, statement):
     parsed_transactions = parse_statement_transactions(statement)
     parsed_transaction = parsed_transactions[0]
     parsed_transaction.startswith('Sent')
@@ -76,7 +78,7 @@ def test_query_statement(blockchain_address, limit, load_chain_spec, activated_a
     assert mock_transaction_list_query.get('limit') == limit
 
 
-def test_statement_transaction_set(preferences, set_locale_files, statement):
+def test_statement_transaction_set(cache_default_token_data, load_chain_spec, preferences, set_locale_files, statement):
     parsed_transactions = parse_statement_transactions(statement)
     preferred_language = preferences.get('preferred_language')
     transaction_set = statement_transaction_set(preferred_language, parsed_transactions)

apps/cic-ussd/tests/cic_ussd/account/test_transaction.py

@@ -36,19 +36,19 @@ def test_aux_transaction_data(preferences, set_locale_files, transactions_list):
     check_aux_data('helpers.sent', 'helpers.to', preferred_language, sender_tx_aux_data)
 
 
-@pytest.mark.parametrize("wei, expected_result", [
+@pytest.mark.parametrize("value, expected_result", [
     (50000000, Decimal('50.00')),
     (100000, Decimal('0.10'))
 ])
-def test_from_wei(wei, expected_result):
+def test_from_wei(cache_default_token_data, expected_result, value):
-    assert from_wei(wei) == expected_result
+    assert from_wei(value) == expected_result
 
 
 @pytest.mark.parametrize("value, expected_result", [
     (50, 50000000),
     (0.10, 100000)
 ])
-def test_to_wei(value, expected_result):
+def test_to_wei(cache_default_token_data, expected_result, value):
     assert to_wei(value) == expected_result
 
 
@@ -96,6 +96,7 @@ def test_validate_transaction_account(activated_account, init_database, transact
 @pytest.mark.parametrize("amount", [50, 0.10])
 def test_outgoing_transaction_processor(activated_account,
                                         amount,
+                                        cache_default_token_data,
                                         celery_session_worker,
                                         load_config,
                                         load_chain_spec,

apps/cic-ussd/tests/cic_ussd/processor/test_menu.py

@@ -1,5 +1,6 @@
 # standard imports
 import json
+import datetime
 
 # external imports
 from chainlib.hash import strip_0x
@@ -14,6 +15,7 @@ from cic_ussd.account.statement import (
 )
 from cic_ussd.account.tokens import get_default_token_symbol
 from cic_ussd.account.transaction import from_wei, to_wei
+from cic_ussd.cache import cache_data, cache_data_key
 from cic_ussd.menu.ussd_menu import UssdMenu
 from cic_ussd.metadata import PersonMetadata
 from cic_ussd.phone_number import Support
@@ -38,24 +40,34 @@ def test_menu_processor(activated_account,
                         load_chain_spec,
                         load_support_phone,
                         load_ussd_menu,
+                        mock_get_adjusted_balance,
                         mock_sync_balance_api_query,
                         mock_transaction_list_query,
                         valid_recipient):
     preferred_language = get_cached_preferred_language(activated_account.blockchain_address)
     available_balance = get_cached_available_balance(activated_account.blockchain_address)
     token_symbol = get_default_token_symbol()
-
+    with_available_balance = 'ussd.kenya.account_balances.available_balance'
-    tax = ''
+    with_fees = 'ussd.kenya.account_balances.with_fees'
-    bonus = ''
-    display_key = 'ussd.kenya.account_balances'
     ussd_menu = UssdMenu.find_by_name('account_balances')
     name = ussd_menu.get('name')
-    resp = response(activated_account, display_key, name, init_database, generic_ussd_session)
+    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
-    assert resp == translation_for(display_key,
+    assert resp == translation_for(with_available_balance,
                                    preferred_language,
                                    available_balance=available_balance,
+                                   token_symbol=token_symbol)
+
+    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    key = cache_data_key(identifier, ':cic.adjusted_balance')
+    adjusted_balance = 45931650.64654012
+    cache_data(key, json.dumps(adjusted_balance))
+    resp = response(activated_account, 'ussd.kenya.account_balances', name, init_database, generic_ussd_session)
+    tax_wei = to_wei(int(available_balance)) - int(adjusted_balance)
+    tax = from_wei(int(tax_wei))
+    assert resp == translation_for(key=with_fees,
+                                   preferred_language=preferred_language,
+                                   available_balance=available_balance,
                                    tax=tax,
-                                   bonus=bonus,
                                    token_symbol=token_symbol)
 
     cached_statement = get_cached_statement(activated_account.blockchain_address)
@@ -123,6 +135,15 @@ def test_menu_processor(activated_account,
                                    account_balance=available_balance,
                                    account_token_name=token_symbol)
 
+    display_key = 'ussd.kenya.start'
+    ussd_menu = UssdMenu.find_by_name('start')
+    name = ussd_menu.get('name')
+    older_timestamp = (activated_account.created - datetime.timedelta(days=35))
+    activated_account.created = older_timestamp
+    init_database.flush()
+    response(activated_account, display_key, name, init_database, generic_ussd_session)
+    assert mock_get_adjusted_balance['timestamp'] == int((datetime.datetime.now() - datetime.timedelta(days=30)).timestamp())
+
     display_key = 'ussd.kenya.transaction_pin_authorization'
     ussd_menu = UssdMenu.find_by_name('transaction_pin_authorization')
     name = ussd_menu.get('name')

apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py

@@ -49,6 +49,7 @@ def test_is_valid_transaction_amount(activated_account, amount, expected_result,
 ])
 def test_has_sufficient_balance(activated_account,
                                 cache_balances,
+                                cache_default_token_data,
                                 expected_result,
                                 generic_ussd_session,
                                 init_database,

apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py

@@ -121,6 +121,7 @@ def test_statement_callback(activated_account, mocker, transactions_list):
 def test_transaction_balances_callback(activated_account,
                                        balances,
                                        cache_balances,
+                                       cache_default_token_data,
                                        cache_person_metadata,
                                        cache_preferences,
                                        load_chain_spec,

apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py

@@ -13,7 +13,8 @@ from cic_ussd.translation import translation_for
 # tests imports
 
 
-def test_transaction(celery_session_worker,
+def test_transaction(cache_default_token_data,
+                     celery_session_worker,
                      load_support_phone,
                      mock_notifier_api,
                      notification_data,

apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py

@@ -30,10 +30,11 @@ def test_generate_statement(activated_account,
 
 
 def test_cache_statement(activated_account,
+                         cache_default_token_data,
                          cache_person_metadata,
+                         cache_preferences,
                          celery_session_worker,
                          init_database,
-                         preferences,
                          transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
@@ -41,7 +42,7 @@ def test_cache_statement(activated_account,
     cached_statement = get_cached_data(key)
     assert cached_statement is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
     result = s_parse_transaction.apply_async().get()
     s_cache_statement = celery.signature(
         'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address]
@@ -61,15 +62,15 @@ def test_cache_statement(activated_account,
 
 def test_parse_transaction(activated_account,
                            cache_person_metadata,
+                           cache_preferences,
                            celery_session_worker,
                            init_database,
-                           preferences,
                            transaction_result):
     recipient_transaction, sender_transaction = transaction_actors(transaction_result)
     assert sender_transaction.get('metadata_id') is None
     assert sender_transaction.get('phone_number') is None
     s_parse_transaction = celery.signature(
-        'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction])
+        'cic_ussd.tasks.processor.parse_transaction', [sender_transaction])
     result = s_parse_transaction.apply_async().get()
     assert result.get('metadata_id') == activated_account.standard_metadata_id()
     assert result.get('phone_number') == activated_account.phone_number

apps/cic-ussd/tests/fixtures/patches/account.py

@@ -41,6 +41,22 @@ def mock_async_balance_api_query(mocker):
     return query_args
 
 
+@pytest.fixture(scope='function')
+def mock_get_adjusted_balance(mocker, task_uuid):
+    query_args = {}
+
+    def get_adjusted_balance(self, token_symbol, balance, timestamp):
+        sync_res = mocker.patch('celery.result.AsyncResult')
+        sync_res.id = task_uuid
+        sync_res.result = 45931650.64654012
+        query_args['balance'] = balance
+        query_args['timestamp'] = timestamp
+        query_args['token_symbol'] = token_symbol
+        return sync_res
+    mocker.patch('cic_eth_aux.erc20_demurrage_token.api.Api.get_adjusted_balance', get_adjusted_balance)
+    return query_args
+
+
 @pytest.fixture(scope='function')
 def mock_notifier_api(mocker):
     sms = {}

apps/cic-ussd/var/lib/locale/ussd.en.yml

@@ -141,12 +141,22 @@ en:
         0. Back
       retry: |-
         %{retry_pin_entry}
-    account_balances: |-
+    account_balances:
-      CON Your balances are as follows:
+      available_balance: |-
-        balance: %{available_balance} %{token_symbol}
+        CON Your balances are as follows:
-        fees: %{tax} %{token_symbol}
+          balance: %{available_balance} %{token_symbol}
-        rewards: %{bonus} %{token_symbol}
+        0. Back
-      0. Back
+      with_fees: |-
+        CON Your balances are as follows:
+          balances: %{available_balance} %{token_symbol}
+          fees: %{tax} %{token_symbol}
+        0. Back
+      with_rewards: |-
+        CON Your balances are as follows:
+          balance: %{available_balance} %{token_symbol}
+          fees: %{tax} %{token_symbol}
+          rewards: %{bonus} %{token_symbol}
+        0. Back
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Next

apps/cic-ussd/var/lib/locale/ussd.sw.yml

@@ -140,12 +140,22 @@ sw:
         0. Nyuma
       retry: |-
         %{retry_pin_entry}
-    account_balances: |-
+    account_balances:
-      CON Salio zako ni zifuatazo:
+      available_balance: |-
-        salio: %{available_balance} %{token_symbol}
+        CON Salio zako ni zifuatazo:
-        ushuru: %{tax} %{token_symbol}
+          salio: %{available_balance} %{token_symbol}
-        tuzo: %{bonus} %{token_symbol}
+        0. Nyuma
-      0. Nyuma
+      with_fees: |-
+        CON Salio zako ni zifuatazo:
+          salio: %{available_balance} %{token_symbol}
+          ushuru: %{tax} %{token_symbol}
+        0. Nyuma
+      with_rewards: |-
+        CON Salio zako ni zifuatazo:
+          salio: %{available_balance} %{token_symbol}
+          ushuru: %{tax} %{token_symbol}
+          tuzo: %{bonus} %{token_symbol}
+        0. Nyuma
     first_transaction_set: |-
       CON %{first_transaction_set}
       1. Mbele