grassrootseconomics/cic-stack
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'philip/sms-cluster-issues' into 'master'

Browse Source 
Censors sensitive config values.

See merge request grassrootseconomics/cic-internal-integration!209
This commit is contained in:
Philip Wafula 2021-07-20 16:18:27 +00:00
commit 24e6db7d87
17 changed files with 363 additions and 2362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-notify/cic_notify/runnable/tasker.py
View File

@ -34,6 +34,8 @@ elif args.v:
config = confini.Config(args.c, args.env_prefix)
config.process()
config.add(args.q, '_CELERY_QUEUE', True)
config.censor('API_KEY', 'AFRICASTALKING')
config.censor('API_USERNAME', 'AFRICASTALKING')
config.censor('PASSWORD', 'DATABASE')
logg.debug('config loaded from {}:\n{}'.format(args.c, config))

5
apps/cic-notify/scripts/migrate.py
View File

@ -35,9 +35,10 @@ elif args.v:
config = confini.Config(args.c, args.env_prefix)
config.process()
config.censor('API_KEY', 'AFRICASTALKING')
config.censor('API_USERNAME', 'AFRICASTALKING')
config.censor('PASSWORD', 'DATABASE')
#config.censor('PASSWORD', 'SSL')
logg.debug('config:\n{}'.format(config))
logg.debug('config loaded from {}:\n{}'.format(args.c, config))
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
if not os.path.isdir(migrations_dir):

50
apps/cic-ussd/cic_ussd/balance.py
View File

@ -15,44 +15,46 @@ from cic_ussd.conversions import from_wei
logg = logging.getLogger()
class BalanceManager:
def __init__(self, address: str, chain_str: str, token_symbol: str):
"""
:param address: Ethereum address of account whose balance is being queried
:type address: str, 0x-hex
:param chain_str: The chain name and network id.
:type chain_str: str
:param token_symbol: ERC20 token symbol of whose balance is being queried
:type token_symbol: str
"""
self.address = address
self.chain_str = chain_str
self.token_symbol = token_symbol
def get_balances(self, asynchronous: bool = False) -> Union[celery.Task, dict]:
def get_balances(
address: str,
chain_str: str,
token_symbol: str,
asynchronous: bool = False,
callback_param: any = None,
callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Union[celery.Task, dict]:
"""
This function queries cic-eth for an account's balances, It provides a means to receive the balance either
asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a
dictionary containing network, outgoing and incoming balances.
:param asynchronous: Boolean value checking whether to return balances asynchronously
:param address: Ethereum address of the recipient
:type address: str, 0x-hex
:param chain_str: The chain name and network id.
:type chain_str: str
:param callback_param:
:type callback_param:
:param callback_task:
:type callback_task:
:param token_symbol: ERC20 token symbol of the account whose balance is being queried.
:type token_symbol: str
:param asynchronous: Boolean value checking whether to return balances asynchronously.
:type asynchronous: bool
:return:
:rtype:
"""
logg.debug(f'Retrieving balance for address: {address}')
if asynchronous:
cic_eth_api = Api(
chain_str=self.chain_str,
chain_str=chain_str,
callback_queue='cic-ussd',
callback_task='cic_ussd.tasks.callback_handler.process_balances_callback',
callback_param=''
callback_task=callback_task,
callback_param=callback_param
)
cic_eth_api.balance(address=self.address, token_symbol=self.token_symbol)
cic_eth_api.balance(address=address, token_symbol=token_symbol)
else:
cic_eth_api = Api(chain_str=self.chain_str)
cic_eth_api = Api(chain_str=chain_str)
balance_request_task = cic_eth_api.balance(
address=self.address,
token_symbol=self.token_symbol)
address=address,
token_symbol=token_symbol)
return balance_request_task.get()[0]

3
apps/cic-ussd/cic_ussd/error.py
View File

@ -48,3 +48,6 @@ class InitializationError(Exception):
pass
class UnknownUssdRecipient(Exception):
"""Raised when a recipient of a transaction is not known to the ussd application."""

9
apps/cic-ussd/cic_ussd/metadata/base.py
View File

@ -127,14 +127,19 @@ class MetadataRequestsHandler(Metadata):
if not isinstance(result_data, dict):
raise ValueError(f'Invalid result data object: {result_data}.')
if result.status_code == 200 and self.cic_type == ':cic.person':
if result.status_code == 200:
if self.cic_type == ':cic.person':
# validate person metadata
person = Person()
person_data = person.deserialize(person_data=result_data)
# format new person data for caching
data = json.dumps(person_data.serialize())
serialized_person_data = person_data.serialize()
data = json.dumps(serialized_person_data)
else:
data = json.dumps(result_data)
# cache metadata
cache_data(key=self.metadata_pointer, data=data)
logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
return result_data

1
apps/cic-ussd/cic_ussd/metadata/preferences.py
View File

@ -1,6 +1,7 @@
# standard imports
# external imports
import celery
# local imports
from .base import MetadataRequestsHandler

8
apps/cic-ussd/cic_ussd/processor.py
View File

@ -12,7 +12,7 @@ from tinydb.table import Document
# local imports
from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement
from cic_ussd.balance import BalanceManager, compute_operational_balance, get_cached_operational_balance
from cic_ussd.balance import compute_operational_balance, get_balances, get_cached_operational_balance
from cic_ussd.chain import Chain
from cic_ussd.db.models.account import Account
from cic_ussd.db.models.base import SessionBase
@ -182,7 +182,6 @@ def process_transaction_pin_authorization(account: Account, display_key: str, se
token_symbol = retrieve_token_symbol()
user_input = ussd_session.get('session_data').get('transaction_amount')
transaction_amount = to_wei(value=int(user_input))
logg.debug('Requires integration to determine user tokens.')
return process_pin_authorization(
account=account,
display_key=display_key,
@ -380,12 +379,9 @@ def process_start_menu(display_key: str, user: Account):
token_symbol = retrieve_token_symbol()
chain_str = Chain.spec.__str__()
blockchain_address = user.blockchain_address
balance_manager = BalanceManager(address=blockchain_address,
chain_str=chain_str,
token_symbol=token_symbol)
# get balances synchronously for display on start menu
balances_data = balance_manager.get_balances()
balances_data = get_balances(address=blockchain_address, chain_str=chain_str, token_symbol=token_symbol)
key = create_cached_data_key(
identifier=bytes.fromhex(blockchain_address[2:]),

12
apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py
View File

@ -8,13 +8,16 @@ import tempfile
import celery
import i18n
import redis
from chainlib.chain import ChainSpec
from confini import Config
# local imports
from cic_ussd.chain import Chain
from cic_ussd.db import dsn_from_config
from cic_ussd.db.models.base import SessionBase
from cic_ussd.metadata.signer import Signer
from cic_ussd.metadata.base import Metadata
from cic_ussd.phone_number import Support
from cic_ussd.redis import InMemoryStore
from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession
from cic_ussd.validator import validate_presence
@ -82,6 +85,15 @@ Signer.key_file_path = key_file_path
i18n.load_path.append(config.get('APP_LOCALE_PATH'))
i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
chain_spec = ChainSpec(
common_name=config.get('CIC_COMMON_NAME'),
engine=config.get('CIC_ENGINE'),
network_id=config.get('CIC_NETWORK_ID')
)
Chain.spec = chain_spec
Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER')
# set up celery
current_app = celery.Celery(__name__)

6
apps/cic-ussd/cic_ussd/tasks/__init__.py
View File

@ -1,8 +1,4 @@
# standard import
import os
import logging
import urllib
import json
# third-party imports
# this must be included for the package to be recognized as a tasks package
@ -14,3 +10,5 @@ from .logger import *
from .ussd_session import *
from .callback_handler import *
from .metadata import *
from .notifications import *
from .processor import *

138
apps/cic-ussd/cic_ussd/tasks/callback_handler.py
View File

@ -7,14 +7,14 @@ from datetime import datetime, timedelta
import celery
# local imports
from cic_ussd.balance import compute_operational_balance, get_balances
from cic_ussd.chain import Chain
from cic_ussd.conversions import from_wei
from cic_ussd.db.models.base import SessionBase
from cic_ussd.db.models.account import Account
from cic_ussd.account import define_account_tx_metadata
from cic_ussd.error import ActionDataNotFoundError
from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key
from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key, get_cached_data
from cic_ussd.tasks.base import CriticalSQLAlchemyTask
from cic_ussd.transactions import IncomingTransactionProcessor
logg = logging.getLogger(__file__)
celery_app = celery.current_app
@ -87,59 +87,106 @@ def process_account_creation_callback(self, result: str, url: str, status_code:
session.close()
@celery_app.task
def process_incoming_transfer_callback(result: dict, param: str, status_code: int):
session = SessionBase.create_session()
@celery_app.task(bind=True)
def process_transaction_callback(self, result: dict, param: str, status_code: int):
if status_code == 0:
chain_str = Chain.spec.__str__()
# collect result data
# collect transaction metadata
destination_token_symbol = result.get('destination_token_symbol')
destination_token_value = result.get('destination_token_value')
recipient_blockchain_address = result.get('recipient')
sender_blockchain_address = result.get('sender')
token_symbol = result.get('destination_token_symbol')
value = result.get('destination_token_value')
source_token_symbol = result.get('source_token_symbol')
source_token_value = result.get('source_token_value')
# try to find users in system
recipient_user = session.query(Account).filter_by(blockchain_address=recipient_blockchain_address).first()
sender_user = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first()
# build stakeholder callback params
recipient_metadata = {
"token_symbol": destination_token_symbol,
"token_value": destination_token_value,
"blockchain_address": recipient_blockchain_address,
"tag": "recipient",
"tx_param": param
}
# check whether recipient is in the system
if not recipient_user:
session.close()
raise ValueError(
f'Tx for recipient: {recipient_blockchain_address} was received but has no matching user in the system.'
)
# retrieve account balances
get_balances(
address=recipient_blockchain_address,
callback_param=recipient_metadata,
chain_str=chain_str,
callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
token_symbol=destination_token_symbol,
asynchronous=True)
# process incoming transactions
incoming_tx_processor = IncomingTransactionProcessor(phone_number=recipient_user.phone_number,
preferred_language=recipient_user.preferred_language,
token_symbol=token_symbol,
value=value)
# only retrieve sender if transaction is a transfer
if param == 'transfer':
sender_metadata = {
"blockchain_address": sender_blockchain_address,
"token_symbol": source_token_symbol,
"token_value": source_token_value,
"tag": "sender",
"tx_param": param
}
if param == 'tokengift':
incoming_tx_processor.process_token_gift_incoming_transactions()
elif param == 'transfer':
if sender_user:
sender_information = define_account_tx_metadata(user=sender_user)
incoming_tx_processor.process_transfer_incoming_transaction(
sender_information=sender_information,
recipient_blockchain_address=recipient_blockchain_address
)
get_balances(
address=sender_blockchain_address,
callback_param=sender_metadata,
chain_str=chain_str,
callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback',
token_symbol=source_token_symbol,
asynchronous=True)
else:
raise ValueError(f'Unexpected status code: {status_code}.')
@celery_app.task(bind=True)
def process_transaction_balances_callback(self, result: list, param: dict, status_code: int):
queue = self.request.delivery_info.get('routing_key')
if status_code == 0:
# retrieve balance data
balances_data = result[0]
operational_balance = compute_operational_balance(balances=balances_data)
# retrieve account's address
blockchain_address = param.get('blockchain_address')
# append balance to transaction metadata
transaction_metadata = param
transaction_metadata['operational_balance'] = operational_balance
# retrieve account's preferences
s_preferences_metadata = celery.signature(
'cic_ussd.tasks.metadata.query_preferences_metadata',
[blockchain_address],
queue=queue
)
# parse metadata and run validations
s_process_account_metadata = celery.signature(
'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
[transaction_metadata],
queue=queue
)
# issue notification of transaction
s_notify_account = celery.signature(
'cic_ussd.tasks.notifications.notify_account_of_transaction',
queue=queue
)
if param.get('tx_param') == 'transfer':
celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async()
if param.get('tx_param') == 'tokengift':
s_process_account_metadata = celery.signature(
'cic_ussd.tasks.processor.process_tx_metadata_for_notification',
[{}, transaction_metadata],
queue=queue
)
celery.chain(s_process_account_metadata, s_notify_account).apply_async()
else:
logg.warning(
f'Tx with sender: {sender_blockchain_address} was received but has no matching user in the system.'
)
incoming_tx_processor.process_transfer_incoming_transaction(
sender_information='GRASSROOTS ECONOMICS',
recipient_blockchain_address=recipient_blockchain_address
)
else:
session.close()
raise ValueError(f'Unexpected transaction param: {param}.')
else:
session.close()
raise ValueError(f'Unexpected status code: {status_code}.')
session.close()
@celery_app.task
def process_balances_callback(result: list, param: str, status_code: int):
@ -151,6 +198,7 @@ def process_balances_callback(result: list, param: str, status_code: int):
salt=':cic.balances_data'
)
cache_data(key=key, data=json.dumps(balances_data))
logg.debug(f'caching: {balances_data} with key: {key}')
else:
raise ValueError(f'Unexpected status code: {status_code}.')

13
apps/cic-ussd/cic_ussd/tasks/metadata.py
View File

@ -26,6 +26,7 @@ def query_person_metadata(blockchain_address: str):
:rtype:
"""
identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
logg.debug(f'Retrieving person metadata for address: {blockchain_address}.')
person_metadata_client = PersonMetadata(identifier=identifier)
person_metadata_client.query()
@ -72,3 +73,15 @@ def add_preferences_metadata(blockchain_address: str, data: dict):
identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
custom_metadata_client = PreferencesMetadata(identifier=identifier)
custom_metadata_client.create(data=data)
@celery_app.task()
def query_preferences_metadata(blockchain_address: str):
"""This method retrieves preferences metadata based on an account's blockchain address.
:param blockchain_address: Blockchain address of an account.
:type blockchain_address: str | Ox-hex
"""
identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address)
logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
person_metadata_client = PreferencesMetadata(identifier=identifier)
return person_metadata_client.query()

70
apps/cic-ussd/cic_ussd/tasks/notifications.py Normal file
View File

@ -0,0 +1,70 @@
# standard imports
import datetime
import logging
# third-party imports
import celery
# local imports
from cic_ussd.notifications import Notifier
from cic_ussd.phone_number import Support
celery_app = celery.current_app
logg = logging.getLogger(__file__)
notifier = Notifier()
@celery_app.task
def notify_account_of_transaction(notification_data: dict):
"""
:param notification_data:
:type notification_data:
:return:
:rtype:
"""
account_tx_role = notification_data.get('account_tx_role')
amount = notification_data.get('amount')
balance = notification_data.get('balance')
phone_number = notification_data.get('phone_number')
preferred_language = notification_data.get('preferred_language')
token_symbol = notification_data.get('token_symbol')
transaction_account_metadata = notification_data.get('transaction_account_metadata')
transaction_type = notification_data.get('transaction_type')
timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p')
if transaction_type == 'tokengift':
support_phone = Support.phone_number
notifier.send_sms_notification(
key='sms.account_successfully_created',
phone_number=phone_number,
preferred_language=preferred_language,
balance=balance,
support_phone=support_phone,
token_symbol=token_symbol
)
if transaction_type == 'transfer':
if account_tx_role == 'recipient':
notifier.send_sms_notification(
key='sms.received_tokens',
phone_number=phone_number,
preferred_language=preferred_language,
amount=amount,
token_symbol=token_symbol,
tx_sender_information=transaction_account_metadata,
timestamp=timestamp,
balance=balance
)
else:
notifier.send_sms_notification(
key='sms.sent_tokens',
phone_number=phone_number,
preferred_language=preferred_language,
amount=amount,
token_symbol=token_symbol,
tx_recipient_information=transaction_account_metadata,
timestamp=timestamp,
balance=balance
)

88
apps/cic-ussd/cic_ussd/tasks/processor.py Normal file
View File

@ -0,0 +1,88 @@
# standard imports
import logging
# third-party imports
import celery
from i18n import config
# local imports
from cic_ussd.account import define_account_tx_metadata
from cic_ussd.db.models.account import Account
from cic_ussd.db.models.base import SessionBase
from cic_ussd.error import UnknownUssdRecipient
from cic_ussd.transactions import from_wei
celery_app = celery.current_app
logg = logging.getLogger(__file__)
@celery_app.task
def process_tx_metadata_for_notification(result: celery.Task, transaction_metadata: dict):
"""
:param result:
:type result:
:param transaction_metadata:
:type transaction_metadata:
:return:
:rtype:
"""
notification_data = {}
# get preferred language
preferred_language = result.get('preferred_language')
if not preferred_language:
preferred_language = config.get('fallback')
notification_data['preferred_language'] = preferred_language
# validate account information against present ussd storage data.
session = SessionBase.create_session()
blockchain_address = transaction_metadata.get('blockchain_address')
tag = transaction_metadata.get('tag')
account = session.query(Account).filter_by(blockchain_address=blockchain_address).first()
if not account and tag == 'recipient':
session.close()
raise UnknownUssdRecipient(
f'Tx for recipient: {blockchain_address} was received but has no matching user in the system.'
)
# get phone number associated with account
phone_number = account.phone_number
notification_data['phone_number'] = phone_number
# get account's role in transaction i.e sender / recipient
tx_param = transaction_metadata.get('tx_param')
notification_data['transaction_type'] = tx_param
# get token amount and symbol
if tag == 'recipient':
account_tx_role = tag
amount = transaction_metadata.get('token_value')
amount = from_wei(value=amount)
token_symbol = transaction_metadata.get('token_symbol')
else:
account_tx_role = tag
amount = transaction_metadata.get('token_value')
amount = from_wei(value=amount)
token_symbol = transaction_metadata.get('token_symbol')
notification_data['account_tx_role'] = account_tx_role
notification_data['amount'] = amount
notification_data['token_symbol'] = token_symbol
# get account's standard ussd identification pattern
if tx_param == 'transfer':
tx_account_metadata = define_account_tx_metadata(user=account)
notification_data['transaction_account_metadata'] = tx_account_metadata
if tag == 'recipient':
notification_data['notification_key'] = 'sms.received_tokens'
else:
notification_data['notification_key'] = 'sms.sent_tokens'
if tx_param == 'tokengift':
notification_data['notification_key'] = 'sms.account_successfully_created'
# get account's balance
notification_data['balance'] = transaction_metadata.get('operational_balance')
return notification_data

60
apps/cic-ussd/cic_ussd/transactions.py
View File

@ -7,9 +7,9 @@ from datetime import datetime
from cic_eth.api import Api
# local imports
from cic_ussd.balance import get_cached_operational_balance
from cic_ussd.balance import get_balances, get_cached_operational_balance
from cic_ussd.notifications import Notifier
from cic_ussd.phone_number import Support
logg = logging.getLogger()
notifier = Notifier()
@ -50,61 +50,6 @@ def to_wei(value: int) -> int:
return int(value * 1e+6)
class IncomingTransactionProcessor:
def __init__(self, phone_number: str, preferred_language: str, token_symbol: str, value: int):
"""
:param phone_number: The recipient's phone number.
:type phone_number: str
:param preferred_language: The user's preferred language.
:type preferred_language: str
:param token_symbol: The symbol for the token the recipient receives.
:type token_symbol: str
:param value: The amount of tokens received in the transactions.
:type value: int
"""
self.phone_number = phone_number
self.preferred_language = preferred_language
self.token_symbol = token_symbol
self.value = value
def process_token_gift_incoming_transactions(self):
"""This function processes incoming transactions with a "tokengift" param, it collects all appropriate data to
send out notifications to users when their accounts are successfully created.
"""
balance = from_wei(value=self.value)
key = 'sms.account_successfully_created'
notifier.send_sms_notification(key=key,
phone_number=self.phone_number,
preferred_language=self.preferred_language,
balance=balance,
token_symbol=self.token_symbol)
def process_transfer_incoming_transaction(self, sender_information: str, recipient_blockchain_address: str):
"""This function processes incoming transactions with the "transfer" param and issues notifications to users
about reception of funds into their accounts.
:param sender_information: A string with a user's full name and phone number.
:type sender_information: str
:param recipient_blockchain_address:
type recipient_blockchain_address: str
"""
key = 'sms.received_tokens'
amount = from_wei(value=self.value)
timestamp = datetime.now().strftime('%d-%m-%y, %H:%M %p')
operational_balance = get_cached_operational_balance(blockchain_address=recipient_blockchain_address)
notifier.send_sms_notification(key=key,
phone_number=self.phone_number,
preferred_language=self.preferred_language,
amount=amount,
token_symbol=self.token_symbol,
tx_sender_information=sender_information,
timestamp=timestamp,
balance=operational_balance)
class OutgoingTransactionProcessor:
def __init__(self, chain_str: str, from_address: str, to_address: str):
@ -116,6 +61,7 @@ class OutgoingTransactionProcessor:
:param to_address: Ethereum address of the recipient
:type to_address: str, 0x-hex
"""
self.chain_str = chain_str
self.cic_eth_api = Api(chain_str=chain_str)
self.from_address = from_address
self.to_address = to_address

2
apps/cic-ussd/var/lib/locale/sms.en.yml
View File

@ -3,5 +3,7 @@ en:
You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
received_tokens: |-
Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
sent_tokens: |-
Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
terms: |-
By using the service, you agree to the terms and conditions at http://grassecon.org/tos

4
apps/cic-ussd/var/lib/locale/sms.sw.yml
View File

@ -2,6 +2,8 @@ sw:
account_successfully_created: |-
Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
received_tokens: |-
Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
sent_tokens: |-
Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}.
terms: |-
Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

2210
apps/data-seeding/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff