From 0672a17d2e38167dd54f3a98c40a6dd869688a3e Mon Sep 17 00:00:00 2001 From: Philip Wafula Date: Fri, 6 Aug 2021 16:29:01 +0000 Subject: [PATCH 1/6] The great bump --- apps/cic-ussd/.config/app.ini | 25 - apps/cic-ussd/.config/database.ini | 10 - apps/cic-ussd/.config/redis.ini | 9 - apps/cic-ussd/.config/test/app.ini | 15 - apps/cic-ussd/.config/test/database.ini | 8 - apps/cic-ussd/.config/test/pgp.ini | 5 - apps/cic-ussd/.config/test/redis.ini | 9 - apps/cic-ussd/.gitlab-ci.yml | 2 +- apps/cic-ussd/cic_ussd/account.py | 49 -- apps/cic-ussd/cic_ussd/account/__init__.py | 0 apps/cic-ussd/cic_ussd/account/balance.py | 90 +++ apps/cic-ussd/cic_ussd/{ => account}/chain.py | 0 apps/cic-ussd/cic_ussd/account/maps.py | 20 + apps/cic-ussd/cic_ussd/account/metadata.py | 44 ++ apps/cic-ussd/cic_ussd/account/statement.py | 111 ++++ apps/cic-ussd/cic_ussd/account/tokens.py | 61 ++ apps/cic-ussd/cic_ussd/account/transaction.py | 172 ++++++ apps/cic-ussd/cic_ussd/balance.py | 92 --- apps/cic-ussd/cic_ussd/{redis.py => cache.py} | 11 +- apps/cic-ussd/cic_ussd/conversions.py | 41 -- .../default/versions/2a329190a9af_.py | 2 +- .../default/versions/f289e8510444_.py | 2 +- apps/cic-ussd/cic_ussd/db/models/account.py | 172 ++++-- apps/cic-ussd/cic_ussd/db/models/base.py | 48 +- .../cic_ussd/db/models/task_tracker.py | 15 + .../cic_ussd/db/models/ussd_session.py | 61 +- apps/cic-ussd/cic_ussd/error.py | 14 +- apps/cic-ussd/cic_ussd/http/__init__.py | 0 apps/cic-ussd/cic_ussd/http/requests.py | 65 ++ apps/cic-ussd/cic_ussd/http/responses.py | 26 + apps/cic-ussd/cic_ussd/http/routes.py | 87 +++ apps/cic-ussd/cic_ussd/menu/ussd_menu.py | 7 +- apps/cic-ussd/cic_ussd/metadata/__init__.py | 48 +- apps/cic-ussd/cic_ussd/metadata/base.py | 80 +-- apps/cic-ussd/cic_ussd/metadata/person.py | 2 +- apps/cic-ussd/cic_ussd/metadata/signer.py | 6 +- apps/cic-ussd/cic_ussd/notifications.py | 5 +- apps/cic-ussd/cic_ussd/operations.py | 521 ---------------- apps/cic-ussd/cic_ussd/phone_number.py | 5 +- apps/cic-ussd/cic_ussd/processor.py | 562 ------------------ apps/cic-ussd/cic_ussd/processor/__init__.py | 0 apps/cic-ussd/cic_ussd/processor/menu.py | 305 ++++++++++ apps/cic-ussd/cic_ussd/processor/ussd.py | 185 ++++++ apps/cic-ussd/cic_ussd/processor/util.py | 77 +++ apps/cic-ussd/cic_ussd/requests.py | 143 ----- .../runnable/daemons/cic_user_server.py | 28 +- .../runnable/daemons/cic_user_tasker.py | 34 +- .../runnable/daemons/cic_user_ussd_server.py | 84 ++- .../cic-ussd/cic_ussd/session/ussd_session.py | 203 ++++++- .../cic_ussd/state_machine/logic/account.py | 248 ++++++++ .../cic_ussd/state_machine/logic/balance.py | 21 - .../cic_ussd/state_machine/logic/menu.py | 35 +- .../cic_ussd/state_machine/logic/pin.py | 57 +- .../cic_ussd/state_machine/logic/sms.py | 37 +- .../state_machine/logic/transaction.py | 95 ++- .../cic_ussd/state_machine/logic/user.py | 292 --------- .../cic_ussd/state_machine/logic/validator.py | 45 +- apps/cic-ussd/cic_ussd/tasks/__init__.py | 7 +- apps/cic-ussd/cic_ussd/tasks/base.py | 3 + .../cic_ussd/tasks/callback_handler.py | 438 ++++++-------- apps/cic-ussd/cic_ussd/tasks/logger.py | 11 - apps/cic-ussd/cic_ussd/tasks/metadata.py | 23 +- apps/cic-ussd/cic_ussd/tasks/notifications.py | 59 +- apps/cic-ussd/cic_ussd/tasks/processor.py | 126 ++-- apps/cic-ussd/cic_ussd/tasks/ussd_session.py | 54 +- apps/cic-ussd/cic_ussd/transactions.py | 79 --- apps/cic-ussd/cic_ussd/validator.py | 42 +- apps/cic-ussd/config/app.ini | 13 + apps/cic-ussd/config/celery.ini | 3 + apps/cic-ussd/{.config => config}/cic.ini | 0 .../{.config_client => config/client}/app.ini | 0 .../client}/client.ini | 0 .../client}/ussd.ini | 0 apps/cic-ussd/config/database.ini | 10 + apps/cic-ussd/{.config => config}/pgp.ini | 0 apps/cic-ussd/config/phone.ini | 5 + apps/cic-ussd/{.config => config}/pip.ini | 0 apps/cic-ussd/config/redis.ini | 5 + apps/cic-ussd/config/test/app.ini | 8 + apps/cic-ussd/config/test/celery.ini | 3 + .../cic-ussd/{.config => config}/test/cic.ini | 2 +- apps/cic-ussd/config/test/database.ini | 9 + .../{.config => config}/test/integration.ini | 0 apps/cic-ussd/config/test/pgp.ini | 5 + apps/cic-ussd/config/test/phone.ini | 5 + .../cic-ussd/{.config => config}/test/pip.ini | 0 apps/cic-ussd/config/test/redis.ini | 5 + apps/cic-ussd/config/test/translations.ini | 3 + apps/cic-ussd/config/test/ussd.ini | 5 + apps/cic-ussd/config/translations.ini | 3 + apps/cic-ussd/config/ussd.ini | 5 + apps/cic-ussd/docker/Dockerfile | 4 +- apps/cic-ussd/docker/Dockerfile_ci | 4 +- apps/cic-ussd/requirements.txt | 20 +- apps/cic-ussd/scripts/migrate.py | 3 +- apps/cic-ussd/setup.cfg | 3 + apps/cic-ussd/states/signup_states.json | 3 +- .../tests/cic_ussd/account/test_balance.py | 68 +++ .../tests/cic_ussd/account/test_maps.py | 28 + .../tests/cic_ussd/account/test_metadata.py | 28 + .../tests/cic_ussd/account/test_statement.py | 85 +++ .../tests/cic_ussd/account/test_tokens.py | 42 ++ .../cic_ussd/account/test_transaction.py | 110 ++++ .../tests/cic_ussd/db/models/test_account.py | 96 +++ .../cic_ussd/db/models/test_task_tracker.py | 7 +- .../tests/cic_ussd/db/models/test_user.py | 40 -- .../cic_ussd/db/models/test_ussd_session.py | 66 +- apps/cic-ussd/tests/cic_ussd/db/test_db.py | 18 +- .../tests/cic_ussd/files/test_local_files.py | 11 +- .../tests/cic_ussd/http/test_requests.py | 69 +++ .../tests/cic_ussd/http/test_responses.py | 18 + .../tests/cic_ussd/http/test_routes.py | 45 ++ .../tests/cic_ussd/menu/test_ussd_menu.py | 5 +- .../tests/cic_ussd/metadata/test_base.py | 44 ++ .../tests/cic_ussd/metadata/test_custom.py | 22 + .../tests/cic_ussd/metadata/test_metadata.py | 80 --- .../tests/cic_ussd/metadata/test_person.py | 22 + .../tests/cic_ussd/metadata/test_phone.py | 23 + .../cic_ussd/metadata/test_preferences.py | 22 + .../tests/cic_ussd/metadata/test_signer.py | 17 - .../cic_ussd/metadata/test_user_metadata.py | 123 ---- .../tests/cic_ussd/processor/test_menu.py | 183 ++++++ .../tests/cic_ussd/processor/test_ussd.py | 121 ++++ .../tests/cic_ussd/processor/test_util.py | 62 ++ .../session/test_cached_ussd_session.py | 72 +++ .../session/test_in_memory_ussd_session.py | 11 - .../state_machine/logic/test_account_logic.py | 154 +++++ .../cic_ussd/state_machine/logic/test_menu.py | 21 - .../state_machine/logic/test_menu_logic.py | 37 ++ .../cic_ussd/state_machine/logic/test_pin.py | 94 --- .../state_machine/logic/test_pin_logic.py | 101 ++++ .../cic_ussd/state_machine/logic/test_sms.py | 36 -- .../state_machine/logic/test_sms_logic.py | 42 ++ .../state_machine/logic/test_transaction.py | 111 ---- .../logic/test_transaction_logic.py | 112 ++++ .../state_machine/logic/test_user_logic.py | 155 ----- .../logic/test_validator_logic.py | 69 ++- .../state_machine/test_state_machine.py | 15 +- .../cic_ussd/tasks/test_callback_handler.py | 178 ++++++ .../tasks/test_callback_handler_tasks.py | 203 ------- .../cic_ussd/tasks/test_metadata_tasks.py | 52 ++ .../tasks/test_notifications_tasks.py | 71 +++ .../tasks/test_persist_session_to_db_tasks.py | 46 -- .../cic_ussd/tasks/test_processor_tasks.py | 75 +++ .../cic_ussd/tasks/test_ussd_session_tasks.py | 33 + apps/cic-ussd/tests/cic_ussd/test_accounts.py | 20 - apps/cic-ussd/tests/cic_ussd/test_cache.py | 33 + .../{test_encoder.py => test_encode.py} | 0 .../tests/cic_ussd/test_notifications.py | 14 +- .../tests/cic_ussd/test_operations.py | 243 -------- .../tests/cic_ussd/test_phone_number.py | 18 + .../cic-ussd/tests/cic_ussd/test_processor.py | 130 ---- apps/cic-ussd/tests/cic_ussd/test_requests.py | 65 -- .../tests/cic_ussd/test_transactions.py | 35 -- .../tests/cic_ussd/test_translation.py | 6 + .../cic-ussd/tests/cic_ussd/test_validator.py | 21 +- apps/cic-ussd/tests/conftest.py | 29 +- .../tests/data/pgp/privatekeys_meta.asc | 241 ++++++++ apps/cic-ussd/tests/fixtures/account.py | 169 ++++++ apps/cic-ussd/tests/fixtures/cache.py | 15 + apps/cic-ussd/tests/fixtures/callback.py | 115 ---- apps/cic-ussd/tests/fixtures/config.py | 130 ++-- apps/cic-ussd/tests/fixtures/db.py | 37 +- apps/cic-ussd/tests/fixtures/metadata.py | 53 ++ apps/cic-ussd/tests/fixtures/mocks.py | 95 --- .../tests/fixtures/patches/__init__.py | 0 .../tests/fixtures/patches/account.py | 104 ++++ apps/cic-ussd/tests/fixtures/redis.py | 11 - apps/cic-ussd/tests/fixtures/requests.py | 20 - .../tests/fixtures/{celery.py => tasker.py} | 17 +- apps/cic-ussd/tests/fixtures/transaction.py | 148 +++++ apps/cic-ussd/tests/fixtures/user.py | 120 ---- apps/cic-ussd/tests/fixtures/ussd_session.py | 75 ++- apps/cic-ussd/tests/fixtures/util.py | 63 ++ apps/cic-ussd/tests/helpers/accounts.py | 6 + apps/cic-ussd/tests/integration/README.md | 2 +- apps/cic-ussd/tests/integration/run.sh | 2 +- .../test_account_creation.tavern.yaml | 25 +- .../test_account_management.tavern.yaml | 14 +- .../transitions/age_setting_transitions.json | 10 +- .../transitions/exit_transitions.json | 3 +- .../gender_setting_transitions.json | 10 +- .../language_setting_transitions.json | 4 +- .../location_setting_transitions.json | 10 +- .../transitions/name_setting_transitions.json | 12 +- .../products_setting_transitions.json | 12 +- .../transitions/signup_transitions.json | 8 +- .../transitions/transaction_transitions.json | 5 +- apps/cic-ussd/var/lib/locale/helpers.en.yml | 19 + apps/cic-ussd/var/lib/locale/helpers.sw.yml | 19 + apps/cic-ussd/var/lib/locale/sms.en.yml | 2 + apps/cic-ussd/var/lib/locale/sms.sw.yml | 4 +- apps/cic-ussd/var/lib/locale/ussd.en.yml | 37 +- apps/cic-ussd/var/lib/locale/ussd.sw.yml | 39 +- apps/data-seeding/cic_ussd/import_task.py | 2 +- 195 files changed, 5791 insertions(+), 4983 deletions(-) delete mode 100644 apps/cic-ussd/.config/app.ini delete mode 100644 apps/cic-ussd/.config/database.ini delete mode 100644 apps/cic-ussd/.config/redis.ini delete mode 100644 apps/cic-ussd/.config/test/app.ini delete mode 100644 apps/cic-ussd/.config/test/database.ini delete mode 100644 apps/cic-ussd/.config/test/pgp.ini delete mode 100644 apps/cic-ussd/.config/test/redis.ini delete mode 100644 apps/cic-ussd/cic_ussd/account.py create mode 100644 apps/cic-ussd/cic_ussd/account/__init__.py create mode 100644 apps/cic-ussd/cic_ussd/account/balance.py rename apps/cic-ussd/cic_ussd/{ => account}/chain.py (100%) create mode 100644 apps/cic-ussd/cic_ussd/account/maps.py create mode 100644 apps/cic-ussd/cic_ussd/account/metadata.py create mode 100644 apps/cic-ussd/cic_ussd/account/statement.py create mode 100644 apps/cic-ussd/cic_ussd/account/tokens.py create mode 100644 apps/cic-ussd/cic_ussd/account/transaction.py delete mode 100644 apps/cic-ussd/cic_ussd/balance.py rename apps/cic-ussd/cic_ussd/{redis.py => cache.py} (80%) delete mode 100644 apps/cic-ussd/cic_ussd/conversions.py create mode 100644 apps/cic-ussd/cic_ussd/http/__init__.py create mode 100644 apps/cic-ussd/cic_ussd/http/requests.py create mode 100644 apps/cic-ussd/cic_ussd/http/responses.py create mode 100644 apps/cic-ussd/cic_ussd/http/routes.py delete mode 100644 apps/cic-ussd/cic_ussd/operations.py delete mode 100644 apps/cic-ussd/cic_ussd/processor.py create mode 100644 apps/cic-ussd/cic_ussd/processor/__init__.py create mode 100644 apps/cic-ussd/cic_ussd/processor/menu.py create mode 100644 apps/cic-ussd/cic_ussd/processor/ussd.py create mode 100644 apps/cic-ussd/cic_ussd/processor/util.py delete mode 100644 apps/cic-ussd/cic_ussd/requests.py create mode 100644 apps/cic-ussd/cic_ussd/state_machine/logic/account.py delete mode 100644 apps/cic-ussd/cic_ussd/state_machine/logic/balance.py delete mode 100644 apps/cic-ussd/cic_ussd/state_machine/logic/user.py delete mode 100644 apps/cic-ussd/cic_ussd/tasks/logger.py delete mode 100644 apps/cic-ussd/cic_ussd/transactions.py create mode 100644 apps/cic-ussd/config/app.ini create mode 100644 apps/cic-ussd/config/celery.ini rename apps/cic-ussd/{.config => config}/cic.ini (100%) rename apps/cic-ussd/{.config_client => config/client}/app.ini (100%) rename apps/cic-ussd/{.config_client => config/client}/client.ini (100%) rename apps/cic-ussd/{.config_client => config/client}/ussd.ini (100%) create mode 100644 apps/cic-ussd/config/database.ini rename apps/cic-ussd/{.config => config}/pgp.ini (100%) create mode 100644 apps/cic-ussd/config/phone.ini rename apps/cic-ussd/{.config => config}/pip.ini (100%) create mode 100644 apps/cic-ussd/config/redis.ini create mode 100644 apps/cic-ussd/config/test/app.ini create mode 100644 apps/cic-ussd/config/test/celery.ini rename apps/cic-ussd/{.config => config}/test/cic.ini (63%) create mode 100644 apps/cic-ussd/config/test/database.ini rename apps/cic-ussd/{.config => config}/test/integration.ini (100%) create mode 100644 apps/cic-ussd/config/test/pgp.ini create mode 100644 apps/cic-ussd/config/test/phone.ini rename apps/cic-ussd/{.config => config}/test/pip.ini (100%) create mode 100644 apps/cic-ussd/config/test/redis.ini create mode 100644 apps/cic-ussd/config/test/translations.ini create mode 100644 apps/cic-ussd/config/test/ussd.ini create mode 100644 apps/cic-ussd/config/translations.ini create mode 100644 apps/cic-ussd/config/ussd.ini create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_balance.py create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_maps.py create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_metadata.py create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_statement.py create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_tokens.py create mode 100644 apps/cic-ussd/tests/cic_ussd/account/test_transaction.py create mode 100644 apps/cic-ussd/tests/cic_ussd/db/models/test_account.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/db/models/test_user.py create mode 100644 apps/cic-ussd/tests/cic_ussd/http/test_requests.py create mode 100644 apps/cic-ussd/tests/cic_ussd/http/test_responses.py create mode 100644 apps/cic-ussd/tests/cic_ussd/http/test_routes.py create mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_base.py create mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_metadata.py create mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_person.py create mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py create mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py create mode 100644 apps/cic-ussd/tests/cic_ussd/processor/test_menu.py create mode 100644 apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py create mode 100644 apps/cic-ussd/tests/cic_ussd/processor/test_util.py create mode 100644 apps/cic-ussd/tests/cic_ussd/session/test_cached_ussd_session.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/session/test_in_memory_ussd_session.py create mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_account_logic.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu.py create mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu_logic.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin.py create mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin_logic.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms.py create mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms_logic.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction.py create mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py create mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler_tasks.py create mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py create mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_persist_session_to_db_tasks.py create mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py create mode 100644 apps/cic-ussd/tests/cic_ussd/tasks/test_ussd_session_tasks.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/test_accounts.py create mode 100644 apps/cic-ussd/tests/cic_ussd/test_cache.py rename apps/cic-ussd/tests/cic_ussd/{test_encoder.py => test_encode.py} (100%) delete mode 100644 apps/cic-ussd/tests/cic_ussd/test_operations.py create mode 100644 apps/cic-ussd/tests/cic_ussd/test_phone_number.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/test_processor.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/test_requests.py delete mode 100644 apps/cic-ussd/tests/cic_ussd/test_transactions.py create mode 100644 apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc create mode 100644 apps/cic-ussd/tests/fixtures/account.py create mode 100644 apps/cic-ussd/tests/fixtures/cache.py delete mode 100644 apps/cic-ussd/tests/fixtures/callback.py create mode 100644 apps/cic-ussd/tests/fixtures/metadata.py delete mode 100644 apps/cic-ussd/tests/fixtures/mocks.py create mode 100644 apps/cic-ussd/tests/fixtures/patches/__init__.py create mode 100644 apps/cic-ussd/tests/fixtures/patches/account.py delete mode 100644 apps/cic-ussd/tests/fixtures/redis.py delete mode 100644 apps/cic-ussd/tests/fixtures/requests.py rename apps/cic-ussd/tests/fixtures/{celery.py => tasker.py} (76%) create mode 100644 apps/cic-ussd/tests/fixtures/transaction.py delete mode 100644 apps/cic-ussd/tests/fixtures/user.py create mode 100644 apps/cic-ussd/tests/fixtures/util.py create mode 100644 apps/cic-ussd/var/lib/locale/helpers.en.yml create mode 100644 apps/cic-ussd/var/lib/locale/helpers.sw.yml diff --git a/apps/cic-ussd/.config/app.ini b/apps/cic-ussd/.config/app.ini deleted file mode 100644 index 75c73d38..00000000 --- a/apps/cic-ussd/.config/app.ini +++ /dev/null @@ -1,25 +0,0 @@ -[app] -ALLOWED_IP=0.0.0.0/0 -LOCALE_FALLBACK=sw -LOCALE_PATH=var/lib/locale/ -MAX_BODY_LENGTH=1024 -PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I= -SERVICE_CODE=*483*46#,*483*061#,*384*96# -SUPPORT_PHONE_NUMBER=0757628885 - -[phone_number] -REGION=KE - -[ussd] -MENU_FILE=data/ussd_menu.json -user = -pass = - -[statemachine] -STATES=states/ -TRANSITIONS=transitions/ - -[client] -host = -port = -ssl = diff --git a/apps/cic-ussd/.config/database.ini b/apps/cic-ussd/.config/database.ini deleted file mode 100644 index 5bce6bcf..00000000 --- a/apps/cic-ussd/.config/database.ini +++ /dev/null @@ -1,10 +0,0 @@ -[database] -NAME=cic_ussd -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=postgresql -DRIVER=psycopg2 -DEBUG=0 -POOL_SIZE=1 diff --git a/apps/cic-ussd/.config/redis.ini b/apps/cic-ussd/.config/redis.ini deleted file mode 100644 index 11335fe9..00000000 --- a/apps/cic-ussd/.config/redis.ini +++ /dev/null @@ -1,9 +0,0 @@ -[celery] -BROKER_URL=redis:// -RESULT_URL=redis:// - -[redis] -HOSTNAME=redis -PASSWORD= -PORT=6379 -DATABASE=0 diff --git a/apps/cic-ussd/.config/test/app.ini b/apps/cic-ussd/.config/test/app.ini deleted file mode 100644 index 7ea1b696..00000000 --- a/apps/cic-ussd/.config/test/app.ini +++ /dev/null @@ -1,15 +0,0 @@ -[app] -ALLOWED_IP=127.0.0.1 -LOCALE_FALLBACK=en -LOCALE_PATH=var/lib/locale/ -MAX_BODY_LENGTH=1024 -PASSWORD_PEPPER=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I= -SERVICE_CODE=*483*46# -SUPPORT_PHONE_NUMBER=0757628885 - -[ussd] -MENU_FILE=/usr/local/lib/python3.8/site-packages/cic_ussd/db/ussd_menu.json - -[statemachine] -STATES=/usr/src/cic-ussd/states/ -TRANSITIONS=/usr/src/cic-ussd/transitions/ diff --git a/apps/cic-ussd/.config/test/database.ini b/apps/cic-ussd/.config/test/database.ini deleted file mode 100644 index 9aa21de6..00000000 --- a/apps/cic-ussd/.config/test/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -NAME=cic_ussd_test -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=sqlite -DRIVER=pysqlite diff --git a/apps/cic-ussd/.config/test/pgp.ini b/apps/cic-ussd/.config/test/pgp.ini deleted file mode 100644 index c06f2838..00000000 --- a/apps/cic-ussd/.config/test/pgp.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pgp] -export_dir = /usr/src/pgp/keys/ -keys_path = /usr/src/secrets/ -private_keys = privatekeys_meta.asc -passphrase = diff --git a/apps/cic-ussd/.config/test/redis.ini b/apps/cic-ussd/.config/test/redis.ini deleted file mode 100644 index 20a4d790..00000000 --- a/apps/cic-ussd/.config/test/redis.ini +++ /dev/null @@ -1,9 +0,0 @@ -[celery] -BROKER_URL = filesystem:// -RESULT_URL = filesystem:// - -[redis] -HOSTNAME=localhost -PASSWORD= -PORT=6379 -DATABASE=0 diff --git a/apps/cic-ussd/.gitlab-ci.yml b/apps/cic-ussd/.gitlab-ci.yml index d55d6d41..e028dc6c 100644 --- a/apps/cic-ussd/.gitlab-ci.yml +++ b/apps/cic-ussd/.gitlab-ci.yml @@ -31,7 +31,7 @@ test-mr-cic-ussd: pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple -r test_requirements.txt - - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests + - export PYTHONPATH=. && pytest -x --cov=cic_ussd --cov-fail-under=90 --cov-report term-missing tests/cic_ussd needs: ["build-mr-cic-ussd"] rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" diff --git a/apps/cic-ussd/cic_ussd/account.py b/apps/cic-ussd/cic_ussd/account.py deleted file mode 100644 index 647ca861..00000000 --- a/apps/cic-ussd/cic_ussd/account.py +++ /dev/null @@ -1,49 +0,0 @@ -# standard imports -import json - -# third-party imports -from cic_eth.api import Api -from cic_types.models.person import Person -from cic_types.processor import generate_metadata_pointer - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.db.models.account import Account -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.redis import get_cached_data - - -def define_account_tx_metadata(user: Account): - # get sender metadata - identifier = blockchain_address_to_metadata_pointer( - blockchain_address=user.blockchain_address - ) - key = generate_metadata_pointer( - identifier=identifier, - cic_type=':cic.person' - ) - account_metadata = get_cached_data(key=key) - - if account_metadata: - account_metadata = json.loads(account_metadata) - person = Person() - deserialized_person = person.deserialize(person_data=account_metadata) - given_name = deserialized_person.given_name - family_name = deserialized_person.family_name - phone_number = deserialized_person.tel - - return f'{given_name} {family_name} {phone_number}' - else: - phone_number = user.phone_number - return phone_number - - -def retrieve_account_statement(blockchain_address: str): - chain_str = Chain.spec.__str__() - cic_eth_api = Api( - chain_str=chain_str, - callback_queue='cic-ussd', - callback_task='cic_ussd.tasks.callback_handler.process_statement_callback', - callback_param=blockchain_address - ) - cic_eth_api.list(address=blockchain_address, limit=9) diff --git a/apps/cic-ussd/cic_ussd/account/__init__.py b/apps/cic-ussd/cic_ussd/account/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-ussd/cic_ussd/account/balance.py b/apps/cic-ussd/cic_ussd/account/balance.py new file mode 100644 index 00000000..eddeb4aa --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/balance.py @@ -0,0 +1,90 @@ +# standard imports +import json +import logging +from typing import Optional + +# third-party imports +from cic_eth.api import Api + +# local imports +from cic_ussd.account.transaction import from_wei +from cic_ussd.cache import cache_data_key, get_cached_data +from cic_ussd.error import CachedDataNotFoundError + +logg = logging.getLogger() + + +def get_balances(address: str, + chain_str: str, + token_symbol: str, + asynchronous: bool = False, + callback_param: any = None, + callback_queue='cic-ussd', + callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Optional[list]: + """This function queries cic-eth for an account's balances, It provides a means to receive the balance either + asynchronously or synchronously.. It returns a dictionary containing the network, outgoing and incoming balances. + :param address: Ethereum address of an account. + :type address: str, 0x-hex + :param chain_str: The chain name and network id. + :type chain_str: str + :param asynchronous: Boolean value checking whether to return balances asynchronously. + :type asynchronous: bool + :param callback_param: Data to be sent along with the callback containing balance data. + :type callback_param: any + :param callback_queue: + :type callback_queue: + :param callback_task: A celery task path to which callback data should be sent. + :type callback_task: str + :param token_symbol: ERC20 token symbol of the account whose balance is being queried. + :type token_symbol: str + :return: A list containing balance data if called synchronously. | None + :rtype: list | None + """ + logg.debug(f'retrieving balance for address: {address}') + if asynchronous: + cic_eth_api = Api( + chain_str=chain_str, + callback_queue=callback_queue, + callback_task=callback_task, + callback_param=callback_param + ) + cic_eth_api.balance(address=address, token_symbol=token_symbol) + else: + cic_eth_api = Api(chain_str=chain_str) + balance_request_task = cic_eth_api.balance( + address=address, + token_symbol=token_symbol) + return balance_request_task.get() + + +def calculate_available_balance(balances: dict) -> float: + """This function calculates an account's balance at a specific point in time by computing the difference from the + outgoing balance and the sum of the incoming and network balances. + :param balances: incoming, network and outgoing balances. + :type balances: dict + :return: Token value of the available balance. + :rtype: float + """ + incoming_balance = balances.get('balance_incoming') + outgoing_balance = balances.get('balance_outgoing') + network_balance = balances.get('balance_network') + + available_balance = (network_balance + incoming_balance) - outgoing_balance + return from_wei(value=available_balance) + + +def get_cached_available_balance(blockchain_address: str) -> float: + """This function attempts to retrieve balance data from the redis cache. + :param blockchain_address: Ethereum address of an account. + :type blockchain_address: str + :raises CachedDataNotFoundError: No cached balance data could be found. + :return: Operational balance of an account. + :rtype: float + """ + identifier = bytes.fromhex(blockchain_address[2:]) + key = cache_data_key(identifier, salt=':cic.balances') + cached_balances = get_cached_data(key=key) + if cached_balances: + return calculate_available_balance(json.loads(cached_balances)) + else: + raise CachedDataNotFoundError(f'No cached available balance for address: {blockchain_address}') diff --git a/apps/cic-ussd/cic_ussd/chain.py b/apps/cic-ussd/cic_ussd/account/chain.py similarity index 100% rename from apps/cic-ussd/cic_ussd/chain.py rename to apps/cic-ussd/cic_ussd/account/chain.py diff --git a/apps/cic-ussd/cic_ussd/account/maps.py b/apps/cic-ussd/cic_ussd/account/maps.py new file mode 100644 index 00000000..aaf78e3a --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/maps.py @@ -0,0 +1,20 @@ +# standard imports + +# external imports + +# local imports + + +def gender(): + return { + '1': 'male', + '2': 'female', + '3': 'other' + } + + +def language(): + return { + '1': 'en', + '2': 'sw' + } diff --git a/apps/cic-ussd/cic_ussd/account/metadata.py b/apps/cic-ussd/cic_ussd/account/metadata.py new file mode 100644 index 00000000..d2f8581b --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/metadata.py @@ -0,0 +1,44 @@ +# standard imports +import json +import logging +from typing import Optional + +# external imports +from chainlib.hash import strip_0x +from cic_types.models.person import Person + +# local imports +from cic_ussd.metadata import PreferencesMetadata + +logg = logging.getLogger(__name__) + + +def get_cached_preferred_language(blockchain_address: str) -> Optional[str]: + """This function retrieves an account's set preferred language from preferences metadata in redis cache. + :param blockchain_address: + :type blockchain_address: + :return: Account's set preferred language | Fallback preferred language. + :rtype: str + """ + identifier = bytes.fromhex(strip_0x(blockchain_address)) + preferences_metadata_handler = PreferencesMetadata(identifier) + cached_preferences_metadata = preferences_metadata_handler.get_cached_metadata() + if cached_preferences_metadata: + preferences_metadata = json.loads(cached_preferences_metadata) + return preferences_metadata.get('preferred_language') + return None + + +def parse_account_metadata(account_metadata: dict) -> str: + """ + :param account_metadata: + :type account_metadata: + :return: + :rtype: + """ + person = Person() + deserialized_person = person.deserialize(person_data=account_metadata) + given_name = deserialized_person.given_name + family_name = deserialized_person.family_name + phone_number = deserialized_person.tel + return f'{given_name} {family_name} {phone_number}' diff --git a/apps/cic-ussd/cic_ussd/account/statement.py b/apps/cic-ussd/cic_ussd/account/statement.py new file mode 100644 index 00000000..94cde296 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/statement.py @@ -0,0 +1,111 @@ +# standard imports +import datetime +import logging +from typing import Optional + +# external imports +import celery +from chainlib.hash import strip_0x +from cic_eth.api import Api + +# local import +from cic_ussd.account.chain import Chain +from cic_ussd.account.transaction import from_wei +from cic_ussd.cache import cache_data_key, get_cached_data +from cic_ussd.translation import translation_for + +logg = logging.getLogger(__name__) + + +def filter_statement_transactions(transaction_list: list) -> list: + """This function parses a transaction list and removes all transactions that entail interactions with the + zero address as the source transaction. + :param transaction_list: Array containing transaction objects. + :type transaction_list: list + :return: Transactions exclusive of the zero address transactions. + :rtype: list + """ + return [tx for tx in transaction_list if tx.get('source_token') != '0x0000000000000000000000000000000000000000'] + + +def generate(querying_party: str, queue: Optional[str], transaction: dict): + """ + :param querying_party: + :type querying_party: + :param queue: + :type queue: + :param transaction: + :type transaction: + :return: + :rtype: + """ + s_generate_statement = celery.signature( + 'cic_ussd.tasks.processor.generate_statement', [querying_party, transaction], queue=queue + ) + s_generate_statement.apply_async() + + +def get_cached_statement(blockchain_address: str) -> bytes: + """This function retrieves an account's cached record of a specific number of transactions in chronological order. + :param blockchain_address: Bytes representation of the hex value of an account's blockchain address. + :type blockchain_address: bytes + :return: Account's transactions statements. + :rtype: str + """ + identifier = bytes.fromhex(strip_0x(blockchain_address)) + key = cache_data_key(identifier=identifier, salt=':cic.statement') + return get_cached_data(key=key) + + +def parse_statement_transactions(statement: list): + """This function extracts information for transaction objects loaded from the redis cache and structures the data in + a format that is appropriate for the ussd interface. + :param statement: A list of transaction objects. + :type statement: list + :return: + :rtype: + """ + parsed_transactions = [] + for transaction in statement: + action_tag = transaction.get('action_tag') + amount = from_wei(transaction.get('token_value')) + direction_tag = transaction.get('direction_tag') + token_symbol = transaction.get('token_symbol') + metadata_id = transaction.get('metadata_id') + timestamp = datetime.datetime.now().strftime('%d/%m/%y, %H:%M') + transaction_repr = f'{action_tag} {amount} {token_symbol} {direction_tag} {metadata_id} {timestamp}' + parsed_transactions.append(transaction_repr) + return parsed_transactions + + +def query_statement(blockchain_address: str, limit: int = 9): + """This function queries cic-eth for a set of chronologically ordered number of transactions associated with + an account. + :param blockchain_address: Ethereum address associated with an account. + :type blockchain_address: str, 0x-hex + :param limit: Number of transactions to be returned. + :type limit: int + """ + logg.debug(f'retrieving balance for address: {blockchain_address}') + chain_str = Chain.spec.__str__() + cic_eth_api = Api( + chain_str=chain_str, + callback_queue='cic-ussd', + callback_task='cic_ussd.tasks.callback_handler.statement_callback', + callback_param=blockchain_address + ) + cic_eth_api.list(address=blockchain_address, limit=limit) + + +def statement_transaction_set(preferred_language: str, transaction_reprs: list): + """ + :param preferred_language: + :type preferred_language: + :param transaction_reprs: + :type transaction_reprs: + :return: + :rtype: + """ + if not transaction_reprs: + return translation_for('helpers.no_transaction_history', preferred_language) + return ''.join(f'{transaction_repr}\n' for transaction_repr in transaction_reprs) diff --git a/apps/cic-ussd/cic_ussd/account/tokens.py b/apps/cic-ussd/cic_ussd/account/tokens.py new file mode 100644 index 00000000..40685764 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/tokens.py @@ -0,0 +1,61 @@ +# standard imports +import json +import logging +from typing import Dict, Optional + +# external imports +from cic_eth.api import Api + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.cache import cache_data_key, get_cached_data +from cic_ussd.error import SeppukuError + + +logg = logging.getLogger(__name__) + + +def get_cached_default_token(chain_str: str) -> Optional[str]: + """This function attempts to retrieve the default token's data from the redis cache. + :param chain_str: chain name and network id. + :type chain_str: str + :return: + :rtype: + """ + logg.debug(f'Retrieving default token from cache for chain: {chain_str}') + key = cache_data_key(identifier=chain_str.encode('utf-8'), salt=':cic.default_token_data') + return get_cached_data(key=key) + + +def get_default_token_symbol(): + """This function attempts to retrieve the default token's symbol from cached default token's data. + :raises SeppukuError: The system should terminate itself because the default token is required for an appropriate + system state. + :return: Default token's symbol. + :rtype: str + """ + chain_str = Chain.spec.__str__() + cached_default_token = get_cached_default_token(chain_str) + if cached_default_token: + default_token_data = json.loads(cached_default_token) + return default_token_data.get('symbol') + else: + logg.warning('Cached default token data not found. Attempting retrieval from default token API') + default_token_data = query_default_token(chain_str) + if default_token_data: + return default_token_data.get('symbol') + else: + raise SeppukuError(f'Could not retrieve default token for: {chain_str}') + + +def query_default_token(chain_str: str): + """This function synchronously queries cic-eth for the deployed system's default token. + :param chain_str: Chain name and network id. + :type chain_str: str + :return: Token's data. + :rtype: dict + """ + logg.debug(f'Querying API for default token on chain: {chain_str}') + cic_eth_api = Api(chain_str=chain_str) + default_token_request_task = cic_eth_api.default_token() + return default_token_request_task.get() diff --git a/apps/cic-ussd/cic_ussd/account/transaction.py b/apps/cic-ussd/cic_ussd/account/transaction.py new file mode 100644 index 00000000..fdd44daa --- /dev/null +++ b/apps/cic-ussd/cic_ussd/account/transaction.py @@ -0,0 +1,172 @@ +# standard import +import decimal +import logging +from typing import Dict, Tuple + +# external import +from cic_eth.api import Api +from sqlalchemy.orm.session import Session + +# local import +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.error import UnknownUssdRecipient +from cic_ussd.translation import translation_for + +logg = logging.getLogger(__name__) + + +def _add_tags(action_tag_key: str, preferred_language: str, direction_tag_key: str, transaction: dict): + """ This function adds action and direction tags to a transaction data object. + :param action_tag_key: Key mapping to a helper entry in the translation files describing an action. + :type action_tag_key: str + :param preferred_language: An account's set preferred language. + :type preferred_language: str + :param direction_tag_key: Key mapping to a helper entry in the translation files describing a transaction's + direction relative to the transaction's subject account. + :type direction_tag_key: str + :param transaction: Parsed transaction data object. + :type transaction: dict + """ + action_tag = translation_for(action_tag_key, preferred_language) + direction_tag = translation_for(direction_tag_key, preferred_language) + transaction['action_tag'] = action_tag + transaction['direction_tag'] = direction_tag + + +def aux_transaction_data(preferred_language: str, transaction: dict) -> dict: + """This function adds auxiliary data to a transaction object offering contextual information relative to the + subject account's role in the transaction. + :param preferred_language: An account's set preferred language. + :type preferred_language: str + :param transaction: Parsed transaction data object. + :type transaction: dict + :return: Transaction object with contextual data. + :rtype: dict + """ + role = transaction.get('role') + if role == 'recipient': + _add_tags('helpers.received', preferred_language, 'helpers.from', transaction) + if role == 'sender': + _add_tags('helpers.sent', preferred_language, 'helpers.to', transaction) + return transaction + + +def from_wei(value: int) -> float: + """This function converts values in Wei to a token in the cic network. + :param value: Value in Wei + :type value: int + :return: SRF equivalent of value in Wei + :rtype: float + """ + value = float(value) / 1e+6 + return truncate(value=value, decimals=2) + + +def to_wei(value: int) -> int: + """This functions converts values from a token in the cic network to Wei. + :param value: Value in SRF + :type value: int + :return: Wei equivalent of value in SRF + :rtype: int + """ + return int(value * 1e+6) + + +def truncate(value: float, decimals: int): + """This function truncates a value to a specified number of decimals places. + :param value: The value to be truncated. + :type value: float + :param decimals: The number of decimals for the value to be truncated to + :type decimals: int + :return: The truncated value. + :rtype: int + """ + decimal.getcontext().rounding = decimal.ROUND_DOWN + contextualized_value = decimal.Decimal(value) + return round(contextualized_value, decimals) + + +def transaction_actors(transaction: dict) -> Tuple[Dict, Dict]: + """ This function parses transaction data into a tuple of transaction data objects representative of + of the source and destination account's involved in a transaction. + :param transaction: Transaction data object. + :type transaction: dict + :return: Recipient and sender transaction data object + :rtype: Tuple[Dict, Dict] + """ + destination_token_symbol = transaction.get('destination_token_symbol') + destination_token_value = transaction.get('destination_token_value') or transaction.get('to_value') + recipient_blockchain_address = transaction.get('recipient') + sender_blockchain_address = transaction.get('sender') + source_token_symbol = transaction.get('source_token_symbol') + source_token_value = transaction.get('source_token_value') or transaction.get('from_value') + + recipient_transaction_data = { + "token_symbol": destination_token_symbol, + "token_value": destination_token_value, + "blockchain_address": recipient_blockchain_address, + "role": "recipient", + } + sender_transaction_data = { + "blockchain_address": sender_blockchain_address, + "token_symbol": source_token_symbol, + "token_value": source_token_value, + "role": "sender", + } + return recipient_transaction_data, sender_transaction_data + + +def validate_transaction_account(session: Session, transaction: dict) -> Account: + """This function checks whether the blockchain address specified in a parsed transaction object resolves to an + account object in the ussd system. + :param session: Database session object. + :type session: Session + :param transaction: Parsed transaction data object. + :type transaction: dict + :return: + :rtype: + """ + blockchain_address = transaction.get('blockchain_address') + role = transaction.get('role') + session = SessionBase.bind_session(session) + account = session.query(Account).filter_by(blockchain_address=blockchain_address).first() + if not account: + if role == 'recipient': + raise UnknownUssdRecipient( + f'Tx for recipient: {blockchain_address} has no matching account in the system.' + ) + if role == 'sender': + logg.warning(f'Tx from sender: {blockchain_address} has no matching account in system.') + + SessionBase.release_session(session) + return account + + +class OutgoingTransaction: + + def __init__(self, chain_str: str, from_address: str, to_address: str): + """ + :param chain_str: The chain name and network id. + :type chain_str: str + :param from_address: Ethereum address of the sender + :type from_address: str, 0x-hex + :param to_address: Ethereum address of the recipient + :type to_address: str, 0x-hex + """ + self.chain_str = chain_str + self.cic_eth_api = Api(chain_str=chain_str) + self.from_address = from_address + self.to_address = to_address + + def transfer(self, amount: int, token_symbol: str): + """This function initiates standard transfers between one account to another + :param amount: The amount of tokens to be sent + :type amount: int + :param token_symbol: ERC20 token symbol of token to send + :type token_symbol: str + """ + self.cic_eth_api.transfer(from_address=self.from_address, + to_address=self.to_address, + value=to_wei(value=amount), + token_symbol=token_symbol) diff --git a/apps/cic-ussd/cic_ussd/balance.py b/apps/cic-ussd/cic_ussd/balance.py deleted file mode 100644 index 76039bb2..00000000 --- a/apps/cic-ussd/cic_ussd/balance.py +++ /dev/null @@ -1,92 +0,0 @@ -# standard imports -import json -import logging -from typing import Union - -# third-party imports -import celery -from cic_eth.api import Api - -# local imports -from cic_ussd.error import CachedDataNotFoundError -from cic_ussd.redis import create_cached_data_key, get_cached_data -from cic_ussd.conversions import from_wei - -logg = logging.getLogger() - - -def get_balances( - address: str, - chain_str: str, - token_symbol: str, - asynchronous: bool = False, - callback_param: any = None, - callback_task='cic_ussd.tasks.callback_handler.process_balances_callback') -> Union[celery.Task, dict]: - """ - This function queries cic-eth for an account's balances, It provides a means to receive the balance either - asynchronously or synchronously depending on the provided value for teh asynchronous parameter. It returns a - dictionary containing network, outgoing and incoming balances. - :param address: Ethereum address of the recipient - :type address: str, 0x-hex - :param chain_str: The chain name and network id. - :type chain_str: str - :param callback_param: - :type callback_param: - :param callback_task: - :type callback_task: - :param token_symbol: ERC20 token symbol of the account whose balance is being queried. - :type token_symbol: str - :param asynchronous: Boolean value checking whether to return balances asynchronously. - :type asynchronous: bool - :return: - :rtype: - """ - logg.debug(f'Retrieving balance for address: {address}') - if asynchronous: - cic_eth_api = Api( - chain_str=chain_str, - callback_queue='cic-ussd', - callback_task=callback_task, - callback_param=callback_param - ) - cic_eth_api.balance(address=address, token_symbol=token_symbol) - else: - cic_eth_api = Api(chain_str=chain_str) - balance_request_task = cic_eth_api.balance( - address=address, - token_symbol=token_symbol) - return balance_request_task.get()[0] - - -def compute_operational_balance(balances: dict) -> float: - """This function calculates the right balance given incoming and outgoing - :param balances: - :type balances: - :return: - :rtype: - """ - incoming_balance = balances.get('balance_incoming') - outgoing_balance = balances.get('balance_outgoing') - network_balance = balances.get('balance_network') - - operational_balance = (network_balance + incoming_balance) - outgoing_balance - return from_wei(value=operational_balance) - - -def get_cached_operational_balance(blockchain_address: str): - """ - :param blockchain_address: - :type blockchain_address: - :return: - :rtype: - """ - key = create_cached_data_key( - identifier=bytes.fromhex(blockchain_address[2:]), - salt=':cic.balances_data' - ) - cached_balance = get_cached_data(key=key) - if cached_balance: - operational_balance = compute_operational_balance(balances=json.loads(cached_balance)) - return operational_balance - else: - raise CachedDataNotFoundError('Cached operational balance not found.') diff --git a/apps/cic-ussd/cic_ussd/redis.py b/apps/cic-ussd/cic_ussd/cache.py similarity index 80% rename from apps/cic-ussd/cic_ussd/redis.py rename to apps/cic-ussd/cic_ussd/cache.py index 99fc52f3..5c45e069 100644 --- a/apps/cic-ussd/cic_ussd/redis.py +++ b/apps/cic-ussd/cic_ussd/cache.py @@ -8,8 +8,8 @@ from redis import Redis logg = logging.getLogger() -class InMemoryStore: - cache: Redis = None +class Cache: + store: Redis = None def cache_data(key: str, data: str): @@ -21,9 +21,10 @@ def cache_data(key: str, data: str): :return: :rtype: """ - cache = InMemoryStore.cache + cache = Cache.store cache.set(name=key, value=data) cache.persist(name=key) + logg.debug(f'caching: {data} with key: {key}.') def get_cached_data(key: str): @@ -33,11 +34,11 @@ def get_cached_data(key: str): :return: :rtype: """ - cache = InMemoryStore.cache + cache = Cache.store return cache.get(name=key) -def create_cached_data_key(identifier: bytes, salt: str): +def cache_data_key(identifier: bytes, salt: str): """ :param identifier: :type identifier: diff --git a/apps/cic-ussd/cic_ussd/conversions.py b/apps/cic-ussd/cic_ussd/conversions.py deleted file mode 100644 index d2de2546..00000000 --- a/apps/cic-ussd/cic_ussd/conversions.py +++ /dev/null @@ -1,41 +0,0 @@ -# standard imports -import decimal - -# third-party imports - -# local imports - - -def truncate(value: float, decimals: int): - """This function truncates a value to a specified number of decimals places. - :param value: The value to be truncated. - :type value: float - :param decimals: The number of decimals for the value to be truncated to - :type decimals: int - :return: The truncated value. - :rtype: int - """ - decimal.getcontext().rounding = decimal.ROUND_DOWN - contextualized_value = decimal.Decimal(value) - return round(contextualized_value, decimals) - - -def from_wei(value: int) -> float: - """This function converts values in Wei to a token in the cic network. - :param value: Value in Wei - :type value: int - :return: platform's default token equivalent of value in Wei - :rtype: float - """ - value = float(value) / 1e+6 - return truncate(value=value, decimals=2) - - -def to_wei(value: int) -> int: - """This functions converts values from a token in the cic network to Wei. - :param value: Value in platform's default token - :type value: int - :return: Wei equivalent of value in platform's default token - :rtype: int - """ - return int(value * 1e+6) diff --git a/apps/cic-ussd/cic_ussd/db/migrations/default/versions/2a329190a9af_.py b/apps/cic-ussd/cic_ussd/db/migrations/default/versions/2a329190a9af_.py index 3149c824..a33dfd9b 100644 --- a/apps/cic-ussd/cic_ussd/db/migrations/default/versions/2a329190a9af_.py +++ b/apps/cic-ussd/cic_ussd/db/migrations/default/versions/2a329190a9af_.py @@ -26,7 +26,7 @@ def upgrade(): sa.Column('msisdn', sa.String(), nullable=False), sa.Column('user_input', sa.String(), nullable=True), sa.Column('state', sa.String(), nullable=False), - sa.Column('session_data', postgresql.JSON(astext_type=sa.Text()), nullable=True), + sa.Column('data', postgresql.JSON(astext_type=sa.Text()), nullable=True), sa.Column('version', sa.Integer(), nullable=False), sa.PrimaryKeyConstraint('id') ) diff --git a/apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py b/apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py index 29d1e470..b871cc9c 100644 --- a/apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py +++ b/apps/cic-ussd/cic_ussd/db/migrations/default/versions/f289e8510444_.py @@ -24,7 +24,7 @@ def upgrade(): sa.Column('preferred_language', sa.String(), nullable=True), sa.Column('password_hash', sa.String(), nullable=True), sa.Column('failed_pin_attempts', sa.Integer(), nullable=False), - sa.Column('account_status', sa.Integer(), nullable=False), + sa.Column('status', sa.Integer(), nullable=False), sa.Column('created', sa.DateTime(), nullable=False), sa.Column('updated', sa.DateTime(), nullable=False), sa.PrimaryKeyConstraint('id') diff --git a/apps/cic-ussd/cic_ussd/db/models/account.py b/apps/cic-ussd/cic_ussd/db/models/account.py index 1e073625..c9d36b6d 100644 --- a/apps/cic-ussd/cic_ussd/db/models/account.py +++ b/apps/cic-ussd/cic_ussd/db/models/account.py @@ -1,11 +1,17 @@ # standard imports +import json + +# external imports +from chainlib.hash import strip_0x +from cic_eth.api import Api # local imports +from cic_ussd.account.metadata import get_cached_preferred_language, parse_account_metadata +from cic_ussd.cache import Cache, cache_data_key, get_cached_data from cic_ussd.db.enum import AccountStatus from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.models.task_tracker import TaskTracker from cic_ussd.encoder import check_password_hash, create_password_hash - -# third party imports from sqlalchemy import Column, Integer, String from sqlalchemy.orm.session import Session @@ -21,9 +27,32 @@ class Account(SessionBase): phone_number = Column(String) password_hash = Column(String) failed_pin_attempts = Column(Integer) - account_status = Column(Integer) + status = Column(Integer) preferred_language = Column(String) + def __init__(self, blockchain_address, phone_number): + self.blockchain_address = blockchain_address + self.phone_number = phone_number + self.password_hash = None + self.failed_pin_attempts = 0 + self.status = AccountStatus.PENDING.value + + def __repr__(self): + return f'' + + def activate_account(self): + """This method is used to reset failed pin attempts and change account status to Active.""" + self.failed_pin_attempts = 0 + self.status = AccountStatus.ACTIVE.value + + def create_password(self, password): + """This method takes a password value and hashes the value before assigning it to the corresponding + `hashed_password` attribute in the user record. + :param password: A password value + :type password: str + """ + self.password_hash = create_password_hash(password) + @staticmethod def get_by_phone_number(phone_number: str, session: Session): """Retrieves an account from a phone number. @@ -39,23 +68,68 @@ class Account(SessionBase): SessionBase.release_session(session=session) return account - def __init__(self, blockchain_address, phone_number): - self.blockchain_address = blockchain_address - self.phone_number = phone_number - self.password_hash = None - self.failed_pin_attempts = 0 - self.account_status = AccountStatus.PENDING.value + def has_preferred_language(self) -> bool: + return get_cached_preferred_language(self.blockchain_address) is not None - def __repr__(self): - return f'' - - def create_password(self, password): - """This method takes a password value and hashes the value before assigning it to the corresponding - `hashed_password` attribute in the user record. - :param password: A password value - :type password: str + def has_valid_pin(self, session: Session): """ - self.password_hash = create_password_hash(password) + :param session: + :type session: + :return: + :rtype: + """ + return self.get_status(session) == AccountStatus.ACTIVE.name and self.password_hash is not None + + def pin_is_blocked(self, session: Session) -> bool: + """ + :param session: + :type session: + :return: + :rtype: + """ + return self.failed_pin_attempts == 3 and self.get_status(session) == AccountStatus.LOCKED.name + + def reset_pin(self, session: Session) -> str: + """This function resets the number of failed pin attempts to zero. It places the account in pin reset status + enabling users to reset their pins. + :param session: Database session object. + :type session: Session + """ + session = SessionBase.bind_session(session=session) + self.failed_pin_attempts = 0 + self.status = AccountStatus.RESET.value + session.add(self) + session.flush() + SessionBase.release_session(session=session) + return f'Pin reset successful.' + + def standard_metadata_id(self) -> str: + """This function creates an account's standard metadata identification information that contains an account owner's + given name, family name and phone number and defaults to a phone number in the absence of metadata. + :return: Standard metadata identification information | e164 formatted phone number. + :rtype: str + """ + identifier = bytes.fromhex(strip_0x(self.blockchain_address)) + key = cache_data_key(identifier, ':cic.person') + account_metadata = get_cached_data(key) + if not account_metadata: + return self.phone_number + account_metadata = json.loads(account_metadata) + return parse_account_metadata(account_metadata) + + def get_status(self, session: Session): + """This function handles account status queries, it checks whether an account's failed pin attempts exceed 2 and + updates the account status locked, it then returns the account status + :return: The account status for a user object + :rtype: str + """ + session = SessionBase.bind_session(session=session) + if self.failed_pin_attempts > 2: + self.status = AccountStatus.LOCKED.value + session.add(self) + session.flush() + SessionBase.release_session(session=session) + return AccountStatus(self.status).name def verify_password(self, password): """This method takes a password value and compares it to the user's corresponding `hashed_password` value to @@ -67,33 +141,41 @@ class Account(SessionBase): """ return check_password_hash(password, self.password_hash) - def reset_account_pin(self): - """This method is used to unlock a user's account.""" - self.failed_pin_attempts = 0 - self.account_status = AccountStatus.RESET.value - def get_account_status(self): - """This method checks whether the account is past the allowed number of failed pin attempts. - If so, it changes the accounts status to Locked. - :return: The account status for a user object - :rtype: str - """ - if self.failed_pin_attempts > 2: - self.account_status = AccountStatus.LOCKED.value - return AccountStatus(self.account_status).name +def create(chain_str: str, phone_number: str, session: Session): + """ + :param chain_str: + :type chain_str: + :param phone_number: + :type phone_number: + :param session: + :type session: + :return: + :rtype: + """ + api = Api(callback_task='cic_ussd.tasks.callback_handler.account_creation_callback', + callback_queue='cic-ussd', + callback_param='', + chain_str=chain_str) + task_uuid = api.create_account().id + TaskTracker.add(session=session, task_uuid=task_uuid) + cache_creation_task_uuid(phone_number=phone_number, task_uuid=task_uuid) - def activate_account(self): - """This method is used to reset failed pin attempts and change account status to Active.""" - self.failed_pin_attempts = 0 - self.account_status = AccountStatus.ACTIVE.value - def has_valid_pin(self): - """This method checks whether the user's account status and if a pin hash is present which implies - pin validity. - :return: The presence of a valid pin and status of the account being active. - :rtype: bool - """ - valid_pin = None - if self.get_account_status() == 'ACTIVE' and self.password_hash is not None: - valid_pin = True - return valid_pin +def cache_creation_task_uuid(phone_number: str, task_uuid: str): + """This function stores the task id that is returned from a task spawned to create a blockchain account in the redis + cache. + :param phone_number: The phone number for the user whose account is being created. + :type phone_number: str + :param task_uuid: A celery task id + :type task_uuid: str + """ + cache = Cache.store + account_creation_request_data = { + 'phone_number': phone_number, + 'sms_notification_sent': False, + 'status': 'PENDING', + 'task_uuid': task_uuid + } + cache.set(task_uuid, json.dumps(account_creation_request_data)) + cache.persist(name=task_uuid) diff --git a/apps/cic-ussd/cic_ussd/db/models/base.py b/apps/cic-ussd/cic_ussd/db/models/base.py index af5b9067..461c8050 100644 --- a/apps/cic-ussd/cic_ussd/db/models/base.py +++ b/apps/cic-ussd/cic_ussd/db/models/base.py @@ -8,11 +8,11 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import ( - StaticPool, - QueuePool, - AssertionPool, - NullPool, - ) + StaticPool, + QueuePool, + AssertionPool, + NullPool, +) logg = logging.getLogger().getChild(__name__) @@ -42,14 +42,12 @@ class SessionBase(Model): localsessions = {} """Contains dictionary of sessions initiated by db model components""" - @staticmethod def create_session(): """Creates a new database session. """ return SessionBase.sessionmaker() - @staticmethod def _set_engine(engine): """Sets the database engine static property @@ -57,7 +55,6 @@ class SessionBase(Model): SessionBase.engine = engine SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine) - @staticmethod def connect(dsn, pool_size=16, debug=False): """Create new database connection engine and connect to database backend. @@ -71,14 +68,14 @@ class SessionBase(Model): if pool_size > 1: logg.info('db using queue pool') e = create_engine( - dsn, - max_overflow=pool_size*3, - pool_pre_ping=True, - pool_size=pool_size, - pool_recycle=60, - poolclass=poolclass, - echo=debug, - ) + dsn, + max_overflow=pool_size * 3, + pool_pre_ping=True, + pool_size=pool_size, + pool_recycle=60, + poolclass=poolclass, + echo=debug, + ) else: if pool_size == 0: poolclass = NullPool @@ -87,20 +84,19 @@ class SessionBase(Model): else: poolclass = StaticPool e = create_engine( - dsn, - poolclass=poolclass, - echo=debug, - ) + dsn, + poolclass=poolclass, + echo=debug, + ) else: logg.info('db connection not poolable') e = create_engine( - dsn, - echo=debug, - ) + dsn, + echo=debug, + ) SessionBase._set_engine(e) - @staticmethod def disconnect(): """Disconnect from database and free resources. @@ -108,18 +104,16 @@ class SessionBase(Model): SessionBase.engine.dispose() SessionBase.engine = None - @staticmethod def bind_session(session=None): localsession = session - if localsession == None: + if localsession is None: localsession = SessionBase.create_session() localsession_key = str(id(localsession)) logg.debug('creating new session {}'.format(localsession_key)) SessionBase.localsessions[localsession_key] = localsession return localsession - @staticmethod def release_session(session=None): session_key = str(id(session)) diff --git a/apps/cic-ussd/cic_ussd/db/models/task_tracker.py b/apps/cic-ussd/cic_ussd/db/models/task_tracker.py index 252a635b..b1d9c16f 100644 --- a/apps/cic-ussd/cic_ussd/db/models/task_tracker.py +++ b/apps/cic-ussd/cic_ussd/db/models/task_tracker.py @@ -3,6 +3,7 @@ import logging # third-party imports from sqlalchemy import Column, String +from sqlalchemy.orm.session import Session # local imports from cic_ussd.db.models.base import SessionBase @@ -17,3 +18,17 @@ class TaskTracker(SessionBase): self.task_uuid = task_uuid task_uuid = Column(String, nullable=False) + + @staticmethod + def add(session: Session, task_uuid: str): + """This function persists celery tasks uuids to storage. + :param session: Database session object. + :type session: Session + :param task_uuid: The uuid for an initiated task. + :type task_uuid: str + """ + session = SessionBase.bind_session(session=session) + task_record = TaskTracker(task_uuid=task_uuid) + session.add(task_record) + session.flush() + SessionBase.release_session(session=session) diff --git a/apps/cic-ussd/cic_ussd/db/models/ussd_session.py b/apps/cic-ussd/cic_ussd/db/models/ussd_session.py index 7b028056..42866abf 100644 --- a/apps/cic-ussd/cic_ussd/db/models/ussd_session.py +++ b/apps/cic-ussd/cic_ussd/db/models/ussd_session.py @@ -2,9 +2,10 @@ import logging # third-party imports -from sqlalchemy import Column, String, Integer +from sqlalchemy import Column, desc, Integer, String from sqlalchemy.dialects.postgresql import JSON from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.orm.session import Session # local imports from cic_ussd.db.models.base import SessionBase @@ -16,26 +17,26 @@ logg = logging.getLogger(__name__) class UssdSession(SessionBase): __tablename__ = 'ussd_session' + data = Column(JSON) external_session_id = Column(String, nullable=False, index=True, unique=True) - service_code = Column(String, nullable=False) msisdn = Column(String, nullable=False) - user_input = Column(String) + service_code = Column(String, nullable=False) state = Column(String, nullable=False) - session_data = Column(JSON) + user_input = Column(String) version = Column(Integer, nullable=False) def set_data(self, key, session, value): - if self.session_data is None: - self.session_data = {} - self.session_data[key] = value + if self.data is None: + self.data = {} + self.data[key] = value # https://stackoverflow.com/questions/42559434/updates-to-json-field-dont-persist-to-db - flag_modified(self, "session_data") + flag_modified(self, "data") session.add(self) def get_data(self, key): - if self.session_data is not None: - return self.session_data.get(key) + if self.data is not None: + return self.data.get(key) else: return None @@ -51,9 +52,37 @@ class UssdSession(SessionBase): session.add(self) @staticmethod - def have_session_for_phone(phone): - r = UssdSession.session.query(UssdSession).filter_by(msisdn=phone).first() - return r is not None + def has_record_for_phone_number(phone_number: str, session: Session): + """ + :param phone_number: + :type phone_number: + :param session: + :type session: + :return: + :rtype: + """ + session = SessionBase.bind_session(session=session) + ussd_session = session.query(UssdSession).filter_by(msisdn=phone_number).first() + SessionBase.release_session(session=session) + return ussd_session is not None + + @staticmethod + def last_ussd_session(phone_number: str, session: Session): + """ + :param phone_number: + :type phone_number: + :param session: + :type session: + :return: + :rtype: + """ + session = SessionBase.bind_session(session=session) + ussd_session = session.query(UssdSession) \ + .filter_by(msisdn=phone_number) \ + .order_by(desc(UssdSession.created)) \ + .first() + SessionBase.release_session(session=session) + return ussd_session def to_json(self): """ This function serializes the in db ussd session object to a JSON object @@ -61,11 +90,11 @@ class UssdSession(SessionBase): :rtype: dict """ return { + "data": self.data, "external_session_id": self.external_session_id, - "service_code": self.service_code, "msisdn": self.msisdn, - "user_input": self.user_input, + "service_code": self.service_code, "state": self.state, - "session_data": self.session_data, + "user_input": self.user_input, "version": self.version } diff --git a/apps/cic-ussd/cic_ussd/error.py b/apps/cic-ussd/cic_ussd/error.py index 1bb7c97a..31bbc745 100644 --- a/apps/cic-ussd/cic_ussd/error.py +++ b/apps/cic-ussd/cic_ussd/error.py @@ -8,27 +8,27 @@ class SessionNotFoundError(Exception): pass -class InvalidFileFormatError(OSError): +class InvalidFileFormatError(Exception): """Raised when the file format is invalid.""" pass -class ActionDataNotFoundError(OSError): - """Raised when action data matching a specific task uuid is not found in the redis cache""" +class AccountCreationDataNotFound(Exception): + """Raised when account creation data matching a specific task uuid is not found in the redis cache""" pass -class MetadataNotFoundError(OSError): +class MetadataNotFoundError(Exception): """Raised when metadata is expected but not available in cache.""" pass -class UnsupportedMethodError(OSError): +class UnsupportedMethodError(Exception): """Raised when the method passed to the make request function is unsupported.""" pass -class CachedDataNotFoundError(OSError): +class CachedDataNotFoundError(Exception): """Raised when the method passed to the make request function is unsupported.""" pass @@ -51,3 +51,5 @@ class InitializationError(Exception): class UnknownUssdRecipient(Exception): """Raised when a recipient of a transaction is not known to the ussd application.""" + + diff --git a/apps/cic-ussd/cic_ussd/http/__init__.py b/apps/cic-ussd/cic_ussd/http/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-ussd/cic_ussd/http/requests.py b/apps/cic-ussd/cic_ussd/http/requests.py new file mode 100644 index 00000000..884fbc57 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/http/requests.py @@ -0,0 +1,65 @@ +# standard imports +import logging +from typing import Optional, Union +from urllib.parse import urlparse, parse_qs + +# external imports +import requests +from requests.exceptions import HTTPError + +# local imports +from cic_ussd.error import UnsupportedMethodError + +logg = logging.getLogger(__file__) + + +def error_handler(result: requests.Response): + """""" + status_code = result.status_code + + if 100 <= status_code < 200: + raise HTTPError(f'Informational errors: {status_code}, reason: {result.reason}') + + elif 300 <= status_code < 400: + raise HTTPError(f'Redirect Issues: {status_code}, reason: {result.reason}') + + elif 400 <= status_code < 500: + raise HTTPError(f'Client Error: {status_code}, reason: {result.reason}') + + elif 500 <= status_code < 600: + raise HTTPError(f'Server Error: {status_code}, reason: {result.reason}') + + +def get_query_parameters(env: dict, query_name: Optional[str] = None) -> Union[dict, str]: + """""" + parsed_url = urlparse(env.get('REQUEST_URI')) + params = parse_qs(parsed_url.query) + if query_name: + return params.get(query_name)[0] + return params + + +def get_request_endpoint(env: dict) -> str: + """""" + return env.get('PATH_INFO') + + +def get_request_method(env: dict) -> str: + """""" + return env.get('REQUEST_METHOD').upper() + + +def make_request(method: str, url: str, data: any = None, headers: dict = None): + """""" + if method == 'GET': + logg.debug(f'Retrieving data from: {url}') + result = requests.get(url=url) + elif method == 'POST': + logg.debug(f'Posting to: {url} with: {data}') + result = requests.post(url=url, data=data, headers=headers) + elif method == 'PUT': + logg.debug(f'Putting to: {url} with: {data}') + result = requests.put(url=url, data=data, headers=headers) + else: + raise UnsupportedMethodError(f'Unsupported method: {method}') + return result diff --git a/apps/cic-ussd/cic_ussd/http/responses.py b/apps/cic-ussd/cic_ussd/http/responses.py new file mode 100644 index 00000000..38a4cf0e --- /dev/null +++ b/apps/cic-ussd/cic_ussd/http/responses.py @@ -0,0 +1,26 @@ +# standard imports +import logging +from typing import Tuple + +# external imports + +# local imports + + +def with_content_headers(headers: list, response: str) -> Tuple[bytes, list]: + """This function calculates the length of a http response body and appends the content length to the headers. + :param headers: A list of tuples defining headers for responses. + :type headers: list + :param response: The response to send for an incoming http request + :type response: str + :return: A tuple containing the response bytes and a list of tuples defining headers + :rtype: tuple + """ + response_bytes = response.encode('utf-8') + content_length = len(response_bytes) + content_length_header = ('Content-Length', str(content_length)) + for position, header in enumerate(headers): + if 'Content-Length' in header: + headers.pop(position) + headers.append(content_length_header) + return response_bytes, headers diff --git a/apps/cic-ussd/cic_ussd/http/routes.py b/apps/cic-ussd/cic_ussd/http/routes.py new file mode 100644 index 00000000..36afa22c --- /dev/null +++ b/apps/cic-ussd/cic_ussd/http/routes.py @@ -0,0 +1,87 @@ +# standard imports +import json +import logging +import re +from urllib.parse import quote_plus + +# external imports +from sqlalchemy import desc +from sqlalchemy.orm.session import Session + +# local imports +from cic_ussd.db.enum import AccountStatus +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.http.requests import get_query_parameters, get_request_method +from cic_ussd.http.responses import with_content_headers + +logg = logging.getLogger(__file__) + + +def _get_locked_accounts(env: dict, session: Session): + offset = 0 + limit = 100 + + locked_accounts_path = r'/accounts/locked/(\d+)?/?(\d+)?' + r = re.match(locked_accounts_path, env.get('PATH_INFO')) + + if r: + if r.lastindex > 1: + offset = r[1] + limit = r[2] + else: + limit = r[1] + session = SessionBase.bind_session(session) + accounts = session.query(Account.blockchain_address)\ + .filter(Account.status == AccountStatus.LOCKED.value, Account.failed_pin_attempts >= 3)\ + .order_by(desc(Account.updated))\ + .offset(offset)\ + .limit(limit)\ + .all() + accounts = [blockchain_address for (blockchain_address,) in accounts] + SessionBase.release_session(session=session) + response = json.dumps(accounts) + return response, '200 OK' + + +def locked_accounts(env: dict, session: Session) -> tuple: + """ + :param env: + :type env: + :param session: + :type session: + :return: + :rtype: + """ + if get_request_method(env) == 'GET': + return _get_locked_accounts(env, session) + return '', '405 Play by the rules' + + +def pin_reset(env: dict, phone_number: str, session: Session): + """""" + account = session.query(Account).filter_by(phone_number=phone_number).first() + if not account: + return '', '404 Not found' + + if get_request_method(env) == 'PUT': + return account.reset_pin(session), '200 OK' + + if get_request_method(env) == 'GET': + status = account.get_status(session) + response = { + 'status': f'{status}' + } + response = json.dumps(response) + return response, '200 OK' + + +def handle_pin_requests(env, session, errors_headers, start_response): + phone_number = get_query_parameters(env=env, query_name='phoneNumber') + phone_number = quote_plus(phone_number) + response, message = pin_reset(env=env, phone_number=phone_number, session=session) + response_bytes, headers = with_content_headers(errors_headers, response) + session.commit() + session.close() + start_response(message, headers) + return [response_bytes] diff --git a/apps/cic-ussd/cic_ussd/menu/ussd_menu.py b/apps/cic-ussd/cic_ussd/menu/ussd_menu.py index e7a31177..5f974cd7 100644 --- a/apps/cic-ussd/cic_ussd/menu/ussd_menu.py +++ b/apps/cic-ussd/cic_ussd/menu/ussd_menu.py @@ -65,11 +65,10 @@ class UssdMenu: :rtype: Document. """ menu = UssdMenu.ussd_menu_db.get(UssdMenu.Menu.name == name) - if not menu: - logg.error("No USSD Menu with name {}".format(name)) - return UssdMenu.ussd_menu_db.get(UssdMenu.Menu.name == 'exit_invalid_request') - else: + if menu: return menu + logg.error("No USSD Menu with name {}".format(name)) + return UssdMenu.ussd_menu_db.get(UssdMenu.Menu.name == 'exit_invalid_request') @staticmethod def set_description(name: str, description: str): diff --git a/apps/cic-ussd/cic_ussd/metadata/__init__.py b/apps/cic-ussd/cic_ussd/metadata/__init__.py index 44c4a217..33989352 100644 --- a/apps/cic-ussd/cic_ussd/metadata/__init__.py +++ b/apps/cic-ussd/cic_ussd/metadata/__init__.py @@ -1,46 +1,10 @@ # standard imports -# third-party imports -import requests -from chainlib.eth.address import to_checksum -from hexathon import ( - add_0x, - strip_0x, - ) +# external imports # local imports -from cic_ussd.error import UnsupportedMethodError - - -def make_request(method: str, url: str, data: any = None, headers: dict = None): - """ - :param method: - :type method: - :param url: - :type url: - :param data: - :type data: - :param headers: - :type headers: - :return: - :rtype: - """ - if method == 'GET': - result = requests.get(url=url) - elif method == 'POST': - result = requests.post(url=url, data=data, headers=headers) - elif method == 'PUT': - result = requests.put(url=url, data=data, headers=headers) - else: - raise UnsupportedMethodError(f'Unsupported method: {method}') - return result - - -def blockchain_address_to_metadata_pointer(blockchain_address: str): - """ - :param blockchain_address: - :type blockchain_address: - :return: - :rtype: - """ - return bytes.fromhex(strip_0x(blockchain_address)) +from .base import Metadata +from .custom import CustomMetadata +from .person import PersonMetadata +from .phone import PhonePointerMetadata +from .preferences import PreferencesMetadata diff --git a/apps/cic-ussd/cic_ussd/metadata/base.py b/apps/cic-ussd/cic_ussd/metadata/base.py index 1f059e25..f184adf6 100644 --- a/apps/cic-ussd/cic_ussd/metadata/base.py +++ b/apps/cic-ussd/cic_ussd/metadata/base.py @@ -5,17 +5,14 @@ import os from typing import Dict, Union # third-part imports -import requests from cic_types.models.person import generate_metadata_pointer, Person # local imports -from cic_ussd.metadata import make_request +from cic_ussd.cache import cache_data, get_cached_data +from cic_ussd.http.requests import error_handler, make_request from cic_ussd.metadata.signer import Signer -from cic_ussd.redis import cache_data -from cic_ussd.error import MetadataStoreError - -logg = logging.getLogger().getChild(__name__) +logg = logging.getLogger(__file__) class Metadata: @@ -27,37 +24,10 @@ class Metadata: base_url = None -def metadata_http_error_handler(result: requests.Response): - """ This function handles and appropriately raises errors from http requests interacting with the metadata server. - :param result: The response object from a http request. - :type result: requests.Response - """ - status_code = result.status_code - - if 100 <= status_code < 200: - raise MetadataStoreError(f'Informational errors: {status_code}, reason: {result.reason}') - - elif 300 <= status_code < 400: - raise MetadataStoreError(f'Redirect Issues: {status_code}, reason: {result.reason}') - - elif 400 <= status_code < 500: - raise MetadataStoreError(f'Client Error: {status_code}, reason: {result.reason}') - - elif 500 <= status_code < 600: - raise MetadataStoreError(f'Server Error: {status_code}, reason: {result.reason}') - - class MetadataRequestsHandler(Metadata): def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'): - """ - :param cic_type: The salt value with which to hash a specific metadata identifier. - :type cic_type: str - :param engine: Encryption used for sending data to the metadata server. - :type engine: str - :param identifier: A unique element of data in bytes necessary for creating a metadata pointer. - :type identifier: bytes - """ + """""" self.cic_type = cic_type self.engine = engine self.headers = { @@ -73,22 +43,16 @@ class MetadataRequestsHandler(Metadata): self.url = os.path.join(self.base_url, self.metadata_pointer) def create(self, data: Union[Dict, str]): - """ This function is responsible for posting data to the metadata server with a corresponding metadata pointer - for storage. - :param data: The data to be stored in the metadata server. - :type data: dict|str - """ + """""" data = json.dumps(data) result = make_request(method='POST', url=self.url, data=data, headers=self.headers) - metadata_http_error_handler(result=result) + + error_handler(result=result) metadata = result.json() - self.edit(data=metadata) + return self.edit(data=metadata) def edit(self, data: Union[Dict, str]): - """ This function is responsible for editing data in the metadata server corresponding to a unique pointer. - :param data: The data to be edited in the metadata server. - :type data: dict - """ + """""" cic_meta_signer = Signer() signature = cic_meta_signer.sign_digest(data=data) algorithm = cic_meta_signer.get_operational_key().get('algo') @@ -104,42 +68,34 @@ class MetadataRequestsHandler(Metadata): formatted_data = json.dumps(formatted_data) result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers) logg.info(f'signed metadata submission status: {result.status_code}.') - metadata_http_error_handler(result=result) + error_handler(result=result) try: decoded_identifier = self.identifier.decode("utf-8") except UnicodeDecodeError: decoded_identifier = self.identifier.hex() logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.') + return result def query(self): - """ - :return: - :rtype: - """ - # retrieve the metadata + """""" result = make_request(method='GET', url=self.url) - metadata_http_error_handler(result=result) - - # json serialize retrieved data + error_handler(result=result) result_data = result.json() - - # validate result data format if not isinstance(result_data, dict): raise ValueError(f'Invalid result data object: {result_data}.') - if result.status_code == 200: if self.cic_type == ':cic.person': - # validate person metadata person = Person() person_data = person.deserialize(person_data=result_data) - - # format new person data for caching serialized_person_data = person_data.serialize() data = json.dumps(serialized_person_data) else: data = json.dumps(result_data) - - # cache metadata cache_data(key=self.metadata_pointer, data=data) logg.debug(f'caching: {data} with key: {self.metadata_pointer}') return result_data + + def get_cached_metadata(self): + """""" + key = generate_metadata_pointer(self.identifier, self.cic_type) + return get_cached_data(key) diff --git a/apps/cic-ussd/cic_ussd/metadata/person.py b/apps/cic-ussd/cic_ussd/metadata/person.py index 63501f4c..dff4fddc 100644 --- a/apps/cic-ussd/cic_ussd/metadata/person.py +++ b/apps/cic-ussd/cic_ussd/metadata/person.py @@ -1,6 +1,6 @@ # standard imports -# third-party imports +# external imports # local imports from .base import MetadataRequestsHandler diff --git a/apps/cic-ussd/cic_ussd/metadata/signer.py b/apps/cic-ussd/cic_ussd/metadata/signer.py index 39544a32..7b55d8bb 100644 --- a/apps/cic-ussd/cic_ussd/metadata/signer.py +++ b/apps/cic-ussd/cic_ussd/metadata/signer.py @@ -29,10 +29,8 @@ class Signer: def __init__(self): self.gpg = gnupg.GPG(gnupghome=self.gpg_path) - # parse key file data - key_file = open(self.key_file_path, 'r') - self.key_data = key_file.read() - key_file.close() + with open(self.key_file_path, 'r') as key_file: + self.key_data = key_file.read() def get_operational_key(self): """ diff --git a/apps/cic-ussd/cic_ussd/notifications.py b/apps/cic-ussd/cic_ussd/notifications.py index 22392e56..2bd3cdce 100644 --- a/apps/cic-ussd/cic_ussd/notifications.py +++ b/apps/cic-ussd/cic_ussd/notifications.py @@ -21,9 +21,6 @@ class Notifier: :param preferred_language: A notification recipient's preferred language. :type preferred_language: str """ - if self.queue is False: - notify_api = Api() - else: - notify_api = Api(queue=self.queue) + notify_api = Api() if self.queue is False else Api(queue=self.queue) message = translation_for(key=key, preferred_language=preferred_language, **kwargs) notify_api.sms(recipient=phone_number, message=message) diff --git a/apps/cic-ussd/cic_ussd/operations.py b/apps/cic-ussd/cic_ussd/operations.py deleted file mode 100644 index bd0e7476..00000000 --- a/apps/cic-ussd/cic_ussd/operations.py +++ /dev/null @@ -1,521 +0,0 @@ -# standard imports -import json -import logging - -# third party imports -import celery -import i18n -from cic_eth.api.api_task import Api -from sqlalchemy.orm.session import Session -from tinydb.table import Document -from typing import Optional - -# local imports -from cic_ussd.db.models.account import Account -from cic_ussd.db.models.base import SessionBase -from cic_ussd.db.models.ussd_session import UssdSession -from cic_ussd.db.models.task_tracker import TaskTracker -from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.processor import custom_display_text, process_request, retrieve_most_recent_ussd_session -from cic_ussd.redis import InMemoryStore -from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession -from cic_ussd.validator import check_known_user, validate_response_type - -logg = logging.getLogger() - - -def add_tasks_to_tracker(session, task_uuid: str): - """This function takes tasks spawned over api interfaces and records their creation time for tracking. - :param session: - :type session: - :param task_uuid: The uuid for an initiated task. - :type task_uuid: str - """ - session = SessionBase.bind_session(session=session) - task_record = TaskTracker(task_uuid=task_uuid) - session.add(task_record) - session.flush() - SessionBase.release_session(session=session) - - -def define_response_with_content(headers: list, response: str) -> tuple: - """This function encodes responses to byte form in order to make feasible for uwsgi response formats. It then - computes the length of the response and appends the content length to the headers. - :param headers: A list of tuples defining headers for responses. - :type headers: list - :param response: The response to send for an incoming http request - :type response: str - :return: A tuple containing the response bytes and a list of tuples defining headers - :rtype: tuple - """ - response_bytes = response.encode('utf-8') - content_length = len(response_bytes) - content_length_header = ('Content-Length', str(content_length)) - # check for content length defaulted to zero in error headers - for position, header in enumerate(headers): - if 'Content-Length' in header: - headers.pop(position) - headers.append(content_length_header) - return response_bytes, headers - - -def create_ussd_session( - external_session_id: str, - phone: str, - service_code: str, - user_input: str, - current_menu: str, - session_data: Optional[dict] = None) -> InMemoryUssdSession: - """ - Creates a new ussd session - :param external_session_id: Session id value provided by AT - :type external_session_id: str - :param phone: A valid phone number - :type phone: str - :param service_code: service code passed over request - :type service_code AT service code - :param user_input: Input from the request - :type user_input: str - :param current_menu: Menu name that is currently being displayed on the ussd session - :type current_menu: str - :param session_data: Any additional data that was persisted during the user's interaction with the system. - :type session_data: dict. - :return: ussd session object - :rtype: Session - """ - session = InMemoryUssdSession( - external_session_id=external_session_id, - msisdn=phone, - user_input=user_input, - state=current_menu, - service_code=service_code, - session_data=session_data - ) - return session - - -def create_or_update_session( - external_session_id: str, - phone: str, - service_code: str, - user_input: str, - current_menu: str, - session, - session_data: Optional[dict] = None) -> InMemoryUssdSession: - """ - Handles the creation or updating of session as necessary. - :param external_session_id: Session id value provided by AT - :type external_session_id: str - :param phone: A valid phone number - :type phone: str - :param service_code: service code passed over request - :type service_code: AT service code - :param user_input: input from the request - :type user_input: str - :param current_menu: Menu name that is currently being displayed on the ussd session - :type current_menu: str - :param session: - :type session: - :param session_data: Any additional data that was persisted during the user's interaction with the system. - :type session_data: dict. - :return: ussd session object - :rtype: InMemoryUssdSession - """ - session = SessionBase.bind_session(session=session) - existing_ussd_session = session.query(UssdSession).filter_by( - external_session_id=external_session_id).first() - - if existing_ussd_session: - ussd_session = update_ussd_session( - ussd_session=existing_ussd_session, - current_menu=current_menu, - user_input=user_input, - session_data=session_data - ) - else: - ussd_session = create_ussd_session( - external_session_id=external_session_id, - phone=phone, - service_code=service_code, - user_input=user_input, - current_menu=current_menu, - session_data=session_data - ) - SessionBase.release_session(session=session) - return ussd_session - - -def get_account_status(phone_number, session: Session) -> str: - """Get the status of a user's account. - :param phone_number: The phone number to be checked. - :type phone_number: str - :param session: - :type session: - :return: The user account status. - :rtype: str - """ - session = SessionBase.bind_session(session=session) - account = Account.get_by_phone_number(phone_number=phone_number, session=session) - status = account.get_account_status() - session.add(account) - session.flush() - SessionBase.release_session(session=session) - - return status - - -def get_latest_input(user_input: str) -> str: - """This function gets the last value entered by the user from the collective user input which follows the pattern of - asterix (*) separated entries. - :param user_input: The data entered by a user. - :type user_input: str - :return: The last element in the user input value. - :rtype: str - """ - return user_input.split('*')[-1] - - -def initiate_account_creation_request(chain_str: str, - external_session_id: str, - phone_number: str, - service_code: str, - session, - user_input: str) -> str: - """This function issues a task to create a blockchain account on cic-eth. It then creates a record of the ussd - session corresponding to the creation of the account and returns a response denoting that the user's account is - being created. - :param chain_str: The chain name and network id. - :type chain_str: str - :param external_session_id: A unique ID from africastalking. - :type external_session_id: str - :param phone_number: The phone number for the account to be created. - :type phone_number: str - :param service_code: The service code dialed. - :type service_code: str - :param session: - :type session: - :param user_input: The input entered by the user. - :type user_input: str - :return: A response denoting that the account is being created. - :rtype: str - """ - # attempt to create a user - cic_eth_api = Api(callback_task='cic_ussd.tasks.callback_handler.process_account_creation_callback', - callback_queue='cic-ussd', - callback_param='', - chain_str=chain_str) - creation_task_id = cic_eth_api.create_account().id - - # record task initiation time - add_tasks_to_tracker(task_uuid=creation_task_id, session=session) - - # cache account creation data - cache_account_creation_task_id(phone_number=phone_number, task_id=creation_task_id) - - # find menu to notify user account is being created - current_menu = UssdMenu.find_by_name(name='account_creation_prompt') - - # create a ussd session session - create_or_update_session( - external_session_id=external_session_id, - phone=phone_number, - service_code=service_code, - current_menu=current_menu.get('name'), - session=session, - user_input=user_input) - - # define response to relay to user - response = define_multilingual_responses( - key='ussd.kenya.account_creation_prompt', locales=['en', 'sw'], prefix='END') - return response - - -def define_multilingual_responses(key: str, locales: list, prefix: str, **kwargs): - """This function returns responses in multiple languages in the interest of enabling responses in more than one - language. - :param key: The key to access some text value from the translation files. - :type key: str - :param locales: A list of the locales to translate the text value to. - :type locales: list - :param prefix: The prefix for the text value either: (CON|END) - :type prefix: str - :param kwargs: Other arguments to be passed to the translator - :type kwargs: kwargs - :return: A string of the text value in multiple languages. - :rtype: str - """ - prefix = prefix.upper() - response = f'{prefix} ' - for locale in locales: - response += i18n.t(key=key, locale=locale, **kwargs) - response += '\n' - return response - - -def persist_session_to_db_task(external_session_id: str, queue: str): - """ - This function creates a signature matching the persist session to db task and runs the task asynchronously. - :param external_session_id: Session id value provided by AT - :type external_session_id: str - :param queue: Celery queue on which task should run - :type queue: str - """ - s_persist_session_to_db = celery.signature( - 'cic_ussd.tasks.ussd_session.persist_session_to_db', - [external_session_id] - ) - s_persist_session_to_db.apply_async(queue=queue) - - -def cache_account_creation_task_id(phone_number: str, task_id: str): - """This function stores the task id that is returned from a task spawned to create a blockchain account in the redis - cache. - :param phone_number: The phone number for the user whose account is being created. - :type phone_number: str - :param task_id: A celery task id - :type task_id: str - """ - redis_cache = InMemoryStore.cache - account_creation_request_data = { - 'phone_number': phone_number, - 'sms_notification_sent': False, - 'status': 'PENDING', - 'task_id': task_id, - } - redis_cache.set(task_id, json.dumps(account_creation_request_data)) - redis_cache.persist(name=task_id) - - -def process_current_menu(account: Account, session: Session, ussd_session: Optional[dict], user_input: str) -> Document: - """This function checks user input and returns a corresponding ussd menu - :param ussd_session: An in db ussd session object. - :type ussd_session: UssdSession - :param account: A account object. - :type account: Account - :param session: - :type session: - :param user_input: The user's input. - :type user_input: str - :return: An in memory ussd menu object. - :rtype: Document - """ - # handle invalid inputs - if ussd_session and user_input == "": - current_menu = UssdMenu.find_by_name(name='exit_invalid_input') - else: - # get current state - latest_input = get_latest_input(user_input=user_input) - session = SessionBase.bind_session(session=session) - current_menu = process_request( - account=account, - session=session, - ussd_session=ussd_session, - user_input=latest_input) - SessionBase.release_session(session=session) - return current_menu - - -def process_menu_interaction_requests(chain_str: str, - external_session_id: str, - phone_number: str, - queue: str, - service_code: str, - session, - user_input: str) -> str: - """This function handles requests intended for interaction with ussd menu, it checks whether a user matching the - provided phone number exists and in the absence of which it creates an account for the user. - In the event that a user exists it processes the request and returns an appropriate response. - :param chain_str: The chain name and network id. - :type chain_str: str - :param external_session_id: Unique session id from AfricasTalking - :type external_session_id: str - :param phone_number: Phone number of the user making the request. - :type phone_number: str - :param queue: The celery queue on which to run tasks - :type queue: str - :param service_code: The service dialed by the user making the request. - :type service_code: str - :param session: - :type session: - :param user_input: The inputs entered by the user. - :type user_input: str - :return: A response based on the request received. - :rtype: str - """ - # check whether the user exists - if not check_known_user(phone_number=phone_number, session=session): - response = initiate_account_creation_request(chain_str=chain_str, - external_session_id=external_session_id, - phone_number=phone_number, - service_code=service_code, - session=session, - user_input=user_input) - - else: - # get account - session = SessionBase.bind_session(session=session) - account = Account.get_by_phone_number(phone_number=phone_number, session=session) - - # retrieve and cache user's metadata - blockchain_address = account.blockchain_address - s_query_person_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_person_metadata', - [blockchain_address] - ) - s_query_person_metadata.apply_async(queue='cic-ussd') - - # find any existing ussd session - existing_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first() - - # validate user inputs - if existing_ussd_session: - current_menu = process_current_menu( - account=account, - session=session, - ussd_session=existing_ussd_session.to_json(), - user_input=user_input - ) - else: - current_menu = process_current_menu( - account=account, - session=session, - ussd_session=None, - user_input=user_input - ) - - last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session) - - if last_ussd_session: - # create or update the ussd session as appropriate - ussd_session = create_or_update_session( - external_session_id=external_session_id, - phone=phone_number, - service_code=service_code, - user_input=user_input, - current_menu=current_menu.get('name'), - session=session, - session_data=last_ussd_session.session_data - ) - else: - ussd_session = create_or_update_session( - external_session_id=external_session_id, - phone=phone_number, - service_code=service_code, - user_input=user_input, - current_menu=current_menu.get('name'), - session=session - ) - - # define appropriate response - response = custom_display_text( - account=account, - display_key=current_menu.get('display_key'), - menu_name=current_menu.get('name'), - session=session, - ussd_session=ussd_session.to_json(), - ) - - # check that the response from the processor is valid - if not validate_response_type(processor_response=response): - raise Exception(f'Invalid response: {response}') - - # persist session to db - persist_session_to_db_task(external_session_id=external_session_id, queue=queue) - SessionBase.release_session(session=session) - - return response - - -def reset_pin(phone_number: str, session: Session) -> str: - """Reset account status from Locked to Pending. - :param phone_number: The phone number belonging to the account to be unlocked. - :type phone_number: str - :param session: - :type session: - :return: The status of the pin reset. - :rtype: str - """ - session = SessionBase.bind_session(session=session) - account = Account.get_by_phone_number(phone_number=phone_number, session=session) - account.reset_account_pin() - session.add(account) - session.flush() - SessionBase.release_session(session=session) - - response = f'Pin reset for user {phone_number} is successful!' - return response - - -def update_ussd_session( - ussd_session: InMemoryUssdSession, - user_input: str, - current_menu: str, - session_data: Optional[dict] = None) -> InMemoryUssdSession: - """ - Updates a ussd session - :param ussd_session: Session id value provided by AT - :type ussd_session: InMemoryUssdSession - :param user_input: Input from the request - :type user_input: str - :param current_menu: Menu name that is currently being displayed on the ussd session - :type current_menu: str - :param session_data: Any additional data that was persisted during the user's interaction with the system. - :type session_data: dict. - :return: ussd session object - :rtype: InMemoryUssdSession - """ - if session_data is None: - session_data = ussd_session.session_data - - session = InMemoryUssdSession( - external_session_id=ussd_session.external_session_id, - msisdn=ussd_session.msisdn, - user_input=user_input, - state=current_menu, - service_code=ussd_session.service_code, - session_data=session_data - ) - return session - - -def save_to_in_memory_ussd_session_data(queue: str, session: Session, session_data: dict, ussd_session: dict): - """This function is used to save information to the session data attribute of a ussd session object in the redis - cache. - :param queue: The queue on which the celery task should run. - :type queue: str - :param session: - :type session: - :param session_data: A dictionary containing data for a specific ussd session in redis that needs to be saved - temporarily. - :type session_data: dict - :param ussd_session: A ussd session passed to the state machine. - :type ussd_session: UssdSession - """ - # define redis cache entry point - cache = InMemoryStore.cache - - # get external session id - external_session_id = ussd_session.get('external_session_id') - - # check for existing session data - existing_session_data = ussd_session.get('session_data') - - # merge old session data with new inputs to session data - if existing_session_data: - session_data = {**existing_session_data, **session_data} - - # get corresponding session record - in_redis_ussd_session = cache.get(external_session_id) - in_redis_ussd_session = json.loads(in_redis_ussd_session) - - # create new in memory ussd session with current ussd session data - create_or_update_session( - external_session_id=external_session_id, - phone=in_redis_ussd_session.get('msisdn'), - service_code=in_redis_ussd_session.get('service_code'), - user_input=in_redis_ussd_session.get('user_input'), - current_menu=in_redis_ussd_session.get('state'), - session=session, - session_data=session_data - ) - persist_session_to_db_task(external_session_id=external_session_id, queue=queue) diff --git a/apps/cic-ussd/cic_ussd/phone_number.py b/apps/cic-ussd/cic_ussd/phone_number.py index 1fc94f6f..0bb0a9cb 100644 --- a/apps/cic-ussd/cic_ussd/phone_number.py +++ b/apps/cic-ussd/cic_ussd/phone_number.py @@ -29,9 +29,10 @@ def process_phone_number(phone_number: str, region: str): pass phone_number_object = phonenumbers.parse(phone_number, region) - parsed_phone_number = phonenumbers.format_number(phone_number_object, phonenumbers.PhoneNumberFormat.E164) + return phonenumbers.format_number( + phone_number_object, phonenumbers.PhoneNumberFormat.E164 + ) - return parsed_phone_number class Support: phone_number = None diff --git a/apps/cic-ussd/cic_ussd/processor.py b/apps/cic-ussd/cic_ussd/processor.py deleted file mode 100644 index 83dbbb74..00000000 --- a/apps/cic-ussd/cic_ussd/processor.py +++ /dev/null @@ -1,562 +0,0 @@ -# standard imports -import datetime -import logging -import json -from typing import Optional - -# third party imports -from sqlalchemy import desc -from cic_eth.api import Api -from sqlalchemy.orm.session import Session -from tinydb.table import Document - -# local imports -from cic_ussd.account import define_account_tx_metadata, retrieve_account_statement -from cic_ussd.balance import compute_operational_balance, get_balances, get_cached_operational_balance -from cic_ussd.chain import Chain -from cic_ussd.db.models.account import Account -from cic_ussd.db.models.base import SessionBase -from cic_ussd.db.models.ussd_session import UssdSession -from cic_ussd.db.enum import AccountStatus -from cic_ussd.error import SeppukuError -from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.phone_number import Support -from cic_ussd.redis import cache_data, create_cached_data_key, get_cached_data -from cic_ussd.state_machine import UssdStateMachine -from cic_ussd.conversions import to_wei, from_wei -from cic_ussd.translation import translation_for -from cic_types.models.person import generate_metadata_pointer, get_contact_data_from_vcard - -logg = logging.getLogger(__name__) - - -def get_default_token_data(): - chain_str = Chain.spec.__str__() - cic_eth_api = Api(chain_str=chain_str) - default_token_request_task = cic_eth_api.default_token() - default_token_data = default_token_request_task.get() - return default_token_data - - -def retrieve_token_symbol(chain_str: str = Chain.spec.__str__()): - """ - :param chain_str: - :type chain_str: - :return: - :rtype: - """ - cache_key = create_cached_data_key( - identifier=chain_str.encode('utf-8'), - salt=':cic.default_token_data' - ) - cached_data = get_cached_data(key=cache_key) - if cached_data: - default_token_data = json.loads(cached_data) - return default_token_data.get('symbol') - else: - logg.warning('Cached default token data not found. Attempting retrieval from default token API') - default_token_data = get_default_token_data() - if default_token_data: - return default_token_data.get('symbol') - else: - raise SeppukuError(f'Could not retrieve default token for: {chain_str}') - - -def process_pin_authorization(account: Account, display_key: str, **kwargs) -> str: - """This method provides translation for all ussd menu entries that follow the pin authorization pattern. - :param account: The account in a running USSD session. - :type account: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :param kwargs: Any additional information required by the text values in the internationalization files. - :type kwargs - :return: A string value corresponding the ussd menu's text value. - :rtype: str - """ - remaining_attempts = 3 - if account.failed_pin_attempts > 0: - return translation_for( - key=f'{display_key}.retry', - preferred_language=account.preferred_language, - remaining_attempts=(remaining_attempts - account.failed_pin_attempts) - ) - else: - return translation_for( - key=f'{display_key}.first', - preferred_language=account.preferred_language, - **kwargs - ) - - -def process_exit_insufficient_balance(account: Account, display_key: str, session: Session, ussd_session: dict): - """This function processes the exit menu letting users their account balance is insufficient to perform a specific - transaction. - :param account: The account requesting access to the ussd menu. - :type account: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :param session: - :type session: - :param ussd_session: A JSON serialized in-memory ussd session object - :type ussd_session: dict - :return: Corresponding translation text response - :rtype: str - """ - # get account balance - operational_balance = get_cached_operational_balance(blockchain_address=account.blockchain_address) - - # compile response data - user_input = ussd_session.get('user_input').split('*')[-1] - transaction_amount = to_wei(value=int(user_input)) - - # get default data - token_symbol = retrieve_token_symbol() - - recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) - - tx_recipient_information = define_account_tx_metadata(user=recipient) - - return translation_for( - key=display_key, - preferred_language=account.preferred_language, - amount=from_wei(transaction_amount), - token_symbol=token_symbol, - recipient_information=tx_recipient_information, - token_balance=operational_balance - ) - - -def process_exit_successful_transaction(account: Account, display_key: str, session: Session, ussd_session: dict): - """This function processes the exit menu after a successful initiation for a transfer of tokens. - :param account: The account requesting access to the ussd menu. - :type account: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :param session: - :type session: - :param ussd_session: A JSON serialized in-memory ussd session object - :type ussd_session: dict - :return: Corresponding translation text response - :rtype: str - """ - transaction_amount = to_wei(int(ussd_session.get('session_data').get('transaction_amount'))) - token_symbol = retrieve_token_symbol() - recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) - tx_recipient_information = define_account_tx_metadata(user=recipient) - tx_sender_information = define_account_tx_metadata(user=account) - - return translation_for( - key=display_key, - preferred_language=account.preferred_language, - transaction_amount=from_wei(transaction_amount), - token_symbol=token_symbol, - recipient_information=tx_recipient_information, - sender_information=tx_sender_information - ) - - -def process_transaction_pin_authorization(account: Account, display_key: str, session: Session, ussd_session: dict): - """This function processes pin authorization where making a transaction is concerned. It constructs a - pre-transaction response menu that shows the details of the transaction. - :param account: The account requesting access to the ussd menu. - :type account: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :param session: - :type session: - :param ussd_session: The USSD session determining what user data needs to be extracted and added to the menu's - text values. - :type ussd_session: UssdSession - :return: Corresponding translation text response - :rtype: str - """ - # compile response data - recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') - recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) - tx_recipient_information = define_account_tx_metadata(user=recipient) - tx_sender_information = define_account_tx_metadata(user=account) - - token_symbol = retrieve_token_symbol() - user_input = ussd_session.get('session_data').get('transaction_amount') - transaction_amount = to_wei(value=int(user_input)) - return process_pin_authorization( - account=account, - display_key=display_key, - recipient_information=tx_recipient_information, - transaction_amount=from_wei(transaction_amount), - token_symbol=token_symbol, - sender_information=tx_sender_information - ) - - -def process_account_balances(user: Account, display_key: str): - """ - :param user: - :type user: - :param display_key: - :type display_key: - :return: - :rtype: - """ - # retrieve cached balance - operational_balance = get_cached_operational_balance(blockchain_address=user.blockchain_address) - - logg.debug('Requires call to retrieve tax and bonus amounts') - tax = '' - bonus = '' - token_symbol = retrieve_token_symbol() - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - operational_balance=operational_balance, - tax=tax, - bonus=bonus, - token_symbol=token_symbol - ) - - -def format_transactions(transactions: list, preferred_language: str, token_symbol: str): - - formatted_transactions = '' - if len(transactions) > 0: - for transaction in transactions: - recipient_phone_number = transaction.get('recipient_phone_number') - sender_phone_number = transaction.get('sender_phone_number') - value = transaction.get('to_value') - timestamp = transaction.get('timestamp') - action_tag = transaction.get('action_tag') - direction = transaction.get('direction') - token_symbol = token_symbol - - if action_tag == 'SENT' or action_tag == 'ULITUMA': - formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {recipient_phone_number} {timestamp}.\n' - else: - formatted_transactions += f'{action_tag} {value} {token_symbol} {direction} {sender_phone_number} {timestamp}. \n' - return formatted_transactions - else: - if preferred_language == 'en': - formatted_transactions = 'NO TRANSACTION HISTORY' - else: - formatted_transactions = 'HAMNA RIPOTI YA MATUMIZI' - return formatted_transactions - - -def process_display_user_metadata(user: Account, display_key: str): - """ - :param user: - :type user: - :param display_key: - :type display_key: - """ - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - cached_metadata = get_cached_data(key) - if cached_metadata: - user_metadata = json.loads(cached_metadata) - contact_data = get_contact_data_from_vcard(vcard=user_metadata.get('vcard')) - logg.debug(f'{contact_data}') - full_name = f'{contact_data.get("given")} {contact_data.get("family")}' - date_of_birth = user_metadata.get('date_of_birth') - year_of_birth = date_of_birth.get('year') - present_year = datetime.datetime.now().year - age = present_year - year_of_birth - gender = user_metadata.get('gender') - products = ', '.join(user_metadata.get('products')) - location = user_metadata.get('location').get('area_name') - - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - full_name=full_name, - age=age, - gender=gender, - location=location, - products=products - ) - else: - # TODO [Philip]: All these translations could be moved to translation files. - logg.warning(f'Expected person metadata but found none in cache for key: {key}') - - absent = '' - if user.preferred_language == 'en': - absent = 'Not provided' - elif user.preferred_language == 'sw': - absent = 'Haijawekwa' - - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - full_name=absent, - gender=absent, - age=absent, - location=absent, - products=absent - ) - - -def process_account_statement(user: Account, display_key: str): - """ - :param user: - :type user: - :param display_key: - :type display_key: - :return: - :rtype: - """ - # retrieve cached statement - identifier = blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address) - key = create_cached_data_key(identifier=identifier, salt=':cic.statement') - transactions = get_cached_data(key=key) - - token_symbol = retrieve_token_symbol() - - first_transaction_set = [] - middle_transaction_set = [] - last_transaction_set = [] - - transactions = json.loads(transactions) - - if len(transactions) > 6: - last_transaction_set += transactions[6:] - middle_transaction_set += transactions[3:][:3] - first_transaction_set += transactions[:3] - # there are probably much cleaner and operational inexpensive ways to do this so find them - elif 3 < len(transactions) < 7: - middle_transaction_set += transactions[3:] - first_transaction_set += transactions[:3] - else: - first_transaction_set += transactions[:3] - - if display_key == 'ussd.kenya.first_transaction_set': - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - first_transaction_set=format_transactions( - transactions=first_transaction_set, - preferred_language=user.preferred_language, - token_symbol=token_symbol - ) - ) - elif display_key == 'ussd.kenya.middle_transaction_set': - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - middle_transaction_set=format_transactions( - transactions=middle_transaction_set, - preferred_language=user.preferred_language, - token_symbol=token_symbol - ) - ) - - elif display_key == 'ussd.kenya.last_transaction_set': - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - last_transaction_set=format_transactions( - transactions=last_transaction_set, - preferred_language=user.preferred_language, - token_symbol=token_symbol - ) - ) - - -def process_start_menu(display_key: str, user: Account): - """This function gets data on an account's balance and token in order to append it to the start of the start menu's - title. It passes said arguments to the translation function and returns the appropriate corresponding text from the - translation files. - :param user: The user requesting access to the ussd menu. - :type user: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :return: Corresponding translation text response - :rtype: str - """ - token_symbol = retrieve_token_symbol() - chain_str = Chain.spec.__str__() - blockchain_address = user.blockchain_address - - # get balances synchronously for display on start menu - balances_data = get_balances(address=blockchain_address, chain_str=chain_str, token_symbol=token_symbol) - - key = create_cached_data_key( - identifier=bytes.fromhex(blockchain_address[2:]), - salt=':cic.balances_data' - ) - cache_data(key=key, data=json.dumps(balances_data)) - - # get operational balance - operational_balance = compute_operational_balance(balances=balances_data) - - # retrieve and cache account's statement - retrieve_account_statement(blockchain_address=blockchain_address) - - return translation_for( - key=display_key, - preferred_language=user.preferred_language, - account_balance=operational_balance, - account_token_name=token_symbol - ) - - -def retrieve_most_recent_ussd_session(phone_number: str, session: Session) -> UssdSession: - # get last ussd session based on user phone number - session = SessionBase.bind_session(session=session) - last_ussd_session = session.query(UssdSession)\ - .filter_by(msisdn=phone_number)\ - .order_by(desc(UssdSession.created))\ - .first() - SessionBase.release_session(session=session) - return last_ussd_session - - -def process_request(account: Account, session, user_input: str, ussd_session: Optional[dict] = None) -> Document: - """This function assesses a request based on the user from the request comes, the session_id and the user's - input. It determines whether the request translates to a return to an existing session by checking whether the - provided session id exists in the database or whether the creation of a new ussd session object is warranted. - It then returns the appropriate ussd menu text values. - :param account: The account requesting access to the ussd menu. - :type account: Account - :param session: - :type session: - :param user_input: The value a user enters in the ussd menu. - :type user_input: str - :param ussd_session: A JSON serialized in-memory ussd session object - :type ussd_session: dict - :return: A ussd menu's corresponding text value. - :rtype: Document - """ - - if ussd_session: - if user_input == "0": - return UssdMenu.parent_menu(menu_name=ussd_session.get('state')) - else: - successive_state = next_state( - account=account, - session=session, - ussd_session=ussd_session, - user_input=user_input) - return UssdMenu.find_by_name(name=successive_state) - else: - if account.has_valid_pin(): - last_ussd_session = retrieve_most_recent_ussd_session(phone_number=account.phone_number, session=session) - - if last_ussd_session: - # get last state - last_state = last_ussd_session.state - # if last state is account_creation_prompt and metadata exists, show start menu - if last_state in [ - 'account_creation_prompt', - 'exit', - 'exit_invalid_pin', - 'exit_invalid_new_pin', - 'exit_pin_mismatch', - 'exit_invalid_request', - 'exit_successful_transaction' - ]: - return UssdMenu.find_by_name(name='start') - else: - return UssdMenu.find_by_name(name=last_state) - else: - if account.failed_pin_attempts >= 3 and account.get_account_status() == AccountStatus.LOCKED.name: - return UssdMenu.find_by_name(name='exit_pin_blocked') - elif account.preferred_language is None: - return UssdMenu.find_by_name(name='initial_language_selection') - else: - return UssdMenu.find_by_name(name='initial_pin_entry') - - -def next_state(account: Account, session, ussd_session: dict, user_input: str) -> str: - """This function navigates the state machine based on the ussd session object and user inputs it receives. - It checks the user input and provides the successive state in the state machine. It then updates the session's - state attribute with the new state. - :param account: The account requesting access to the ussd menu. - :type account: Account - :param session: - :type session: - :param ussd_session: A JSON serialized in-memory ussd session object - :type ussd_session: dict - :param user_input: The value a user enters in the ussd menu. - :type user_input: str - :return: A string value corresponding the successive give a specific state in the state machine. - """ - state_machine = UssdStateMachine(ussd_session=ussd_session) - state_machine.scan_data((user_input, ussd_session, account, session)) - new_state = state_machine.state - - return new_state - - -def process_exit_invalid_menu_option(display_key: str, preferred_language: str): - return translation_for( - key=display_key, - preferred_language=preferred_language, - support_phone=Support.phone_number - ) - - -def custom_display_text( - account: Account, - display_key: str, - menu_name: str, - session: Session, - ussd_session: dict) -> str: - """This function extracts the appropriate session data based on the current menu name. It then inserts them as - keywords in the i18n function. - :param account: The account in a running USSD session. - :type account: Account - :param display_key: The path in the translation files defining an appropriate ussd response - :type display_key: str - :param menu_name: The name by which a specific menu can be identified. - :type menu_name: str - :param session: - :type session: - :param ussd_session: A JSON serialized in-memory ussd session object - :type ussd_session: dict - :return: A string value corresponding the ussd menu's text value. - :rtype: str - """ - if menu_name == 'transaction_pin_authorization': - return process_transaction_pin_authorization( - account=account, - display_key=display_key, - session=session, - ussd_session=ussd_session) - elif menu_name == 'exit_insufficient_balance': - return process_exit_insufficient_balance( - account=account, - display_key=display_key, - session=session, - ussd_session=ussd_session) - elif menu_name == 'exit_successful_transaction': - return process_exit_successful_transaction( - account=account, - display_key=display_key, - session=session, - ussd_session=ussd_session) - elif menu_name == 'start': - return process_start_menu(display_key=display_key, user=account) - elif 'pin_authorization' in menu_name: - return process_pin_authorization( - account=account, - display_key=display_key, - session=session) - elif 'enter_current_pin' in menu_name: - return process_pin_authorization( - account=account, - display_key=display_key, - session=session) - elif menu_name == 'account_balances': - return process_account_balances(display_key=display_key, user=account) - elif 'transaction_set' in menu_name: - return process_account_statement(display_key=display_key, user=account) - elif menu_name == 'display_user_metadata': - return process_display_user_metadata(display_key=display_key, user=account) - elif menu_name == 'exit_invalid_menu_option': - return process_exit_invalid_menu_option(display_key=display_key, preferred_language=account.preferred_language) - else: - return translation_for(key=display_key, preferred_language=account.preferred_language) diff --git a/apps/cic-ussd/cic_ussd/processor/__init__.py b/apps/cic-ussd/cic_ussd/processor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-ussd/cic_ussd/processor/menu.py b/apps/cic-ussd/cic_ussd/processor/menu.py new file mode 100644 index 00000000..9896c9e1 --- /dev/null +++ b/apps/cic-ussd/cic_ussd/processor/menu.py @@ -0,0 +1,305 @@ +# standard imports +import json +import logging + +# external imports +import i18n.config +from sqlalchemy.orm.session import Session + +# local imports +from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance +from cic_ussd.account.chain import Chain +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.account.statement import ( + get_cached_statement, + parse_statement_transactions, + query_statement, + statement_transaction_set +) +from cic_ussd.account.transaction import from_wei, to_wei +from cic_ussd.account.tokens import get_default_token_symbol +from cic_ussd.cache import cache_data_key, cache_data +from cic_ussd.db.models.account import Account +from cic_ussd.metadata import PersonMetadata +from cic_ussd.phone_number import Support +from cic_ussd.processor.util import latest_input, parse_person_metadata +from cic_ussd.translation import translation_for + +logg = logging.getLogger(__name__) + + +class MenuProcessor: + def __init__(self, account: Account, display_key: str, menu_name: str, session: Session, ussd_session: dict): + self.account = account + self.display_key = display_key + self.identifier = bytes.fromhex(self.account.blockchain_address[2:]) + self.menu_name = menu_name + self.session = session + self.ussd_session = ussd_session + + def account_balances(self) -> str: + """ + :return: + :rtype: + """ + available_balance = get_cached_available_balance(self.account.blockchain_address) + logg.debug('Requires call to retrieve tax and bonus amounts') + tax = '' + bonus = '' + token_symbol = get_default_token_symbol() + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + return translation_for( + key=self.display_key, + preferred_language=preferred_language, + available_balance=available_balance, + tax=tax, + bonus=bonus, + token_symbol=token_symbol + ) + + def account_statement(self) -> str: + """ + :return: + :rtype: + """ + cached_statement = get_cached_statement(self.account.blockchain_address) + statement = json.loads(cached_statement) + statement_transactions = parse_statement_transactions(statement) + transaction_sets = [statement_transactions[tx:tx+3] for tx in range(0, len(statement_transactions), 3)] + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + first_transaction_set = [] + middle_transaction_set = [] + last_transaction_set = [] + if transaction_sets: + first_transaction_set = statement_transaction_set(preferred_language, transaction_sets[0]) + if len(transaction_sets) >= 2: + middle_transaction_set = statement_transaction_set(preferred_language, transaction_sets[1]) + if len(transaction_sets) >= 3: + last_transaction_set = statement_transaction_set(preferred_language, transaction_sets[2]) + if self.display_key == 'ussd.kenya.first_transaction_set': + return translation_for( + self.display_key, preferred_language, first_transaction_set=first_transaction_set + ) + if self.display_key == 'ussd.kenya.middle_transaction_set': + return translation_for( + self.display_key, preferred_language, middle_transaction_set=middle_transaction_set + ) + if self.display_key == 'ussd.kenya.last_transaction_set': + return translation_for( + self.display_key, preferred_language, last_transaction_set=last_transaction_set + ) + + def help(self): + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + return translation_for(self.display_key, preferred_language, support_phone=Support.phone_number) + + def person_metadata(self) -> str: + """ + :return: + :rtype: + """ + person_metadata = PersonMetadata(self.identifier) + cached_person_metadata = person_metadata.get_cached_metadata() + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + if cached_person_metadata: + return parse_person_metadata(cached_person_metadata, self.display_key, preferred_language) + absent = translation_for('helpers.not_provided', preferred_language) + return translation_for( + self.display_key, + preferred_language, + full_name=absent, + gender=absent, + age=absent, + location=absent, + products=absent + ) + + def pin_authorization(self, **kwargs) -> str: + """ + :param kwargs: + :type kwargs: + :return: + :rtype: + """ + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + if self.account.failed_pin_attempts == 0: + return translation_for(f'{self.display_key}.first', preferred_language, **kwargs) + + remaining_attempts = 3 + remaining_attempts -= self.account.failed_pin_attempts + retry_pin_entry = translation_for( + 'ussd.kenya.retry_pin_entry', preferred_language, remaining_attempts=remaining_attempts + ) + return translation_for( + f'{self.display_key}.retry', preferred_language, retry_pin_entry=retry_pin_entry + ) + + def start_menu(self): + """ + :return: + :rtype: + """ + token_symbol = get_default_token_symbol() + blockchain_address = self.account.blockchain_address + balances = get_balances(blockchain_address, Chain.spec.__str__(), token_symbol, False)[0] + key = cache_data_key(self.identifier, ':cic.balances') + cache_data(key, json.dumps(balances)) + available_balance = calculate_available_balance(balances) + + query_statement(blockchain_address) + + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + return translation_for( + self.display_key, preferred_language, account_balance=available_balance, account_token_name=token_symbol + ) + + def transaction_pin_authorization(self) -> str: + """ + :return: + :rtype: + """ + recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number') + recipient = Account.get_by_phone_number(recipient_phone_number, self.session) + tx_recipient_information = recipient.standard_metadata_id() + tx_sender_information = self.account.standard_metadata_id() + token_symbol = get_default_token_symbol() + user_input = self.ussd_session.get('data').get('transaction_amount') + transaction_amount = to_wei(value=int(user_input)) + return self.pin_authorization( + recipient_information=tx_recipient_information, + transaction_amount=from_wei(transaction_amount), + token_symbol=token_symbol, + sender_information=tx_sender_information + ) + + def exit_insufficient_balance(self): + """ + :return: + :rtype: + """ + available_balance = get_cached_available_balance(self.account.blockchain_address) + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + session_data = self.ussd_session.get('data') + transaction_amount = session_data.get('transaction_amount') + transaction_amount = to_wei(value=int(transaction_amount)) + token_symbol = get_default_token_symbol() + recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number') + recipient = Account.get_by_phone_number(recipient_phone_number, self.session) + tx_recipient_information = recipient.standard_metadata_id() + return translation_for( + self.display_key, + preferred_language, + amount=from_wei(transaction_amount), + token_symbol=token_symbol, + recipient_information=tx_recipient_information, + token_balance=available_balance + ) + + def exit_invalid_menu_option(self): + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + return translation_for(self.display_key, preferred_language, support_phone=Support.phone_number) + + def exit_pin_blocked(self): + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + return translation_for('ussd.kenya.exit_pin_blocked', preferred_language, support_phone=Support.phone_number) + + def exit_successful_transaction(self): + """ + :return: + :rtype: + """ + amount = int(self.ussd_session.get('data').get('transaction_amount')) + preferred_language = get_cached_preferred_language(self.account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + transaction_amount = to_wei(amount) + token_symbol = get_default_token_symbol() + recipient_phone_number = self.ussd_session.get('data').get('recipient_phone_number') + recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=self.session) + tx_recipient_information = recipient.standard_metadata_id() + tx_sender_information = self.account.standard_metadata_id() + return translation_for( + self.display_key, + preferred_language, + transaction_amount=from_wei(transaction_amount), + token_symbol=token_symbol, + recipient_information=tx_recipient_information, + sender_information=tx_sender_information + ) + + +def response(account: Account, display_key: str, menu_name: str, session: Session, ussd_session: dict) -> str: + """This function extracts the appropriate session data based on the current menu name. It then inserts them as + keywords in the i18n function. + :param account: The account in a running USSD session. + :type account: Account + :param display_key: The path in the translation files defining an appropriate ussd response + :type display_key: str + :param menu_name: The name by which a specific menu can be identified. + :type menu_name: str + :param session: + :type session: + :param ussd_session: A JSON serialized in-memory ussd session object + :type ussd_session: dict + :return: A string value corresponding the ussd menu's text value. + :rtype: str + """ + menu_processor = MenuProcessor(account, display_key, menu_name, session, ussd_session) + + if menu_name == 'start': + return menu_processor.start_menu() + + if menu_name == 'help': + return menu_processor.help() + + if menu_name == 'transaction_pin_authorization': + return menu_processor.transaction_pin_authorization() + + if menu_name == 'exit_insufficient_balance': + return menu_processor.exit_insufficient_balance() + + if menu_name == 'exit_successful_transaction': + return menu_processor.exit_successful_transaction() + + if menu_name == 'account_balances': + return menu_processor.account_balances() + + if 'pin_authorization' in menu_name: + return menu_processor.pin_authorization() + + if 'enter_current_pin' in menu_name: + return menu_processor.pin_authorization() + + if 'transaction_set' in menu_name: + return menu_processor.account_statement() + + if menu_name == 'display_user_metadata': + return menu_processor.person_metadata() + + if menu_name == 'exit_invalid_menu_option': + return menu_processor.exit_invalid_menu_option() + + if menu_name == 'exit_pin_blocked': + return menu_processor.exit_pin_blocked() + + preferred_language = get_cached_preferred_language(account.blockchain_address) + + return translation_for(display_key, preferred_language) diff --git a/apps/cic-ussd/cic_ussd/processor/ussd.py b/apps/cic-ussd/cic_ussd/processor/ussd.py new file mode 100644 index 00000000..8e383d9b --- /dev/null +++ b/apps/cic-ussd/cic_ussd/processor/ussd.py @@ -0,0 +1,185 @@ +# standard imports +from typing import Optional + +# external imports +import celery +import i18n +from sqlalchemy.orm.session import Session +from tinydb.table import Document + +# local imports +from cic_ussd.db.models.account import Account, create +from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.models.ussd_session import UssdSession +from cic_ussd.menu.ussd_menu import UssdMenu +from cic_ussd.processor.menu import response +from cic_ussd.processor.util import latest_input, resume_last_ussd_session +from cic_ussd.session.ussd_session import create_or_update_session, persist_ussd_session +from cic_ussd.state_machine import UssdStateMachine +from cic_ussd.translation import translation_for +from cic_ussd.validator import is_valid_response + + +def handle_menu(account: Account, session: Session) -> Document: + """ + :param account: + :type account: + :param session: + :type session: + :return: + :rtype: + """ + if account.pin_is_blocked(session): + return UssdMenu.find_by_name('exit_pin_blocked') + + if account.has_valid_pin(session): + last_ussd_session = UssdSession.last_ussd_session(account.phone_number, session) + if last_ussd_session: + return resume_last_ussd_session(last_ussd_session.state) + + elif not account.has_preferred_language(): + return UssdMenu.find_by_name('initial_language_selection') + else: + return UssdMenu.find_by_name('initial_pin_entry') + + +def get_menu(account: Account, + session: Session, + user_input: str, + ussd_session: Optional[dict]) -> Document: + """ + :param account: + :type account: + :param session: + :type session: + :param user_input: + :type user_input: + :param ussd_session: + :type ussd_session: + :return: + :rtype: + """ + user_input = latest_input(user_input) + if not ussd_session: + return handle_menu(account, session) + if user_input == '': + return UssdMenu.find_by_name(name='exit_invalid_input') + if user_input == '0': + return UssdMenu.parent_menu(ussd_session.get('state')) + session = SessionBase.bind_session(session) + state = next_state(account, session, user_input, ussd_session) + return UssdMenu.find_by_name(state) + + +def handle_menu_operations(chain_str: str, + external_session_id: str, + phone_number: str, + queue: str, + service_code: str, + session, + user_input: str): + """ + :param chain_str: + :type chain_str: + :param external_session_id: + :type external_session_id: + :param phone_number: + :type phone_number: + :param queue: + :type queue: + :param service_code: + :type service_code: + :param session: + :type session: + :param user_input: + :type user_input: + :return: + :rtype: + """ + session = SessionBase.bind_session(session=session) + account: Account = Account.get_by_phone_number(phone_number, session) + if account: + return handle_account_menu_operations(account, external_session_id, queue, session, service_code, user_input) + create(chain_str, phone_number, session) + menu = UssdMenu.find_by_name('account_creation_prompt') + preferred_language = i18n.config.get('fallback') + create_or_update_session( + external_session_id=external_session_id, + msisdn=phone_number, + service_code=service_code, + state=menu.get('name'), + session=session, + user_input=user_input) + persist_ussd_session(external_session_id, queue) + return translation_for('ussd.kenya.account_creation_prompt', preferred_language) + + +def handle_account_menu_operations(account: Account, + external_session_id: str, + queue: str, + session: Session, + service_code: str, + user_input: str): + """ + :param account: + :type account: + :param external_session_id: + :type external_session_id: + :param queue: + :type queue: + :param session: + :type session: + :param service_code: + :type service_code: + :param user_input: + :type user_input: + :return: + :rtype: + """ + phone_number = account.phone_number + s_query_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_person_metadata', [account.blockchain_address], queue='cic-ussd') + s_query_person_metadata.apply_async() + s_query_preferences_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_preferences_metadata', [account.blockchain_address], queue='cic-ussd') + s_query_preferences_metadata.apply_async() + existing_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first() + last_ussd_session = UssdSession.last_ussd_session(phone_number, session) + if existing_ussd_session: + menu = get_menu(account, session, user_input, existing_ussd_session.to_json()) + else: + menu = get_menu(account, session, user_input, None) + if last_ussd_session: + ussd_session = create_or_update_session( + external_session_id, phone_number, service_code, user_input, menu.get('name'), session, + last_ussd_session.data + ) + else: + ussd_session = create_or_update_session( + external_session_id, phone_number, service_code, user_input, menu.get('name'), session, None + ) + menu_response = response( + account, menu.get('display_key'), menu.get('name'), session, ussd_session.to_json() + ) + if not is_valid_response(menu_response): + raise ValueError(f'Invalid response: {response}') + persist_ussd_session(external_session_id, queue) + return menu_response + + +def next_state(account: Account, session, user_input: str, ussd_session: dict) -> str: + """ + :param account: + :type account: + :param session: + :type session: + :param user_input: + :type user_input: + :param ussd_session: + :type ussd_session: + :return: + :rtype: + """ + state_machine = UssdStateMachine(ussd_session=ussd_session) + state_machine.scan_data((user_input, ussd_session, account, session)) + return state_machine.state diff --git a/apps/cic-ussd/cic_ussd/processor/util.py b/apps/cic-ussd/cic_ussd/processor/util.py new file mode 100644 index 00000000..fc8ea38e --- /dev/null +++ b/apps/cic-ussd/cic_ussd/processor/util.py @@ -0,0 +1,77 @@ +# standard imports +import datetime +import json + +# external imports +from cic_types.models.person import get_contact_data_from_vcard +from tinydb.table import Document + +# local imports +from cic_ussd.menu.ussd_menu import UssdMenu +from cic_ussd.translation import translation_for + + +def latest_input(user_input: str) -> str: + """ + :param user_input: + :type user_input: + :return: + :rtype: + """ + return user_input.split('*')[-1] + + +def parse_person_metadata(cached_metadata: str, display_key: str, preferred_language: str) -> str: + """This function extracts person metadata formatted to suite display on the ussd interface. + :param cached_metadata: Person metadata JSON str. + :type cached_metadata: str + :param display_key: Path to an entry in menu data in translation files. + :type display_key: str + :param preferred_language: An account's set preferred language. + :type preferred_language: str + :return: + :rtype: + """ + user_metadata = json.loads(cached_metadata) + contact_data = get_contact_data_from_vcard(user_metadata.get('vcard')) + full_name = f'{contact_data.get("given")} {contact_data.get("family")}' + date_of_birth = user_metadata.get('date_of_birth') + year_of_birth = date_of_birth.get('year') + present_year = datetime.datetime.now().year + age = present_year - year_of_birth + gender = user_metadata.get('gender') + products = ', '.join(user_metadata.get('products')) + location = user_metadata.get('location').get('area_name') + + return translation_for( + key=display_key, + preferred_language=preferred_language, + full_name=full_name, + age=age, + gender=gender, + location=location, + products=products + ) + + +def resume_last_ussd_session(last_state: str) -> Document: + """ + :param last_state: + :type last_state: + :return: + :rtype: + """ + # TODO [Philip]: This can be cleaned further + non_reusable_states = [ + 'account_creation_prompt', + 'exit', + 'exit_invalid_pin', + 'exit_invalid_new_pin', + 'exit_invalid_request', + 'exit_pin_blocked', + 'exit_pin_mismatch', + 'exit_successful_transaction' + ] + if last_state in non_reusable_states: + return UssdMenu.find_by_name('start') + return UssdMenu.find_by_name(last_state) diff --git a/apps/cic-ussd/cic_ussd/requests.py b/apps/cic-ussd/cic_ussd/requests.py deleted file mode 100644 index 897d8539..00000000 --- a/apps/cic-ussd/cic_ussd/requests.py +++ /dev/null @@ -1,143 +0,0 @@ -# standard imports -from typing import Optional, Tuple, Union -import json -import logging -import re -from typing import Optional, Union -from urllib.parse import urlparse, parse_qs - -# third-party imports -from sqlalchemy import desc -from sqlalchemy.orm.session import Session - -# local imports -from cic_ussd.db.models.account import Account -from cic_ussd.db.models.base import SessionBase -from cic_ussd.db.enum import AccountStatus -from cic_ussd.operations import get_account_status, reset_pin -from cic_ussd.validator import check_known_user - - -logg = logging.getLogger(__file__) - - -def get_query_parameters(env: dict, query_name: Optional[str] = None) -> Union[dict, str]: - """Gets value of the request query parameters. - :param env: Object containing server and request information. - :type env: dict - :param query_name: The specific query parameter to fetch. - :type query_name: str - :return: Query parameters from the request. - :rtype: dict | str - """ - parsed_url = urlparse(env.get('REQUEST_URI')) - params = parse_qs(parsed_url.query) - if query_name: - param = params.get(query_name)[0] - return param - return params - - -def get_request_endpoint(env: dict) -> str: - """Gets value of the request url path. - :param env: Object containing server and request information - :type env: dict - :return: Endpoint that has been touched by the call - :rtype: str - """ - return env.get('PATH_INFO') - - -def get_request_method(env: dict) -> str: - """Gets value of the request method. - :param env: Object containing server and request information. - :type env: dict - :return: Request method. - :rtype: str - """ - return env.get('REQUEST_METHOD').upper() - - -def get_account_creation_callback_request_data(env: dict) -> tuple: - """This function retrieves data from a callback - :param env: Object containing server and request information. - :type env: dict - :return: A tuple containing the status, result and task_id for a celery task spawned to create a blockchain - account. - :rtype: tuple - """ - - callback_data = env.get('wsgi.input') - status = callback_data.get('status') - task_id = callback_data.get('root_id') - result = callback_data.get('result') - - return status, task_id, result - - -def process_pin_reset_requests(env: dict, phone_number: str, session: Session): - """This function processes requests that are responsible for the pin reset functionality. It processes GET and PUT - requests responsible for returning an account's status and - :param env: A dictionary of values representing data sent on the api. - :type env: dict - :param phone_number: The phone of the user whose pin is being reset. - :type phone_number: str - :param session: - :type session: - :return: A response denoting the result of the request to reset the user's pin. - :rtype: str - """ - if not check_known_user(phone_number=phone_number, session=session): - return f'No user matching {phone_number} was found.', '404 Not Found' - - if get_request_method(env) == 'PUT': - return reset_pin(phone_number=phone_number, session=session), '200 OK' - - if get_request_method(env) == 'GET': - status = get_account_status(phone_number=phone_number, session=session) - response = { - 'status': f'{status}' - } - response = json.dumps(response) - return response, '200 OK' - - -def process_locked_accounts_requests(env: dict, session: Session) -> tuple: - """This function authenticates staff requests and returns a serialized JSON formatted list of blockchain addresses - of accounts for which the PIN has been locked due to too many failed attempts. - :param env: A dictionary of values representing data sent on the api. - :type env: dict - :param session: - :type session: - :return: A tuple containing a serialized list of blockchain addresses for locked accounts and corresponding message - for the response. - :rtype: tuple - """ - session = SessionBase.bind_session(session=session) - response = '' - - if get_request_method(env) == 'GET': - offset = 0 - limit = 100 - - locked_accounts_path = r'/accounts/locked/(\d+)?/?(\d+)?' - r = re.match(locked_accounts_path, env.get('PATH_INFO')) - - if r: - if r.lastindex > 1: - offset = r[1] - limit = r[2] - else: - limit = r[1] - - locked_accounts = session.query(Account.blockchain_address).filter( - Account.account_status == AccountStatus.LOCKED.value, - Account.failed_pin_attempts >= 3).order_by(desc(Account.updated)).offset(offset).limit(limit).all() - - # convert lists to scalar blockchain addresses - locked_accounts = [blockchain_address for (blockchain_address, ) in locked_accounts] - - SessionBase.release_session(session=session) - response = json.dumps(locked_accounts) - return response, '200 OK' - return response, '405 Play by the rules' diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py index a995b315..66bfba08 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_server.py @@ -5,7 +5,7 @@ requests offering control of user account states to a staff behind the client. # standard imports import logging -from urllib.parse import quote_plus + # third-party imports from confini import Config @@ -13,12 +13,11 @@ from confini import Config # local imports from cic_ussd.db import dsn_from_config from cic_ussd.db.models.base import SessionBase -from cic_ussd.operations import define_response_with_content -from cic_ussd.requests import (get_request_endpoint, - get_query_parameters, - process_pin_reset_requests, - process_locked_accounts_requests) +from cic_ussd.http.requests import get_request_endpoint +from cic_ussd.http.responses import with_content_headers +from cic_ussd.http.routes import locked_accounts, handle_pin_requests from cic_ussd.runnable.server_base import exportable_parser, logg + args = exportable_parser.parse_args() # define log levels @@ -28,7 +27,7 @@ elif args.v: logging.getLogger().setLevel(logging.INFO) # parse config -config = Config(args.c, env_prefix=args.env_prefix) +config = Config(args.c, args.env_prefix) config.process() config.censor('PASSWORD', 'DATABASE') logg.debug('config loaded from {}:\n{}'.format(args.c, config)) @@ -56,20 +55,13 @@ def application(env, start_response): session = SessionBase.create_session() if get_request_endpoint(env) == '/pin': - phone_number = get_query_parameters(env=env, query_name='phoneNumber') - phone_number = quote_plus(phone_number) - response, message = process_pin_reset_requests(env=env, phone_number=phone_number, session=session) - response_bytes, headers = define_response_with_content(headers=errors_headers, response=response) - session.commit() - session.close() - start_response(message, headers) - return [response_bytes] + return handle_pin_requests(env, session, errors_headers, start_response) - # handle requests for locked accounts - response, message = process_locked_accounts_requests(env=env, session=session) - response_bytes, headers = define_response_with_content(headers=headers, response=response) + response, message = locked_accounts(env, session) + response_bytes, headers = with_content_headers(headers, response) start_response(message, headers) session.commit() session.close() return [response_bytes] + diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py index 8f4cc4a0..c689d02e 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_tasker.py @@ -12,13 +12,13 @@ from chainlib.chain import ChainSpec from confini import Config # local imports -from cic_ussd.chain import Chain +from cic_ussd.account.chain import Chain +from cic_ussd.cache import Cache from cic_ussd.db import dsn_from_config from cic_ussd.db.models.base import SessionBase from cic_ussd.metadata.signer import Signer from cic_ussd.metadata.base import Metadata from cic_ussd.phone_number import Support -from cic_ussd.redis import InMemoryStore from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession from cic_ussd.validator import validate_presence @@ -34,7 +34,8 @@ arg_parser.add_argument('-c', type=str, default=config_directory, help='config d arg_parser.add_argument('-q', type=str, default='cic-ussd', help='queue name for worker tasks') arg_parser.add_argument('-v', action='store_true', help='be verbose') arg_parser.add_argument('-vv', action='store_true', help='be more verbose') -arg_parser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +arg_parser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, + help='environment prefix for variables to overwrite configuration') args = arg_parser.parse_args() # define log levels @@ -52,7 +53,8 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config)) # connect to database data_source_name = dsn_from_config(config) -SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) +SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), + debug=config.true('DATABASE_DEBUG')) # verify database connection with minimal sanity query session = SessionBase.create_session() @@ -60,12 +62,12 @@ session.execute('SELECT version_num FROM alembic_version') session.close() # define universal redis cache access -InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'), - port=config.get('REDIS_PORT'), - password=config.get('REDIS_PASSWORD'), - db=config.get('REDIS_DATABASE'), - decode_responses=True) -InMemoryUssdSession.redis_cache = InMemoryStore.cache +Cache.store = redis.StrictRedis(host=config.get('REDIS_HOST'), + port=config.get('REDIS_PORT'), + password=config.get('REDIS_PASSWORD'), + db=config.get('REDIS_DATABASE'), + decode_responses=True) +InMemoryUssdSession.store = Cache.store # define metadata URL Metadata.base_url = config.get('CIC_META_URL') @@ -82,8 +84,8 @@ if key_file_path: Signer.key_file_path = key_file_path # set up translations -i18n.load_path.append(config.get('APP_LOCALE_PATH')) -i18n.set('fallback', config.get('APP_LOCALE_FALLBACK')) +i18n.load_path.append(config.get('LOCALE_PATH')) +i18n.set('fallback', config.get('LOCALE_FALLBACK')) chain_spec = ChainSpec( common_name=config.get('CIC_COMMON_NAME'), @@ -92,8 +94,7 @@ chain_spec = ChainSpec( ) Chain.spec = chain_spec -Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER') - +Support.phone_number = config.get('OFFICE_SUPPORT_PHONE') # set up celery current_app = celery.Celery(__name__) @@ -122,12 +123,12 @@ if result[:4] == 'file': result_queue = tempfile.mkdtemp() current_app.conf.update({ 'result_backend': 'file://{}'.format(result_queue), - }) + }) logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(result_queue)) else: current_app.conf.update({ 'result_backend': result, - }) + }) import cic_ussd.tasks @@ -147,4 +148,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py index 4a138818..ac624327 100644 --- a/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py +++ b/apps/cic-ussd/cic_ussd/runnable/daemons/cic_user_ussd_server.py @@ -14,26 +14,25 @@ from chainlib.chain import ChainSpec from confini import Config # local imports -from cic_ussd.chain import Chain +from cic_ussd.account.chain import Chain +from cic_ussd.account.tokens import query_default_token +from cic_ussd.cache import cache_data, cache_data_key, Cache from cic_ussd.db import dsn_from_config from cic_ussd.db.models.base import SessionBase from cic_ussd.encoder import PasswordEncoder from cic_ussd.error import InitializationError from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser +from cic_ussd.http.requests import get_request_endpoint, get_request_method +from cic_ussd.http.responses import with_content_headers from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.metadata.signer import Signer from cic_ussd.metadata.base import Metadata -from cic_ussd.operations import (define_response_with_content, - process_menu_interaction_requests, - define_multilingual_responses) +from cic_ussd.metadata.signer import Signer from cic_ussd.phone_number import process_phone_number, Support, E164Format -from cic_ussd.processor import get_default_token_data -from cic_ussd.redis import cache_data, create_cached_data_key, InMemoryStore -from cic_ussd.requests import (get_request_endpoint, - get_request_method) +from cic_ussd.processor.ussd import handle_menu_operations from cic_ussd.runnable.server_base import exportable_parser, logg from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession from cic_ussd.state_machine import UssdStateMachine +from cic_ussd.translation import translation_for from cic_ussd.validator import check_ip, check_request_content_length, validate_phone_number, validate_presence args = exportable_parser.parse_args() @@ -57,8 +56,8 @@ SessionBase.connect(data_source_name, debug=config.true('DATABASE_DEBUG')) # set up translations -i18n.load_path.append(config.get('APP_LOCALE_PATH')) -i18n.set('fallback', config.get('APP_LOCALE_FALLBACK')) +i18n.load_path.append(config.get('LOCALE_PATH')) +i18n.set('fallback', config.get('LOCALE_FALLBACK')) # set Fernet key PasswordEncoder.set_key(config.get('APP_PASSWORD_PEPPER')) @@ -69,12 +68,12 @@ ussd_menu_db = create_local_file_data_stores(file_location=config.get('USSD_MENU UssdMenu.ussd_menu_db = ussd_menu_db # define universal redis cache access -InMemoryStore.cache = redis.StrictRedis(host=config.get('REDIS_HOSTNAME'), - port=config.get('REDIS_PORT'), - password=config.get('REDIS_PASSWORD'), - db=config.get('REDIS_DATABASE'), - decode_responses=True) -InMemoryUssdSession.redis_cache = InMemoryStore.cache +Cache.store = redis.StrictRedis(host=config.get('REDIS_HOST'), + port=config.get('REDIS_PORT'), + password=config.get('REDIS_PASSWORD'), + db=config.get('REDIS_DATABASE'), + decode_responses=True) +InMemoryUssdSession.store = Cache.store # define metadata URL Metadata.base_url = config.get('CIC_META_URL') @@ -94,8 +93,8 @@ Signer.key_file_path = key_file_path celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) # load states and transitions data -states = json_file_parser(filepath=config.get('STATEMACHINE_STATES')) -transitions = json_file_parser(filepath=config.get('STATEMACHINE_TRANSITIONS')) +states = json_file_parser(filepath=config.get('MACHINE_STATES')) +transitions = json_file_parser(filepath=config.get('MACHINE_TRANSITIONS')) chain_spec = ChainSpec( common_name=config.get('CIC_COMMON_NAME'), @@ -108,24 +107,22 @@ UssdStateMachine.states = states UssdStateMachine.transitions = transitions # retrieve default token data -default_token_data = get_default_token_data() chain_str = Chain.spec.__str__() +default_token_data = query_default_token(chain_str) + # cache default token for re-usability if default_token_data: - cache_key = create_cached_data_key( - identifier=chain_str.encode('utf-8'), - salt=':cic.default_token_data' - ) + cache_key = cache_data_key(chain_str.encode('utf-8'), ':cic.default_token_data') cache_data(key=cache_key, data=json.dumps(default_token_data)) else: raise InitializationError(f'Default token data for: {chain_str} not found.') -valid_service_codes = config.get('APP_SERVICE_CODE').split(",") +valid_service_codes = config.get('USSD_SERVICE_CODE').split(",") -E164Format.region = config.get('PHONE_NUMBER_REGION') -Support.phone_number = config.get('APP_SUPPORT_PHONE_NUMBER') +E164Format.region = config.get('E164_REGION') +Support.phone_number = config.get('OFFICE_SUPPORT_PHONE') def application(env, start_response): @@ -168,49 +165,37 @@ def application(env, start_response): except TypeError: user_input = "" - # add validation for phone number if phone_number: phone_number = process_phone_number(phone_number=phone_number, region=E164Format.region) - # validate ip address if not check_ip(config=config, env=env): start_response('403 Sneaky, sneaky', errors_headers) return [] - # validate content length if not check_request_content_length(config=config, env=env): start_response('400 Size matters', errors_headers) return [] - # validate service code if service_code not in valid_service_codes: - response = define_multilingual_responses( - key='ussd.kenya.invalid_service_code', - locales=['en', 'sw'], - prefix='END', - valid_service_code=valid_service_codes[0]) - response_bytes, headers = define_response_with_content(headers=headers, response=response) + response = translation_for( + 'ussd.kenya.invalid_service_code', + i18n.config.get('fallback'), + valid_service_code=valid_service_codes[0] + ) + response_bytes, headers = with_content_headers(headers, response) start_response('200 OK', headers) return [response_bytes] - # validate phone number if not validate_phone_number(phone_number): logg.error('invalid phone number {}'.format(phone_number)) start_response('400 Invalid phone number format', errors_headers) return [] logg.debug('session {} started for {}'.format(external_session_id, phone_number)) - # handle menu interaction requests - chain_str = chain_spec.__str__() - response = process_menu_interaction_requests(chain_str=chain_str, - external_session_id=external_session_id, - phone_number=phone_number, - queue=args.q, - service_code=service_code, - session=session, - user_input=user_input) - - response_bytes, headers = define_response_with_content(headers=headers, response=response) + response = handle_menu_operations( + chain_str, external_session_id, phone_number, args.q, service_code, session, user_input + ) + response_bytes, headers = with_content_headers(headers, response) start_response('200 OK,', headers) session.commit() session.close() @@ -223,4 +208,3 @@ def application(env, start_response): session.close() start_response('405 Play by the rules', errors_headers) return [] - diff --git a/apps/cic-ussd/cic_ussd/session/ussd_session.py b/apps/cic-ussd/cic_ussd/session/ussd_session.py index 2b8e0ae3..c5ec9fbb 100644 --- a/apps/cic-ussd/cic_ussd/session/ussd_session.py +++ b/apps/cic-ussd/cic_ussd/session/ussd_session.py @@ -3,9 +3,15 @@ import logging from typing import Optional import json -# third party imports +# external imports +import celery from redis import Redis +from sqlalchemy.orm.session import Session +# local imports +from cic_ussd.cache import Cache +from cic_ussd.db.models.base import SessionBase +from cic_ussd.db.models.ussd_session import UssdSession as DbUssdSession logg = logging.getLogger() @@ -13,18 +19,18 @@ logg = logging.getLogger() class UssdSession: """ This class defines the USSD session object that is called whenever a user interacts with the system. - :cvar redis_cache: The in-memory redis cache. - :type redis_cache: Redis + :cvar store: The in-memory redis cache. + :type store: Redis """ - redis_cache: Redis = None + store: Redis = None def __init__(self, external_session_id: str, - service_code: str, msisdn: str, - user_input: str, + service_code: str, state: str, - session_data: Optional[dict] = None): + user_input: str, + data: Optional[dict] = None): """ This function is called whenever a USSD session object is created and saves the instance to a JSON DB. :param external_session_id: The Africa's Talking session ID. @@ -37,16 +43,17 @@ class UssdSession: :type user_input: str. :param state: The name of the USSD menu that the user was interacting with. :type state: str. - :param session_data: Any additional data that was persisted during the user's interaction with the system. - :type session_data: dict. + :param data: Any additional data that was persisted during the user's interaction with the system. + :type data: dict. """ + self.data = data self.external_session_id = external_session_id - self.service_code = service_code self.msisdn = msisdn - self.user_input = user_input + self.service_code = service_code self.state = state - self.session_data = session_data - session = self.redis_cache.get(external_session_id) + self.user_input = user_input + + session = self.store.get(external_session_id) if session: session = json.loads(session) self.version = session.get('version') + 1 @@ -54,16 +61,16 @@ class UssdSession: self.version = 1 self.session = { + 'data': self.data, 'external_session_id': self.external_session_id, - 'service_code': self.service_code, 'msisdn': self.msisdn, - 'user_input': self.user_input, + 'service_code': self.service_code, 'state': self.state, - 'session_data': self.session_data, + 'user_input': self.user_input, 'version': self.version } - self.redis_cache.set(self.external_session_id, json.dumps(self.session)) - self.redis_cache.persist(self.external_session_id) + self.store.set(self.external_session_id, json.dumps(self.session)) + self.store.persist(self.external_session_id) def set_data(self, key: str, value: str) -> None: """ @@ -73,10 +80,10 @@ class UssdSession: :param value: The actual data to be stored in the session data. :type value: str. """ - if self.session_data is None: - self.session_data = {} - self.session_data[key] = value - self.redis_cache.set(self.external_session_id, json.dumps(self.session)) + if self.data is None: + self.data = {} + self.data[key] = value + self.store.set(self.external_session_id, json.dumps(self.session)) def get_data(self, key: str) -> Optional[str]: """ @@ -86,8 +93,8 @@ class UssdSession: :return: This function returns the queried data if found, else it doesn't return any value. :rtype: str. """ - if self.session_data is not None: - return self.session_data.get(key) + if self.data is not None: + return self.data.get(key) else: return None @@ -97,11 +104,155 @@ class UssdSession: :rtype: dict """ return { + "data": self.data, "external_session_id": self.external_session_id, - "service_code": self.service_code, "msisdn": self.msisdn, "user_input": self.user_input, + "service_code": self.service_code, "state": self.state, - "session_data": self.session_data, "version": self.version } + + +def create_ussd_session( + state: str, + external_session_id: str, + msisdn: str, + service_code: str, + user_input: str, + data: Optional[dict] = None) -> UssdSession: + """ + :param state: + :type state: + :param external_session_id: + :type external_session_id: + :param msisdn: + :type msisdn: + :param service_code: + :type service_code: + :param user_input: + :type user_input: + :param data: + :type data: + :return: + :rtype: + """ + return UssdSession(external_session_id=external_session_id, + msisdn=msisdn, + user_input=user_input, + state=state, + service_code=service_code, + data=data + ) + + +def update_ussd_session(ussd_session: UssdSession, + user_input: str, + state: str, + data: Optional[dict] = None) -> UssdSession: + """""" + if data is None: + data = ussd_session.data + + return UssdSession( + external_session_id=ussd_session.external_session_id, + msisdn=ussd_session.msisdn, + user_input=user_input, + state=state, + service_code=ussd_session.service_code, + data=data + ) + + +def create_or_update_session(external_session_id: str, + msisdn: str, + service_code: str, + user_input: str, + state: str, + session, + data: Optional[dict] = None) -> UssdSession: + """ + :param external_session_id: + :type external_session_id: + :param msisdn: + :type msisdn: + :param service_code: + :type service_code: + :param user_input: + :type user_input: + :param state: + :type state: + :param session: + :type session: + :param data: + :type data: + :return: + :rtype: + """ + session = SessionBase.bind_session(session=session) + existing_ussd_session = session.query(DbUssdSession).filter_by( + external_session_id=external_session_id).first() + + if existing_ussd_session: + ussd_session = update_ussd_session(ussd_session=existing_ussd_session, + state=state, + user_input=user_input, + data=data + ) + else: + ussd_session = create_ussd_session(external_session_id=external_session_id, + msisdn=msisdn, + service_code=service_code, + user_input=user_input, + state=state, + data=data + ) + SessionBase.release_session(session=session) + return ussd_session + + +def persist_ussd_session(external_session_id: str, queue: Optional[str]): + """This function asynchronously retrieves a cached ussd session object matching an external ussd session id and adds + it to persistent storage. + :param external_session_id: Session id value provided by ussd service provided. + :type external_session_id: str + :param queue: Name of worker queue to submit tasks to. + :type queue: str + """ + s_persist_ussd_session = celery.signature( + 'cic_ussd.tasks.ussd_session.persist_session_to_db', + [external_session_id], + queue=queue + ) + s_persist_ussd_session.apply_async() + + +def save_session_data(queue: Optional[str], session: Session, data: dict, ussd_session: dict): + """This function is stores information to the session data attribute of a cached ussd session object. + :param data: A dictionary containing data for a specific ussd session in redis that needs to be saved + temporarily. + :type data: dict + :param queue: The queue on which the celery task should run. + :type queue: str + :param session: Database session object. + :type session: Session + :param ussd_session: A ussd session passed to the state machine. + :type ussd_session: UssdSession + """ + cache = Cache.store + external_session_id = ussd_session.get('external_session_id') + existing_session_data = ussd_session.get('data') + if existing_session_data: + data = {**existing_session_data, **data} + in_redis_ussd_session = cache.get(external_session_id) + in_redis_ussd_session = json.loads(in_redis_ussd_session) + create_or_update_session( + external_session_id=external_session_id, + msisdn=in_redis_ussd_session.get('msisdn'), + service_code=in_redis_ussd_session.get('service_code'), + user_input=in_redis_ussd_session.get('user_input'), + state=in_redis_ussd_session.get('state'), + session=session, + data=data + ) + persist_ussd_session(external_session_id=external_session_id, queue=queue) diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/account.py b/apps/cic-ussd/cic_ussd/state_machine/logic/account.py new file mode 100644 index 00000000..ca08e41c --- /dev/null +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/account.py @@ -0,0 +1,248 @@ +# standard imports +import json +import logging +from typing import Tuple + +# third-party imports +import celery +import i18n +from chainlib.hash import strip_0x +from cic_types.models.person import get_contact_data_from_vcard, generate_vcard_from_contact_data, manage_identity_data + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.account.maps import gender, language +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.db.models.account import Account +from cic_ussd.db.models.base import SessionBase +from cic_ussd.error import MetadataNotFoundError +from cic_ussd.metadata import PersonMetadata +from cic_ussd.session.ussd_session import save_session_data +from cic_ussd.translation import translation_for +from sqlalchemy.orm.session import Session + +logg = logging.getLogger(__file__) + + +def change_preferred_language(state_machine_data: Tuple[str, dict, Account, Session]): + """ + :param state_machine_data: + :type state_machine_data: + :return: + :rtype: + """ + user_input, ussd_session, account, session = state_machine_data + r_user_input = language().get(user_input) + session = SessionBase.bind_session(session) + account.preferred_language = r_user_input + session.add(account) + session.flush() + SessionBase.release_session(session) + + preferences_data = { + 'preferred_language': r_user_input + } + + s = celery.signature( + 'cic_ussd.tasks.metadata.add_preferences_metadata', + [account.blockchain_address, preferences_data], + queue='cic-ussd' + ) + return s.apply_async() + + +def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account, Session]): + """This function sets user's account to active. + :param state_machine_data: A tuple containing user input, a ussd session and user object. + :type state_machine_data: tuple + """ + user_input, ussd_session, account, session = state_machine_data + session = SessionBase.bind_session(session=session) + account.activate_account() + session.add(account) + session.flush() + SessionBase.release_session(session=session) + + +def parse_gender(account: Account, user_input: str): + """ + :param account: + :type account: + :param user_input: + :type user_input: + :return: + :rtype: + """ + preferred_language = get_cached_preferred_language(account.blockchain_address) + if not preferred_language: + preferred_language = i18n.config.get('fallback') + r_user_input = gender().get(user_input) + return translation_for(f'helpers.{r_user_input}', preferred_language) + + +def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): + """This function saves first name data to the ussd session in the redis cache. + :param state_machine_data: A tuple containing user input, a ussd session and user object. + :type state_machine_data: tuple + """ + user_input, ussd_session, account, session = state_machine_data + session = SessionBase.bind_session(session=session) + current_state = ussd_session.get('state') + + key = '' + if 'given_name' in current_state: + key = 'given_name' + + if 'date_of_birth' in current_state: + key = 'date_of_birth' + + if 'family_name' in current_state: + key = 'family_name' + + if 'gender' in current_state: + key = 'gender' + user_input = parse_gender(account, user_input) + + if 'location' in current_state: + key = 'location' + + if 'products' in current_state: + key = 'products' + + if ussd_session.get('data'): + data = ussd_session.get('data') + data[key] = user_input + else: + data = { + key: user_input + } + save_session_data('cic-ussd', session, data, ussd_session) + SessionBase.release_session(session) + + +def parse_person_metadata(account: Account, metadata: dict): + """ + :param account: + :type account: + :param metadata: + :type metadata: + :return: + :rtype: + """ + set_gender = metadata.get('gender') + given_name = metadata.get('given_name') + family_name = metadata.get('family_name') + email = metadata.get('email') + + if isinstance(metadata.get('date_of_birth'), dict): + date_of_birth = metadata.get('date_of_birth') + else: + date_of_birth = { + "year": int(metadata.get('date_of_birth')[:4]) + } + if isinstance(metadata.get('location'), dict): + location = metadata.get('location') + else: + location = { + "area_name": metadata.get('location') + } + if isinstance(metadata.get('products'), list): + products = metadata.get('products') + else: + products = metadata.get('products').split(',') + + phone_number = account.phone_number + date_registered = int(account.created.replace().timestamp()) + blockchain_address = account.blockchain_address + chain_spec = f'{Chain.spec.common_name()}:{Chain.spec.engine()}: {Chain.spec.chain_id()}' + + if isinstance(metadata.get('identities'), dict): + identities = metadata.get('identities') + else: + identities = manage_identity_data( + blockchain_address=blockchain_address, + blockchain_type=Chain.spec.engine(), + chain_spec=chain_spec + ) + + return { + "date_registered": date_registered, + "date_of_birth": date_of_birth, + "gender": set_gender, + "identities": identities, + "location": location, + "products": products, + "vcard": generate_vcard_from_contact_data( + email=email, + family_name=family_name, + given_name=given_name, + tel=phone_number + ) + } + + +def save_complete_person_metadata(state_machine_data: Tuple[str, dict, Account, Session]): + """This function persists elements of the user metadata stored in session data + :param state_machine_data: A tuple containing user input, a ussd session and user object. + :type state_machine_data: tuple + """ + user_input, ussd_session, account, session = state_machine_data + metadata = ussd_session.get('data') + person_metadata = parse_person_metadata(account, metadata) + blockchain_address = account.blockchain_address + s_create_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.create_person_metadata', [blockchain_address, person_metadata], queue='cic-ussd') + s_create_person_metadata.apply_async() + + +def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account, Session]): + """ + :param state_machine_data: + :type state_machine_data: + :return: + :rtype: + """ + user_input, ussd_session, account, session = state_machine_data + blockchain_address = account.blockchain_address + identifier = bytes.fromhex(strip_0x(blockchain_address)) + person_metadata = PersonMetadata(identifier) + cached_person_metadata = person_metadata.get_cached_metadata() + + if not cached_person_metadata: + raise MetadataNotFoundError(f'Expected user metadata but found none in cache for key: {blockchain_address}') + + person_metadata = json.loads(cached_person_metadata) + data = ussd_session.get('data') + contact_data = {} + vcard = person_metadata.get('vcard') + if vcard: + contact_data = get_contact_data_from_vcard(vcard) + person_metadata.pop('vcard') + given_name = data.get('given_name') or contact_data.get('given') + family_name = data.get('family_name') or contact_data.get('family') + date_of_birth = data.get('date_of_birth') or person_metadata.get('date_of_birth') + set_gender = data.get('gender') or person_metadata.get('gender') + location = data.get('location') or person_metadata.get('location') + products = data.get('products') or person_metadata.get('products') + if isinstance(date_of_birth, str): + year = int(date_of_birth) + person_metadata['date_of_birth'] = {'year': year} + person_metadata['gender'] = set_gender + person_metadata['given_name'] = given_name + person_metadata['family_name'] = family_name + if isinstance(location, str): + location_data = person_metadata.get('location') + location_data['area_name'] = location + person_metadata['location'] = location_data + person_metadata['products'] = products + if contact_data: + contact_data.pop('given') + contact_data.pop('family') + contact_data.pop('tel') + person_metadata = {**person_metadata, **contact_data} + parsed_person_metadata = parse_person_metadata(account, person_metadata) + s_edit_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.create_person_metadata', + [blockchain_address, parsed_person_metadata] + ) + s_edit_person_metadata.apply_async(queue='cic-ussd') diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py b/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py deleted file mode 100644 index 64f1dd61..00000000 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/balance.py +++ /dev/null @@ -1,21 +0,0 @@ -# standard imports -import logging -from typing import Tuple - -# third-party imports -from sqlalchemy.orm.session import Session - -# local imports -from cic_ussd.db.models.account import Account - -logg = logging.getLogger(__file__) - - -def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account, Session]): - """This function compiles a brief statement of a user's last three inbound and outbound transactions and send the - same as a message on their selected avenue for notification. - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: str - """ - user_input, ussd_session, user, session = state_machine_data - logg.debug('This section requires integration with cic-eth. (The last 6 transactions would be sent as an sms.)') diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py b/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py index bac773b9..e8c80fce 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/menu.py @@ -5,44 +5,47 @@ ussd menu facilitating the return of appropriate menu responses based on said us # standard imports from typing import Tuple +# external imports +from sqlalchemy.orm.session import Session + # local imports from cic_ussd.db.models.account import Account -def menu_one_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_one_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that user input matches a string with value '1' :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str :return: A user input's match with '1' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '1' -def menu_two_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_two_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that user input matches a string with value '2' :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A user input's match with '2' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '2' -def menu_three_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_three_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """This function checks that user input matches a string with value '3' :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: tuple :return: A user input's match with '3' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '3' -def menu_four_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_four_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """ This function checks that user input matches a string with value '4' :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -50,11 +53,11 @@ def menu_four_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '4' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '4' -def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_five_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """ This function checks that user input matches a string with value '5' :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -62,11 +65,11 @@ def menu_five_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '5' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '5' -def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_six_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """ This function checks that user input matches a string with value '6' :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -74,11 +77,11 @@ def menu_six_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: :return: A user input's match with '6' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '6' -def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """ This function checks that user input matches a string with value '00' :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -86,11 +89,11 @@ def menu_zero_zero_selected(state_machine_data: Tuple[str, dict, Account]) -> bo :return: A user input's match with '00' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '00' -def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account]) -> bool: +def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: """ This function checks that user input matches a string with value '99' :param state_machine_data: A tuple containing user input, a ussd session and user object. @@ -98,5 +101,5 @@ def menu_ninety_nine_selected(state_machine_data: Tuple[str, dict, Account]) -> :return: A user input's match with '99' :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data return user_input == '99' diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py b/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py index 9419d248..50f9be12 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/pin.py @@ -16,8 +16,7 @@ from cic_ussd.db.models.account import Account from cic_ussd.db.models.base import SessionBase from cic_ussd.db.enum import AccountStatus from cic_ussd.encoder import create_password_hash, check_password_hash -from cic_ussd.operations import persist_session_to_db_task, create_or_update_session -from cic_ussd.redis import InMemoryStore +from cic_ussd.session.ussd_session import create_or_update_session, persist_ussd_session logg = logging.getLogger(__file__) @@ -31,7 +30,7 @@ def is_valid_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool :return: A pin's validity :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data pin_is_valid = False matcher = r'^\d{4}$' if re.match(matcher, user_input): @@ -46,8 +45,11 @@ def is_authorized_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data - return user.verify_password(password=user_input) + user_input, ussd_session, account, session = state_machine_data + is_verified_password = account.verify_password(password=user_input) + if not is_verified_password: + account.failed_pin_attempts += 1 + return is_verified_password def is_locked_account(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: @@ -57,8 +59,8 @@ def is_locked_account(state_machine_data: Tuple[str, dict, Account, Session]) -> :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data - return user.get_account_status() == AccountStatus.LOCKED.name + user_input, ussd_session, account, session = state_machine_data + return account.get_status(session) == AccountStatus.LOCKED.name def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): @@ -67,38 +69,25 @@ def save_initial_pin_to_session_data(state_machine_data: Tuple[str, dict, Accoun :type state_machine_data: tuple """ user_input, ussd_session, user, session = state_machine_data - - # define redis cache entry point - cache = InMemoryStore.cache - - # get external session id - external_session_id = ussd_session.get('external_session_id') - - # get corresponding session record - in_redis_ussd_session = cache.get(external_session_id) - in_redis_ussd_session = json.loads(in_redis_ussd_session) - - # set initial pin data initial_pin = create_password_hash(user_input) - if ussd_session.get('session_data'): - session_data = ussd_session.get('session_data') - session_data['initial_pin'] = initial_pin + if ussd_session.get('data'): + data = ussd_session.get('data') + data['initial_pin'] = initial_pin else: - session_data = { + data = { 'initial_pin': initial_pin } - - # create new in memory ussd session with current ussd session data + external_session_id = ussd_session.get('external_session_id') create_or_update_session( external_session_id=external_session_id, - phone=in_redis_ussd_session.get('msisdn'), - service_code=in_redis_ussd_session.get('service_code'), + msisdn=ussd_session.get('msisdn'), + service_code=ussd_session.get('service_code'), user_input=user_input, - current_menu=in_redis_ussd_session.get('state'), + state=ussd_session.get('state'), session=session, - session_data=session_data + data=data ) - persist_session_to_db_task(external_session_id=external_session_id, queue='cic-ussd') + persist_ussd_session(external_session_id, 'cic-ussd') def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: @@ -109,7 +98,7 @@ def pins_match(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: :rtype: bool """ user_input, ussd_session, user, session = state_machine_data - initial_pin = ussd_session.get('session_data').get('initial_pin') + initial_pin = ussd_session.get('data').get('initial_pin') return check_password_hash(user_input, initial_pin) @@ -120,7 +109,7 @@ def complete_pin_change(state_machine_data: Tuple[str, dict, Account, Session]): """ user_input, ussd_session, user, session = state_machine_data session = SessionBase.bind_session(session=session) - password_hash = ussd_session.get('session_data').get('initial_pin') + password_hash = ussd_session.get('data').get('initial_pin') user.password_hash = password_hash session.add(user) session.flush() @@ -134,8 +123,8 @@ def is_blocked_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bo :return: A match between two pin values. :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data - return user.get_account_status() == AccountStatus.LOCKED.name + user_input, ussd_session, account, session = state_machine_data + return account.get_status(session) == AccountStatus.LOCKED.name def is_valid_new_pin(state_machine_data: Tuple[str, dict, Account, Session]) -> bool: diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py b/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py index 6f33a555..7988a06c 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/sms.py @@ -1,23 +1,28 @@ # standard imports -import logging from typing import Tuple +# external imports +from sqlalchemy.orm.session import Session + # local imports +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.account.tokens import get_default_token_symbol from cic_ussd.db.models.account import Account - -logg = logging.getLogger() +from cic_ussd.notifications import Notifier +from cic_ussd.phone_number import Support -def send_terms_to_user_if_required(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user, session = state_machine_data - logg.debug('Requires integration to cic-notify.') - - -def process_mini_statement_request(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user, session = state_machine_data - logg.debug('Requires integration to cic-notify.') - - -def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account]): - user_input, ussd_session, user, session = state_machine_data - logg.debug('Requires integration to cic-notify.') \ No newline at end of file +def upsell_unregistered_recipient(state_machine_data: Tuple[str, dict, Account, Session]): + """""" + user_input, ussd_session, account, session = state_machine_data + notifier = Notifier() + phone_number = ussd_session.get('data')['recipient_phone_number'] + preferred_language = get_cached_preferred_language(account.blockchain_address) + token_symbol = get_default_token_symbol() + tx_sender_information = account.standard_metadata_id() + notifier.send_sms_notification('sms.upsell_unregistered_recipient', + phone_number, + preferred_language, + tx_sender_information=tx_sender_information, + token_symbol=token_symbol, + support_phone=Support.phone_number) diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py index c2ac5e45..a80e2317 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/transaction.py @@ -1,24 +1,21 @@ # standard imports -import json import logging from typing import Tuple # third party imports import celery -from sqlalchemy.orm.session import Session # local imports -from cic_ussd.balance import compute_operational_balance -from cic_ussd.chain import Chain +from cic_ussd.account.balance import get_cached_available_balance +from cic_ussd.account.chain import Chain +from cic_ussd.account.tokens import get_default_token_symbol +from cic_ussd.account.transaction import OutgoingTransaction +from cic_ussd.db.enum import AccountStatus from cic_ussd.db.models.account import Account from cic_ussd.db.models.base import SessionBase -from cic_ussd.db.enum import AccountStatus -from cic_ussd.operations import save_to_in_memory_ussd_session_data from cic_ussd.phone_number import process_phone_number, E164Format -from cic_ussd.processor import retrieve_token_symbol -from cic_ussd.redis import create_cached_data_key, get_cached_data -from cic_ussd.transactions import OutgoingTransactionProcessor - +from cic_ussd.session.ussd_session import save_session_data +from sqlalchemy.orm.session import Session logg = logging.getLogger(__file__) @@ -31,13 +28,15 @@ def is_valid_recipient(state_machine_data: Tuple[str, dict, Account, Session]) - :return: A user's validity :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data phone_number = process_phone_number(user_input, E164Format.region) session = SessionBase.bind_session(session=session) recipient = Account.get_by_phone_number(phone_number=phone_number, session=session) SessionBase.release_session(session=session) - is_not_initiator = phone_number != user.phone_number - has_active_account_status = user.get_account_status() == AccountStatus.ACTIVE.name + is_not_initiator = phone_number != account.phone_number + has_active_account_status = False + if recipient: + has_active_account_status = recipient.get_status(session) == AccountStatus.ACTIVE.name return is_not_initiator and has_active_account_status and recipient is not None @@ -49,7 +48,7 @@ def is_valid_transaction_amount(state_machine_data: Tuple[str, dict, Account, Se :return: A transaction amount's validity :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data try: return int(user_input) > 0 except ValueError: @@ -64,16 +63,8 @@ def has_sufficient_balance(state_machine_data: Tuple[str, dict, Account, Session :return: An account balance's validity :rtype: bool """ - user_input, ussd_session, user, session = state_machine_data - # get cached balance - key = create_cached_data_key( - identifier=bytes.fromhex(user.blockchain_address[2:]), - salt=':cic.balances_data' - ) - cached_balance = get_cached_data(key=key) - operational_balance = compute_operational_balance(balances=json.loads(cached_balance)) - - return int(user_input) <= operational_balance + user_input, ussd_session, account, session = state_machine_data + return int(user_input) <= get_cached_available_balance(account.blockchain_address) def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): @@ -81,17 +72,13 @@ def save_recipient_phone_to_session_data(state_machine_data: Tuple[str, dict, Ac :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data - session_data = ussd_session.get('session_data') or {} + session_data = ussd_session.get('data') or {} recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region) session_data['recipient_phone_number'] = recipient_phone_number - save_to_in_memory_ussd_session_data( - queue='cic-ussd', - session=session, - session_data=session_data, - ussd_session=ussd_session) + save_session_data('cic-ussd', session, session_data, ussd_session) def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account, Session]): @@ -101,18 +88,13 @@ def retrieve_recipient_metadata(state_machine_data: Tuple[str, dict, Account, Se :return: :rtype: """ - user_input, ussd_session, user, session = state_machine_data - - recipient_phone_number = process_phone_number(phone_number=user_input, region=E164Format.region) - recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) + user_input, ussd_session, account, session = state_machine_data + recipient_phone_number = process_phone_number(user_input, E164Format.region) + recipient = Account.get_by_phone_number(recipient_phone_number, session) blockchain_address = recipient.blockchain_address - - # retrieve and cache account's metadata s_query_person_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_person_metadata', - [blockchain_address] - ) - s_query_person_metadata.apply_async(queue='cic-ussd') + 'cic_ussd.tasks.metadata.query_person_metadata', [blockchain_address], queue='cic-ussd') + s_query_person_metadata.apply_async() def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): @@ -120,16 +102,11 @@ def save_transaction_amount_to_session_data(state_machine_data: Tuple[str, dict, :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data - session_data = ussd_session.get('session_data') or {} + session_data = ussd_session.get('data') or {} session_data['transaction_amount'] = user_input - - save_to_in_memory_ussd_session_data( - queue='cic-ussd', - session=session, - session_data=session_data, - ussd_session=ussd_session) + save_session_data('cic-ussd', session, session_data, ussd_session) def process_transaction_request(state_machine_data: Tuple[str, dict, Account, Session]): @@ -137,20 +114,18 @@ def process_transaction_request(state_machine_data: Tuple[str, dict, Account, Se :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data - # retrieve token symbol chain_str = Chain.spec.__str__() - # get user from phone number - recipient_phone_number = ussd_session.get('session_data').get('recipient_phone_number') + recipient_phone_number = ussd_session.get('data').get('recipient_phone_number') recipient = Account.get_by_phone_number(phone_number=recipient_phone_number, session=session) to_address = recipient.blockchain_address - from_address = user.blockchain_address - amount = int(ussd_session.get('session_data').get('transaction_amount')) - token_symbol = retrieve_token_symbol(chain_str=chain_str) + from_address = account.blockchain_address + amount = int(ussd_session.get('data').get('transaction_amount')) + token_symbol = get_default_token_symbol() - outgoing_tx_processor = OutgoingTransactionProcessor(chain_str=chain_str, - from_address=from_address, - to_address=to_address) - outgoing_tx_processor.process_outgoing_transfer_transaction(amount=amount, token_symbol=token_symbol) + outgoing_tx_processor = OutgoingTransaction(chain_str=chain_str, + from_address=from_address, + to_address=to_address) + outgoing_tx_processor.transfer(amount=amount, token_symbol=token_symbol) diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py b/apps/cic-ussd/cic_ussd/state_machine/logic/user.py deleted file mode 100644 index 7c01f971..00000000 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/user.py +++ /dev/null @@ -1,292 +0,0 @@ -# standard imports -import json -import logging -from typing import Tuple - -# third-party imports -import celery -from cic_types.models.person import generate_metadata_pointer -from cic_types.models.person import generate_vcard_from_contact_data, manage_identity_data -from sqlalchemy.orm.session import Session - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.db.models.account import Account -from cic_ussd.db.models.base import SessionBase -from cic_ussd.error import MetadataNotFoundError -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.operations import save_to_in_memory_ussd_session_data -from cic_ussd.redis import get_cached_data - -logg = logging.getLogger(__file__) - - -def change_preferred_language_to_en(state_machine_data: Tuple[str, dict, Account, Session]): - """This function changes the user's preferred language to english. - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: tuple - """ - user_input, ussd_session, user, session = state_machine_data - session = SessionBase.bind_session(session=session) - user.preferred_language = 'en' - session.add(user) - session.flush() - SessionBase.release_session(session=session) - - preferences_data = { - 'preferred_language': 'en' - } - - s = celery.signature( - 'cic_ussd.tasks.metadata.add_preferences_metadata', - [user.blockchain_address, preferences_data] - ) - s.apply_async(queue='cic-ussd') - - -def change_preferred_language_to_sw(state_machine_data: Tuple[str, dict, Account, Session]): - """This function changes the user's preferred language to swahili. - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: tuple - """ - user_input, ussd_session, account, session = state_machine_data - session = SessionBase.bind_session(session=session) - account.preferred_language = 'sw' - session.add(account) - session.flush() - SessionBase.release_session(session=session) - - preferences_data = { - 'preferred_language': 'sw' - } - - s = celery.signature( - 'cic_ussd.tasks.metadata.add_preferences_metadata', - [account.blockchain_address, preferences_data] - ) - s.apply_async(queue='cic-ussd') - - -def update_account_status_to_active(state_machine_data: Tuple[str, dict, Account, Session]): - """This function sets user's account to active. - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: tuple - """ - user_input, ussd_session, account, session = state_machine_data - session = SessionBase.bind_session(session=session) - account.activate_account() - session.add(account) - session.flush() - SessionBase.release_session(session=session) - - -def process_gender_user_input(user: Account, user_input: str): - """ - :param user: - :type user: - :param user_input: - :type user_input: - :return: - :rtype: - """ - gender = "" - if user.preferred_language == 'en': - if user_input == '1': - gender = 'Male' - elif user_input == '2': - gender = 'Female' - elif user_input == '3': - gender = 'Other' - else: - if user_input == '1': - gender = 'Mwanaume' - elif user_input == '2': - gender = 'Mwanamke' - elif user_input == '3': - gender = 'Nyingine' - return gender - - -def save_metadata_attribute_to_session_data(state_machine_data: Tuple[str, dict, Account, Session]): - """This function saves first name data to the ussd session in the redis cache. - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: tuple - """ - user_input, ussd_session, user, session = state_machine_data - session = SessionBase.bind_session(session=session) - # get current menu - current_state = ussd_session.get('state') - - # define session data key from current state - key = '' - if 'given_name' in current_state: - key = 'given_name' - - if 'date_of_birth' in current_state: - key = 'date_of_birth' - - if 'family_name' in current_state: - key = 'family_name' - - if 'gender' in current_state: - key = 'gender' - user_input = process_gender_user_input(user=user, user_input=user_input) - - if 'location' in current_state: - key = 'location' - - if 'products' in current_state: - key = 'products' - - # check if there is existing session data - if ussd_session.get('session_data'): - session_data = ussd_session.get('session_data') - session_data[key] = user_input - else: - session_data = { - key: user_input - } - save_to_in_memory_ussd_session_data( - queue='cic-ussd', - session=session, - session_data=session_data, - ussd_session=ussd_session) - - -def format_user_metadata(metadata: dict, user: Account): - """ - :param metadata: - :type metadata: - :param user: - :type user: - :return: - :rtype: - """ - gender = metadata.get('gender') - given_name = metadata.get('given_name') - family_name = metadata.get('family_name') - - if isinstance(metadata.get('date_of_birth'), dict): - date_of_birth = metadata.get('date_of_birth') - else: - date_of_birth = { - "year": int(metadata.get('date_of_birth')[:4]) - } - - # check whether there's existing location data - if isinstance(metadata.get('location'), dict): - location = metadata.get('location') - else: - location = { - "area_name": metadata.get('location') - } - # check whether it is a list - if isinstance(metadata.get('products'), list): - products = metadata.get('products') - else: - products = metadata.get('products').split(',') - - phone_number = user.phone_number - date_registered = int(user.created.replace().timestamp()) - blockchain_address = user.blockchain_address - chain_spec = f'{Chain.spec.common_name()}:{Chain.spec.network_id()}' - identities = manage_identity_data( - blockchain_address=blockchain_address, - blockchain_type=Chain.spec.engine(), - chain_spec=chain_spec - ) - return { - "date_registered": date_registered, - "date_of_birth": date_of_birth, - "gender": gender, - "identities": identities, - "location": location, - "products": products, - "vcard": generate_vcard_from_contact_data( - family_name=family_name, - given_name=given_name, - tel=phone_number - ) - } - - -def save_complete_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]): - """This function persists elements of the user metadata stored in session data - :param state_machine_data: A tuple containing user input, a ussd session and user object. - :type state_machine_data: tuple - """ - user_input, ussd_session, user, session = state_machine_data - - # get session data - metadata = ussd_session.get('session_data') - - # format metadata appropriately - user_metadata = format_user_metadata(metadata=metadata, user=user) - - blockchain_address = user.blockchain_address - s_create_person_metadata = celery.signature( - 'cic_ussd.tasks.metadata.create_person_metadata', - [blockchain_address, user_metadata] - ) - s_create_person_metadata.apply_async(queue='cic-ussd') - - -def edit_user_metadata_attribute(state_machine_data: Tuple[str, dict, Account, Session]): - user_input, ussd_session, user, session = state_machine_data - blockchain_address = user.blockchain_address - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - user_metadata = get_cached_data(key=key) - - if not user_metadata: - raise MetadataNotFoundError(f'Expected user metadata but found none in cache for key: {blockchain_address}') - - given_name = ussd_session.get('session_data').get('given_name') - family_name = ussd_session.get('session_data').get('family_name') - date_of_birth = ussd_session.get('session_data').get('date_of_birth') - gender = ussd_session.get('session_data').get('gender') - location = ussd_session.get('session_data').get('location') - products = ussd_session.get('session_data').get('products') - - # validate user metadata - user_metadata = json.loads(user_metadata) - - # edit specific metadata attribute - if given_name: - user_metadata['given_name'] = given_name - if family_name: - user_metadata['family_name'] = family_name - if date_of_birth and len(date_of_birth) == 4: - year = int(date_of_birth[:4]) - user_metadata['date_of_birth'] = { - 'year': year - } - if gender: - user_metadata['gender'] = gender - if location: - # get existing location metadata: - location_data = user_metadata.get('location') - location_data['area_name'] = location - user_metadata['location'] = location_data - if products: - user_metadata['products'] = products - - user_metadata = format_user_metadata(metadata=user_metadata, user=user) - - s_edit_person_metadata = celery.signature( - 'cic_ussd.tasks.metadata.create_person_metadata', - [blockchain_address, user_metadata] - ) - s_edit_person_metadata.apply_async(queue='cic-ussd') - - -def get_user_metadata(state_machine_data: Tuple[str, dict, Account, Session]): - user_input, ussd_session, user, session = state_machine_data - blockchain_address = user.blockchain_address - s_get_user_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_person_metadata', - [blockchain_address] - ) - s_get_user_metadata.apply_async(queue='cic-ussd') diff --git a/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py b/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py index 80747042..e29d6b86 100644 --- a/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py +++ b/apps/cic-ussd/cic_ussd/state_machine/logic/validator.py @@ -4,67 +4,58 @@ import re from typing import Tuple # third-party imports -from cic_types.models.person import generate_metadata_pointer +from chainlib.hash import strip_0x +from sqlalchemy.orm.session import Session # local imports from cic_ussd.db.models.account import Account -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.redis import get_cached_data +from cic_ussd.metadata import PersonMetadata + logg = logging.getLogger() -def has_cached_user_metadata(state_machine_data: Tuple[str, dict, Account]): +def has_cached_person_metadata(state_machine_data: Tuple[str, dict, Account, Session]): """This function checks whether the attributes of the user's metadata constituting a profile are filled out. :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user, session = state_machine_data - # check for user metadata in cache - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - user_metadata = get_cached_data(key=key) - return user_metadata is not None + user_input, ussd_session, account, session = state_machine_data + identifier = bytes.fromhex(strip_0x(account.blockchain_address)) + metadata_client = PersonMetadata(identifier) + return metadata_client.get_cached_metadata() is not None -def is_valid_name(state_machine_data: Tuple[str, dict, Account]): +def is_valid_name(state_machine_data: Tuple[str, dict, Account, Session]): """This function checks that a user provided name is valid :param state_machine_data: A tuple containing user input, a ussd session and user object. :type state_machine_data: str """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data name_matcher = "^[a-zA-Z]+$" valid_name = re.match(name_matcher, user_input) - if valid_name: - return True - else: - return False + return bool(valid_name) -def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account]): +def is_valid_gender_selection(state_machine_data: Tuple[str, dict, Account, Session]): """ :param state_machine_data: :type state_machine_data: :return: :rtype: """ - user_input, ussd_session, user, session = state_machine_data - selection_matcher = "^[1-2]$" - if re.match(selection_matcher, user_input): - return True - else: - return False + user_input, ussd_session, account, session = state_machine_data + selection_matcher = "^[1-3]$" + return bool(re.match(selection_matcher, user_input)) -def is_valid_date(state_machine_data: Tuple[str, dict, Account]): +def is_valid_date(state_machine_data: Tuple[str, dict, Account, Session]): """ :param state_machine_data: :type state_machine_data: :return: :rtype: """ - user_input, ussd_session, user, session = state_machine_data + user_input, ussd_session, account, session = state_machine_data # For MVP this value is defaulting to year return len(user_input) == 4 and int(user_input) >= 1900 diff --git a/apps/cic-ussd/cic_ussd/tasks/__init__.py b/apps/cic-ussd/cic_ussd/tasks/__init__.py index d6967d59..6ec61242 100644 --- a/apps/cic-ussd/cic_ussd/tasks/__init__.py +++ b/apps/cic-ussd/cic_ussd/tasks/__init__.py @@ -1,14 +1,13 @@ # standard import # third-party imports -# this must be included for the package to be recognized as a tasks package import celery -celery_app = celery.current_app # export external celery task modules -from .logger import * -from .ussd_session import * from .callback_handler import * from .metadata import * from .notifications import * from .processor import * +from .ussd_session import * + +celery_app = celery.current_app diff --git a/apps/cic-ussd/cic_ussd/tasks/base.py b/apps/cic-ussd/cic_ussd/tasks/base.py index 0f9a7438..2af08088 100644 --- a/apps/cic-ussd/cic_ussd/tasks/base.py +++ b/apps/cic-ussd/cic_ussd/tasks/base.py @@ -1,4 +1,5 @@ # standard imports +import logging # third-party imports import celery @@ -8,6 +9,8 @@ import sqlalchemy from cic_ussd.error import MetadataStoreError from cic_ussd.db.models.base import SessionBase +logg = logging.getLogger(__name__) + class BaseTask(celery.Task): diff --git a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py index cace9219..de9125ba 100644 --- a/apps/cic-ussd/cic_ussd/tasks/callback_handler.py +++ b/apps/cic-ussd/cic_ussd/tasks/callback_handler.py @@ -1,19 +1,22 @@ # standard imports import json import logging -from datetime import datetime, timedelta +from datetime import timedelta # third-party imports import celery +from chainlib.hash import strip_0x # local imports -from cic_ussd.balance import compute_operational_balance, get_balances -from cic_ussd.chain import Chain -from cic_ussd.conversions import from_wei +from cic_ussd.account.balance import get_balances, calculate_available_balance +from cic_ussd.account.statement import generate +from cic_ussd.cache import Cache, cache_data, cache_data_key, get_cached_data +from cic_ussd.account.chain import Chain from cic_ussd.db.models.base import SessionBase from cic_ussd.db.models.account import Account -from cic_ussd.error import ActionDataNotFoundError -from cic_ussd.redis import InMemoryStore, cache_data, create_cached_data_key, get_cached_data +from cic_ussd.account.statement import filter_statement_transactions +from cic_ussd.account.transaction import transaction_actors +from cic_ussd.error import AccountCreationDataNotFound from cic_ussd.tasks.base import CriticalSQLAlchemyTask logg = logging.getLogger(__file__) @@ -21,8 +24,10 @@ celery_app = celery.current_app @celery_app.task(bind=True, base=CriticalSQLAlchemyTask) -def process_account_creation_callback(self, result: str, url: str, status_code: int): +def account_creation_callback(self, result: str, url: str, status_code: int): """This function defines a task that creates a user and + :param self: Reference providing access to the callback task instance. + :type self: celery.Task :param result: The blockchain address for the created account :type result: str :param url: URL provided to callback task in cic-eth should http be used for callback. @@ -30,266 +35,183 @@ def process_account_creation_callback(self, result: str, url: str, status_code: :param status_code: The status of the task to create an account :type status_code: int """ + task_uuid = self.request.root_id + cached_account_creation_data = get_cached_data(task_uuid) + + if not cached_account_creation_data: + raise AccountCreationDataNotFound(f'No account creation data found for task id: {task_uuid}') + + if status_code != 0: + raise ValueError(f'Unexpected status code: {status_code}') + + account_creation_data = json.loads(cached_account_creation_data) + account_creation_data['status'] = 'CREATED' + cache_data(task_uuid, json.dumps(account_creation_data)) + + phone_number = account_creation_data.get('phone_number') + session = SessionBase.create_session() - cache = InMemoryStore.cache - task_id = self.request.root_id - - # get account creation status - account_creation_data = cache.get(task_id) - - # check status - if account_creation_data: - account_creation_data = json.loads(account_creation_data) - if status_code == 0: - # update redis data - account_creation_data['status'] = 'CREATED' - cache.set(name=task_id, value=json.dumps(account_creation_data)) - cache.persist(task_id) - - phone_number = account_creation_data.get('phone_number') - - # create user - user = Account(blockchain_address=result, phone_number=phone_number) - session.add(user) - session.commit() - session.close() - - queue = self.request.delivery_info.get('routing_key') - - # add phone number metadata lookup - s_phone_pointer = celery.signature( - 'cic_ussd.tasks.metadata.add_phone_pointer', - [result, phone_number] - ) - s_phone_pointer.apply_async(queue=queue) - - # add custom metadata tags - custom_metadata = { - "tags": ["ussd", "individual"] - } - s_custom_metadata = celery.signature( - 'cic_ussd.tasks.metadata.add_custom_metadata', - [result, custom_metadata] - ) - s_custom_metadata.apply_async(queue=queue) - - # expire cache - cache.expire(task_id, timedelta(seconds=180)) - - else: - session.close() - cache.expire(task_id, timedelta(seconds=180)) - - else: - session.close() - raise ActionDataNotFoundError(f'Account creation task: {task_id}, returned unexpected response: {status_code}') - + account = Account(blockchain_address=result, phone_number=phone_number) + session.add(account) + session.commit() session.close() + queue = self.request.delivery_info.get('routing_key') + s_phone_pointer = celery.signature( + 'cic_ussd.tasks.metadata.add_phone_pointer', [result, phone_number], queue=queue + ) + s_phone_pointer.apply_async() + + custom_metadata = {"tags": ["ussd", "individual"]} + s_custom_metadata = celery.signature( + 'cic_ussd.tasks.metadata.add_custom_metadata', [result, custom_metadata], queue=queue + ) + s_custom_metadata.apply_async() + Cache.store.expire(task_uuid, timedelta(seconds=180)) + + +@celery_app.task +def balances_callback(result: list, param: str, status_code: int): + """ + :param result: + :type result: + :param param: + :type param: + :param status_code: + :type status_code: + :return: + :rtype: + """ + if status_code != 0: + raise ValueError(f'Unexpected status code: {status_code}.') + + balances = result[0] + identifier = bytes.fromhex(strip_0x(param)) + key = cache_data_key(identifier, ':cic.balances') + cache_data(key, json.dumps(balances)) + @celery_app.task(bind=True) -def process_transaction_callback(self, result: dict, param: str, status_code: int): - if status_code == 0: - chain_str = Chain.spec.__str__() +def statement_callback(self, result, param: str, status_code: int): + """ + :param self: + :type self: + :param result: + :type result: + :param param: + :type param: + :param status_code: + :type status_code: + :return: + :rtype: + """ + if status_code != 0: + raise ValueError(f'Unexpected status code: {status_code}.') - # collect transaction metadata - destination_token_symbol = result.get('destination_token_symbol') - destination_token_value = result.get('destination_token_value') - recipient_blockchain_address = result.get('recipient') - sender_blockchain_address = result.get('sender') - source_token_symbol = result.get('source_token_symbol') - source_token_value = result.get('source_token_value') + queue = self.request.delivery_info.get('routing_key') + statement_transactions = filter_statement_transactions(result) + for transaction in statement_transactions: + recipient_transaction, sender_transaction = transaction_actors(transaction) + if recipient_transaction.get('blockchain_address') == param: + generate(param, queue, recipient_transaction) + if sender_transaction.get('blockchain_address') == param: + generate(param, queue, sender_transaction) - # build stakeholder callback params - recipient_metadata = { - "token_symbol": destination_token_symbol, - "token_value": destination_token_value, - "blockchain_address": recipient_blockchain_address, - "tag": "recipient", - "tx_param": param + +@celery_app.task(bind=True) +def transaction_balances_callback(self, result: list, param: dict, status_code: int): + """ + :param self: + :type self: + :param result: + :type result: + :param param: + :type param: + :param status_code: + :type status_code: + :return: + :rtype: + """ + if status_code != 0: + raise ValueError(f'Unexpected status code: {status_code}.') + + balances_data = result[0] + available_balance = calculate_available_balance(balances_data) + transaction = param + blockchain_address = param.get('blockchain_address') + transaction['available_balance'] = available_balance + queue = self.request.delivery_info.get('routing_key') + + s_preferences_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_preferences_metadata', [blockchain_address], queue=queue + ) + s_process_account_metadata = celery.signature( + 'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue + ) + s_notify_account = celery.signature('cic_ussd.tasks.notifications.transaction', queue=queue) + + if param.get('transaction_type') == 'transfer': + celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async() + + if param.get('transaction_type') == 'tokengift': + s_process_account_metadata = celery.signature( + 'cic_ussd.tasks.processor.parse_transaction', [{}, transaction], queue=queue + ) + celery.chain(s_process_account_metadata, s_notify_account).apply_async() + + +@celery_app.task +def transaction_callback(result: dict, param: str, status_code: int): + """ + :param result: + :type result: + :param param: + :type param: + :param status_code: + :type status_code: + :return: + :rtype: + """ + if status_code != 0: + raise ValueError(f'Unexpected status code: {status_code}.') + + chain_str = Chain.spec.__str__() + destination_token_symbol = result.get('destination_token_symbol') + destination_token_value = result.get('destination_token_value') + recipient_blockchain_address = result.get('recipient') + sender_blockchain_address = result.get('sender') + source_token_symbol = result.get('source_token_symbol') + source_token_value = result.get('source_token_value') + + recipient_metadata = { + "token_symbol": destination_token_symbol, + "token_value": destination_token_value, + "blockchain_address": recipient_blockchain_address, + "role": "recipient", + "transaction_type": param + } + + get_balances( + address=recipient_blockchain_address, + callback_param=recipient_metadata, + chain_str=chain_str, + callback_task='cic_ussd.tasks.callback_handler.transaction_balances_callback', + token_symbol=destination_token_symbol, + asynchronous=True) + + if param == 'transfer': + sender_metadata = { + "blockchain_address": sender_blockchain_address, + "token_symbol": source_token_symbol, + "token_value": source_token_value, + "role": "sender", + "transaction_type": param } - # retrieve account balances get_balances( - address=recipient_blockchain_address, - callback_param=recipient_metadata, + address=sender_blockchain_address, + callback_param=sender_metadata, chain_str=chain_str, - callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback', - token_symbol=destination_token_symbol, + callback_task='cic_ussd.tasks.callback_handler.transaction_balances_callback', + token_symbol=source_token_symbol, asynchronous=True) - - # only retrieve sender if transaction is a transfer - if param == 'transfer': - sender_metadata = { - "blockchain_address": sender_blockchain_address, - "token_symbol": source_token_symbol, - "token_value": source_token_value, - "tag": "sender", - "tx_param": param - } - - get_balances( - address=sender_blockchain_address, - callback_param=sender_metadata, - chain_str=chain_str, - callback_task='cic_ussd.tasks.callback_handler.process_transaction_balances_callback', - token_symbol=source_token_symbol, - asynchronous=True) - else: - raise ValueError(f'Unexpected status code: {status_code}.') - - -@celery_app.task(bind=True) -def process_transaction_balances_callback(self, result: list, param: dict, status_code: int): - queue = self.request.delivery_info.get('routing_key') - if status_code == 0: - # retrieve balance data - balances_data = result[0] - operational_balance = compute_operational_balance(balances=balances_data) - - # retrieve account's address - blockchain_address = param.get('blockchain_address') - - # append balance to transaction metadata - transaction_metadata = param - transaction_metadata['operational_balance'] = operational_balance - - # retrieve account's preferences - s_preferences_metadata = celery.signature( - 'cic_ussd.tasks.metadata.query_preferences_metadata', - [blockchain_address], - queue=queue - ) - - # parse metadata and run validations - s_process_account_metadata = celery.signature( - 'cic_ussd.tasks.processor.process_tx_metadata_for_notification', - [transaction_metadata], - queue=queue - ) - - # issue notification of transaction - s_notify_account = celery.signature( - 'cic_ussd.tasks.notifications.notify_account_of_transaction', - queue=queue - ) - - if param.get('tx_param') == 'transfer': - celery.chain(s_preferences_metadata, s_process_account_metadata, s_notify_account).apply_async() - - if param.get('tx_param') == 'tokengift': - s_process_account_metadata = celery.signature( - 'cic_ussd.tasks.processor.process_tx_metadata_for_notification', - [{}, transaction_metadata], - queue=queue - ) - celery.chain(s_process_account_metadata, s_notify_account).apply_async() - else: - raise ValueError(f'Unexpected status code: {status_code}.') - - -@celery_app.task -def process_balances_callback(result: list, param: str, status_code: int): - if status_code == 0: - balances_data = result[0] - blockchain_address = balances_data.get('address') - key = create_cached_data_key( - identifier=bytes.fromhex(blockchain_address[2:]), - salt=':cic.balances_data' - ) - cache_data(key=key, data=json.dumps(balances_data)) - logg.debug(f'caching: {balances_data} with key: {key}') - else: - raise ValueError(f'Unexpected status code: {status_code}.') - - -# TODO: clean up this handler -def define_transaction_action_tag( - preferred_language: str, - sender_blockchain_address: str, - param: str): - # check if out going ot incoming transaction - if sender_blockchain_address == param: - # check preferred language - if preferred_language == 'en': - action_tag = 'SENT' - direction = 'TO' - else: - action_tag = 'ULITUMA' - direction = 'KWA' - else: - if preferred_language == 'en': - action_tag = 'RECEIVED' - direction = 'FROM' - else: - action_tag = 'ULIPOKEA' - direction = 'KUTOKA' - return action_tag, direction - - -@celery_app.task -def process_statement_callback(result, param: str, status_code: int): - if status_code == 0: - # create session - processed_transactions = [] - - # process transaction data to cache - for transaction in result: - sender_blockchain_address = transaction.get('sender') - recipient_address = transaction.get('recipient') - source_token = transaction.get('source_token') - - # filter out any transactions that are "gassy" - if '0x0000000000000000000000000000000000000000' in source_token: - pass - else: - session = SessionBase.create_session() - # describe a processed transaction - processed_transaction = {} - - # check if sender is in the system - sender: Account = session.query(Account).filter_by(blockchain_address=sender_blockchain_address).first() - owner: Account = session.query(Account).filter_by(blockchain_address=param).first() - if sender: - processed_transaction['sender_phone_number'] = sender.phone_number - - action_tag, direction = define_transaction_action_tag( - preferred_language=owner.preferred_language, - sender_blockchain_address=sender_blockchain_address, - param=param - ) - processed_transaction['action_tag'] = action_tag - processed_transaction['direction'] = direction - - else: - processed_transaction['sender_phone_number'] = 'GRASSROOTS ECONOMICS' - - # check if recipient is in the system - recipient: Account = session.query(Account).filter_by(blockchain_address=recipient_address).first() - if recipient: - processed_transaction['recipient_phone_number'] = recipient.phone_number - - else: - logg.warning(f'Tx with recipient not found in cic-ussd') - - session.close() - - # add transaction values - processed_transaction['to_value'] = from_wei(value=transaction.get('to_value')).__str__() - processed_transaction['from_value'] = from_wei(value=transaction.get('from_value')).__str__() - - raw_timestamp = transaction.get('timestamp') - timestamp = datetime.utcfromtimestamp(raw_timestamp).strftime('%d/%m/%y, %H:%M') - processed_transaction['timestamp'] = timestamp - - processed_transactions.append(processed_transaction) - - # cache account statement - identifier = bytes.fromhex(param[2:]) - key = create_cached_data_key(identifier=identifier, salt=':cic.statement') - data = json.dumps(processed_transactions) - - # cache statement data - cache_data(key=key, data=data) - else: - raise ValueError(f'Unexpected status code: {status_code}.') diff --git a/apps/cic-ussd/cic_ussd/tasks/logger.py b/apps/cic-ussd/cic_ussd/tasks/logger.py deleted file mode 100644 index 5b1bf917..00000000 --- a/apps/cic-ussd/cic_ussd/tasks/logger.py +++ /dev/null @@ -1,11 +0,0 @@ -# third-party imports -import celery -import logging - -celery_app = celery.current_app -logg = logging.getLogger() - - -@celery_app.task() -def log_it_plz(whatever): - logg.info('logged it plz: {}'.format(whatever)) diff --git a/apps/cic-ussd/cic_ussd/tasks/metadata.py b/apps/cic-ussd/cic_ussd/tasks/metadata.py index 932d444f..ff16970d 100644 --- a/apps/cic-ussd/cic_ussd/tasks/metadata.py +++ b/apps/cic-ussd/cic_ussd/tasks/metadata.py @@ -6,11 +6,7 @@ import celery from hexathon import strip_0x # local imports -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.metadata.custom import CustomMetadata -from cic_ussd.metadata.person import PersonMetadata -from cic_ussd.metadata.phone import PhonePointerMetadata -from cic_ussd.metadata.preferences import PreferencesMetadata +from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata from cic_ussd.tasks.base import CriticalMetadataTask celery_app = celery.current_app @@ -25,8 +21,7 @@ def query_person_metadata(blockchain_address: str): :return: :rtype: """ - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) - logg.debug(f'Retrieving person metadata for address: {blockchain_address}.') + identifier = bytes.fromhex(strip_0x(blockchain_address)) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.query() @@ -41,14 +36,14 @@ def create_person_metadata(blockchain_address: str, data: dict): :return: :rtype: """ - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + identifier = bytes.fromhex(strip_0x(blockchain_address)) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.create(data=data) @celery_app.task def edit_person_metadata(blockchain_address: str, data: dict): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + identifier = bytes.fromhex(strip_0x(blockchain_address)) person_metadata_client = PersonMetadata(identifier=identifier) person_metadata_client.edit(data=data) @@ -63,16 +58,16 @@ def add_phone_pointer(self, blockchain_address: str, phone_number: str): @celery_app.task() def add_custom_metadata(blockchain_address: str, data: dict): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + identifier = bytes.fromhex(strip_0x(blockchain_address)) custom_metadata_client = CustomMetadata(identifier=identifier) custom_metadata_client.create(data=data) @celery_app.task() def add_preferences_metadata(blockchain_address: str, data: dict): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) - custom_metadata_client = PreferencesMetadata(identifier=identifier) - custom_metadata_client.create(data=data) + identifier = bytes.fromhex(strip_0x(blockchain_address)) + preferences_metadata_client = PreferencesMetadata(identifier=identifier) + preferences_metadata_client.create(data=data) @celery_app.task() @@ -81,7 +76,7 @@ def query_preferences_metadata(blockchain_address: str): :param blockchain_address: Blockchain address of an account. :type blockchain_address: str | Ox-hex """ - identifier = blockchain_address_to_metadata_pointer(blockchain_address=blockchain_address) + identifier = bytes.fromhex(strip_0x(blockchain_address)) logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.') person_metadata_client = PreferencesMetadata(identifier=identifier) return person_metadata_client.query() diff --git a/apps/cic-ussd/cic_ussd/tasks/notifications.py b/apps/cic-ussd/cic_ussd/tasks/notifications.py index 1e0937f0..c290bf51 100644 --- a/apps/cic-ussd/cic_ussd/tasks/notifications.py +++ b/apps/cic-ussd/cic_ussd/tasks/notifications.py @@ -6,6 +6,7 @@ import logging import celery # local imports +from cic_ussd.account.transaction import from_wei from cic_ussd.notifications import Notifier from cic_ussd.phone_number import Support @@ -15,56 +16,46 @@ notifier = Notifier() @celery_app.task -def notify_account_of_transaction(notification_data: dict): +def transaction(notification_data: dict): """ :param notification_data: :type notification_data: :return: :rtype: """ - - account_tx_role = notification_data.get('account_tx_role') - amount = notification_data.get('amount') - balance = notification_data.get('balance') + role = notification_data.get('role') + amount = from_wei(notification_data.get('token_value')) + balance = notification_data.get('available_balance') phone_number = notification_data.get('phone_number') preferred_language = notification_data.get('preferred_language') token_symbol = notification_data.get('token_symbol') - transaction_account_metadata = notification_data.get('transaction_account_metadata') + transaction_account_metadata = notification_data.get('metadata_id') transaction_type = notification_data.get('transaction_type') - timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p') if transaction_type == 'tokengift': - support_phone = Support.phone_number notifier.send_sms_notification( key='sms.account_successfully_created', phone_number=phone_number, preferred_language=preferred_language, - balance=balance, - support_phone=support_phone, - token_symbol=token_symbol - ) + support_phone=Support.phone_number) if transaction_type == 'transfer': - if account_tx_role == 'recipient': - notifier.send_sms_notification( - key='sms.received_tokens', - phone_number=phone_number, - preferred_language=preferred_language, - amount=amount, - token_symbol=token_symbol, - tx_sender_information=transaction_account_metadata, - timestamp=timestamp, - balance=balance - ) - else: - notifier.send_sms_notification( - key='sms.sent_tokens', - phone_number=phone_number, - preferred_language=preferred_language, - amount=amount, - token_symbol=token_symbol, - tx_recipient_information=transaction_account_metadata, - timestamp=timestamp, - balance=balance - ) + if role == 'recipient': + notifier.send_sms_notification('sms.received_tokens', + phone_number=phone_number, + preferred_language=preferred_language, + amount=amount, + token_symbol=token_symbol, + tx_sender_information=transaction_account_metadata, + timestamp=timestamp, + balance=balance) + if role == 'sender': + notifier.send_sms_notification('sms.sent_tokens', + phone_number=phone_number, + preferred_language=preferred_language, + amount=amount, + token_symbol=token_symbol, + tx_recipient_information=transaction_account_metadata, + timestamp=timestamp, + balance=balance) diff --git a/apps/cic-ussd/cic_ussd/tasks/processor.py b/apps/cic-ussd/cic_ussd/tasks/processor.py index 7ff6c955..4a614c15 100644 --- a/apps/cic-ussd/cic_ussd/tasks/processor.py +++ b/apps/cic-ussd/cic_ussd/tasks/processor.py @@ -1,88 +1,84 @@ # standard imports +import json import logging # third-party imports import celery -from i18n import config +import i18n +from chainlib.hash import strip_0x # local imports -from cic_ussd.account import define_account_tx_metadata -from cic_ussd.db.models.account import Account +from cic_ussd.account.statement import get_cached_statement +from cic_ussd.account.transaction import aux_transaction_data, validate_transaction_account +from cic_ussd.cache import cache_data, cache_data_key from cic_ussd.db.models.base import SessionBase -from cic_ussd.error import UnknownUssdRecipient -from cic_ussd.transactions import from_wei celery_app = celery.current_app logg = logging.getLogger(__file__) +@celery_app.task(bind=True) +def generate_statement(self, querying_party: str, transaction: dict): + """""" + queue = self.request.delivery_info.get('routing_key') + + s_preferences = celery.signature( + 'cic_ussd.tasks.metadata.query_preferences_metadata', [querying_party], queue=queue + ) + s_parse_transaction = celery.signature( + 'cic_ussd.tasks.processor.parse_transaction', [transaction], queue=queue + ) + s_cache_statement = celery.signature( + 'cic_ussd.tasks.processor.cache_statement', [querying_party], queue=queue + ) + celery.chain(s_preferences, s_parse_transaction, s_cache_statement).apply_async() + + @celery_app.task -def process_tx_metadata_for_notification(result: celery.Task, transaction_metadata: dict): +def cache_statement(parsed_transaction: dict, querying_party: str): """ - :param result: - :type result: - :param transaction_metadata: - :type transaction_metadata: + :param parsed_transaction: + :type parsed_transaction: + :param querying_party: + :type querying_party: :return: :rtype: """ - notification_data = {} + cached_statement = get_cached_statement(querying_party) + statement_transactions = [] + if cached_statement: + statement_transactions = json.loads(cached_statement) + statement_transactions.append(parsed_transaction) + data = json.dumps(statement_transactions) + identifier = bytes.fromhex(strip_0x(querying_party)) + key = cache_data_key(identifier, ':cic.statement') + cache_data(key, data) - # get preferred language - preferred_language = result.get('preferred_language') + +@celery_app.task +def parse_transaction(preferences: dict, transaction: dict) -> dict: + """This function parses transaction objects and collates all relevant data for system use i.e: + - An account's set preferred language. + - Account identifier that facilitates notification. + - Contextual tags i.e action and direction tags. + :param preferences: An account's set preferences. + :type preferences: dict + :param transaction: Transaction object. + :type transaction: dict + :return: Transaction object with contextual data for use in the system. + :rtype: dict + """ + preferred_language = preferences.get('preferred_language') if not preferred_language: - preferred_language = config.get('fallback') - notification_data['preferred_language'] = preferred_language + preferred_language = i18n.config.get('fallback') - # validate account information against present ussd storage data. + transaction = aux_transaction_data(preferred_language, transaction) session = SessionBase.create_session() - blockchain_address = transaction_metadata.get('blockchain_address') - tag = transaction_metadata.get('tag') - account = session.query(Account).filter_by(blockchain_address=blockchain_address).first() - if not account and tag == 'recipient': - session.close() - raise UnknownUssdRecipient( - f'Tx for recipient: {blockchain_address} was received but has no matching user in the system.' - ) - - # get phone number associated with account - phone_number = account.phone_number - notification_data['phone_number'] = phone_number - - # get account's role in transaction i.e sender / recipient - tx_param = transaction_metadata.get('tx_param') - notification_data['transaction_type'] = tx_param - - # get token amount and symbol - if tag == 'recipient': - account_tx_role = tag - amount = transaction_metadata.get('token_value') - amount = from_wei(value=amount) - token_symbol = transaction_metadata.get('token_symbol') - else: - account_tx_role = tag - amount = transaction_metadata.get('token_value') - amount = from_wei(value=amount) - token_symbol = transaction_metadata.get('token_symbol') - notification_data['account_tx_role'] = account_tx_role - notification_data['amount'] = amount - notification_data['token_symbol'] = token_symbol - - # get account's standard ussd identification pattern - if tx_param == 'transfer': - tx_account_metadata = define_account_tx_metadata(user=account) - notification_data['transaction_account_metadata'] = tx_account_metadata - - if tag == 'recipient': - notification_data['notification_key'] = 'sms.received_tokens' - else: - notification_data['notification_key'] = 'sms.sent_tokens' - - if tx_param == 'tokengift': - notification_data['notification_key'] = 'sms.account_successfully_created' - - # get account's balance - notification_data['balance'] = transaction_metadata.get('operational_balance') - - return notification_data + account = validate_transaction_account(session, transaction) + metadata_id = account.standard_metadata_id() + transaction['metadata_id'] = metadata_id + transaction['phone_number'] = account.phone_number + session.commit() + session.close() + return transaction diff --git a/apps/cic-ussd/cic_ussd/tasks/ussd_session.py b/apps/cic-ussd/cic_ussd/tasks/ussd_session.py index dac199f2..e3e952e0 100644 --- a/apps/cic-ussd/cic_ussd/tasks/ussd_session.py +++ b/apps/cic-ussd/cic_ussd/tasks/ussd_session.py @@ -7,10 +7,10 @@ import celery from celery.utils.log import get_logger # local imports +from cic_ussd.cache import Cache, get_cached_data from cic_ussd.db.models.base import SessionBase from cic_ussd.db.models.ussd_session import UssdSession from cic_ussd.error import SessionNotFoundError -from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession from cic_ussd.tasks.base import CriticalSQLAlchemyTask celery_app = celery.current_app @@ -28,46 +28,36 @@ def persist_session_to_db(external_session_id: str): :raises SessionNotFoundError: If the session object is not found in memory. :raises VersionTooLowError: If the session's version doesn't match the latest version. """ - # create session session = SessionBase.create_session() - - # get ussd session in redis cache - in_memory_session = InMemoryUssdSession.redis_cache.get(external_session_id) - - # process persistence to db - if in_memory_session: - in_memory_session = json.loads(in_memory_session) - in_db_ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first() - if in_db_ussd_session: - in_db_ussd_session.update( + cached_ussd_session = get_cached_data(external_session_id) + if cached_ussd_session: + cached_ussd_session = json.loads(cached_ussd_session) + ussd_session = session.query(UssdSession).filter_by(external_session_id=external_session_id).first() + if ussd_session: + ussd_session.update( session=session, - user_input=in_memory_session.get('user_input'), - state=in_memory_session.get('state'), - version=in_memory_session.get('version'), + user_input=cached_ussd_session.get('user_input'), + state=cached_ussd_session.get('state'), + version=cached_ussd_session.get('version'), ) else: - in_db_ussd_session = UssdSession( + ussd_session = UssdSession( external_session_id=external_session_id, - service_code=in_memory_session.get('service_code'), - msisdn=in_memory_session.get('msisdn'), - user_input=in_memory_session.get('user_input'), - state=in_memory_session.get('state'), - version=in_memory_session.get('version'), + service_code=cached_ussd_session.get('service_code'), + msisdn=cached_ussd_session.get('msisdn'), + user_input=cached_ussd_session.get('user_input'), + state=cached_ussd_session.get('state'), + version=cached_ussd_session.get('version'), ) - - # handle the updating of session data for persistence to db - session_data = in_memory_session.get('session_data') - - if session_data: - for key, value in session_data.items(): - in_db_ussd_session.set_data(key=key, value=value, session=session) - - session.add(in_db_ussd_session) + data = cached_ussd_session.get('data') + if data: + for key, value in data.items(): + ussd_session.set_data(key=key, value=value, session=session) + session.add(ussd_session) session.commit() session.close() - InMemoryUssdSession.redis_cache.expire(external_session_id, timedelta(minutes=1)) + Cache.store.expire(external_session_id, timedelta(minutes=1)) else: session.close() raise SessionNotFoundError('Session does not exist!') - session.close() diff --git a/apps/cic-ussd/cic_ussd/transactions.py b/apps/cic-ussd/cic_ussd/transactions.py deleted file mode 100644 index e9964ce3..00000000 --- a/apps/cic-ussd/cic_ussd/transactions.py +++ /dev/null @@ -1,79 +0,0 @@ -# standard imports -import decimal -import logging -from datetime import datetime - -# third-party imports -from cic_eth.api import Api - -# local imports -from cic_ussd.balance import get_balances, get_cached_operational_balance -from cic_ussd.notifications import Notifier -from cic_ussd.phone_number import Support - -logg = logging.getLogger() -notifier = Notifier() - - -def truncate(value: float, decimals: int): - """This function truncates a value to a specified number of decimals places. - :param value: The value to be truncated. - :type value: float - :param decimals: The number of decimals for the value to be truncated to - :type decimals: int - :return: The truncated value. - :rtype: int - """ - decimal.getcontext().rounding = decimal.ROUND_DOWN - contextualized_value = decimal.Decimal(value) - return round(contextualized_value, decimals) - - -def from_wei(value: int) -> float: - """This function converts values in Wei to a token in the cic network. - :param value: Value in Wei - :type value: int - :return: SRF equivalent of value in Wei - :rtype: float - """ - value = float(value) / 1e+6 - return truncate(value=value, decimals=2) - - -def to_wei(value: int) -> int: - """This functions converts values from a token in the cic network to Wei. - :param value: Value in SRF - :type value: int - :return: Wei equivalent of value in SRF - :rtype: int - """ - return int(value * 1e+6) - - -class OutgoingTransactionProcessor: - - def __init__(self, chain_str: str, from_address: str, to_address: str): - """ - :param chain_str: The chain name and network id. - :type chain_str: str - :param from_address: Ethereum address of the sender - :type from_address: str, 0x-hex - :param to_address: Ethereum address of the recipient - :type to_address: str, 0x-hex - """ - self.chain_str = chain_str - self.cic_eth_api = Api(chain_str=chain_str) - self.from_address = from_address - self.to_address = to_address - - def process_outgoing_transfer_transaction(self, amount: int, token_symbol: str): - """This function initiates standard transfers between one account to another - :param amount: The amount of tokens to be sent - :type amount: int - :param token_symbol: ERC20 token symbol of token to send - :type token_symbol: str - """ - self.cic_eth_api.transfer(from_address=self.from_address, - to_address=self.to_address, - value=to_wei(value=amount), - token_symbol=token_symbol) diff --git a/apps/cic-ussd/cic_ussd/validator.py b/apps/cic-ussd/cic_ussd/validator.py index a1706f9a..a3dfde06 100644 --- a/apps/cic-ussd/cic_ussd/validator.py +++ b/apps/cic-ussd/cic_ussd/validator.py @@ -1,15 +1,13 @@ # standard imports +import ipaddress import logging import os import re -import ipaddress # third-party imports from confini import Config # local imports -from cic_ussd.db.models.account import Account -from cic_ussd.db.models.base import SessionBase logg = logging.getLogger(__file__) @@ -46,23 +44,6 @@ def check_request_content_length(config: Config, env: dict): config.get('APP_MAX_BODY_LENGTH')) -def check_known_user(phone_number: str, session): - """This method attempts to ascertain whether the user already exists and is known to the system. - It sends a get request to the platform application and attempts to retrieve the user's data which it persists in - memory. - :param phone_number: A valid phone number - :type phone_number: str - :param session: - :type session: - :return: Is known phone number - :rtype: boolean - """ - session = SessionBase.bind_session(session=session) - account = session.query(Account).filter_by(phone_number=phone_number).first() - SessionBase.release_session(session=session) - return account is not None - - def check_request_method(env: dict): """ Checks whether request method is POST @@ -74,17 +55,6 @@ def check_request_method(env: dict): return env.get('REQUEST_METHOD').upper() == 'POST' -def check_session_id(session_id: str): - """ - Checks whether session id is present - :param session_id: Session id value provided by AT - :type session_id: str - :return: Session id presence - :rtype: boolean - """ - return session_id is not None - - def validate_phone_number(phone: str): """ Check if phone number is in the correct format. @@ -93,12 +63,10 @@ def validate_phone_number(phone: str): :return: Whether the phone number is of the correct format. :rtype: bool """ - if phone and re.match('[+]?[0-9]{10,12}$', phone): - return True - return False + return bool(phone and re.match('[+]?[0-9]{10,12}$', phone)) -def validate_response_type(processor_response: str) -> bool: +def is_valid_response(processor_response: str) -> bool: """ This function checks the prefix for a corresponding menu's text from the response offered by the Ussd Processor and determines whether the response should prompt the end of a ussd session or the @@ -111,9 +79,7 @@ def validate_response_type(processor_response: str) -> bool: if len(processor_response) > 164: logg.warning(f'Warning, text has length {len(processor_response)}, display may be truncated') - if re.match(matcher, processor_response): - return True - return False + return bool(re.match(matcher, processor_response)) def validate_presence(path: str): diff --git a/apps/cic-ussd/config/app.ini b/apps/cic-ussd/config/app.ini new file mode 100644 index 00000000..4c5e19db --- /dev/null +++ b/apps/cic-ussd/config/app.ini @@ -0,0 +1,13 @@ +[app] +allowed_ip=0.0.0.0/0 +max_body_length=1024 +password_pepper= + +[machine] +states=states/ +transitions=transitions/ + +[client] +host = +port = +ssl = diff --git a/apps/cic-ussd/config/celery.ini b/apps/cic-ussd/config/celery.ini new file mode 100644 index 00000000..657a27f7 --- /dev/null +++ b/apps/cic-ussd/config/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url=redis:// +result_url=redis:// diff --git a/apps/cic-ussd/.config/cic.ini b/apps/cic-ussd/config/cic.ini similarity index 100% rename from apps/cic-ussd/.config/cic.ini rename to apps/cic-ussd/config/cic.ini diff --git a/apps/cic-ussd/.config_client/app.ini b/apps/cic-ussd/config/client/app.ini similarity index 100% rename from apps/cic-ussd/.config_client/app.ini rename to apps/cic-ussd/config/client/app.ini diff --git a/apps/cic-ussd/.config_client/client.ini b/apps/cic-ussd/config/client/client.ini similarity index 100% rename from apps/cic-ussd/.config_client/client.ini rename to apps/cic-ussd/config/client/client.ini diff --git a/apps/cic-ussd/.config_client/ussd.ini b/apps/cic-ussd/config/client/ussd.ini similarity index 100% rename from apps/cic-ussd/.config_client/ussd.ini rename to apps/cic-ussd/config/client/ussd.ini diff --git a/apps/cic-ussd/config/database.ini b/apps/cic-ussd/config/database.ini new file mode 100644 index 00000000..649dd17c --- /dev/null +++ b/apps/cic-ussd/config/database.ini @@ -0,0 +1,10 @@ +[database] +name=cic_ussd +user=postgres +password= +host=localhost +port=5432 +engine=postgresql +driver=psycopg2 +debug=0 +pool_size=1 diff --git a/apps/cic-ussd/.config/pgp.ini b/apps/cic-ussd/config/pgp.ini similarity index 100% rename from apps/cic-ussd/.config/pgp.ini rename to apps/cic-ussd/config/pgp.ini diff --git a/apps/cic-ussd/config/phone.ini b/apps/cic-ussd/config/phone.ini new file mode 100644 index 00000000..64e9ae50 --- /dev/null +++ b/apps/cic-ussd/config/phone.ini @@ -0,0 +1,5 @@ +[e164] +region=KE + +[office] +support_phone=0757628885 diff --git a/apps/cic-ussd/.config/pip.ini b/apps/cic-ussd/config/pip.ini similarity index 100% rename from apps/cic-ussd/.config/pip.ini rename to apps/cic-ussd/config/pip.ini diff --git a/apps/cic-ussd/config/redis.ini b/apps/cic-ussd/config/redis.ini new file mode 100644 index 00000000..d9f54d0b --- /dev/null +++ b/apps/cic-ussd/config/redis.ini @@ -0,0 +1,5 @@ +[redis] +host=redis +database=0 +password= +port=6379 diff --git a/apps/cic-ussd/config/test/app.ini b/apps/cic-ussd/config/test/app.ini new file mode 100644 index 00000000..1aa92a72 --- /dev/null +++ b/apps/cic-ussd/config/test/app.ini @@ -0,0 +1,8 @@ +[app] +allowed_ip=127.0.0.1 +max_body_length=1024 +password_pepper=QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I= + +[machine] +states=states/ +transitions=transitions/ diff --git a/apps/cic-ussd/config/test/celery.ini b/apps/cic-ussd/config/test/celery.ini new file mode 100644 index 00000000..30a519c7 --- /dev/null +++ b/apps/cic-ussd/config/test/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url = filesystem:// +result_url = filesystem:// diff --git a/apps/cic-ussd/.config/test/cic.ini b/apps/cic-ussd/config/test/cic.ini similarity index 63% rename from apps/cic-ussd/.config/test/cic.ini rename to apps/cic-ussd/config/test/cic.ini index 0363ae1f..1feb345d 100644 --- a/apps/cic-ussd/.config/test/cic.ini +++ b/apps/cic-ussd/config/test/cic.ini @@ -2,4 +2,4 @@ engine = evm common_name = bloxberg network_id = 8996 -meta_url = http://localhost:63380 +meta_url = http://test-meta.io diff --git a/apps/cic-ussd/config/test/database.ini b/apps/cic-ussd/config/test/database.ini new file mode 100644 index 00000000..9726d3d2 --- /dev/null +++ b/apps/cic-ussd/config/test/database.ini @@ -0,0 +1,9 @@ +[database] +name=cic_ussd_test +user=postgres +password= +host=localhost +port=5432 +engine=sqlite +driver=pysqlite +debug= diff --git a/apps/cic-ussd/.config/test/integration.ini b/apps/cic-ussd/config/test/integration.ini similarity index 100% rename from apps/cic-ussd/.config/test/integration.ini rename to apps/cic-ussd/config/test/integration.ini diff --git a/apps/cic-ussd/config/test/pgp.ini b/apps/cic-ussd/config/test/pgp.ini new file mode 100644 index 00000000..421b6a1d --- /dev/null +++ b/apps/cic-ussd/config/test/pgp.ini @@ -0,0 +1,5 @@ +[pgp] +export_dir = +keys_path =tests/data/pgp +private_keys = privatekeys_meta.asc +passphrase = merman diff --git a/apps/cic-ussd/config/test/phone.ini b/apps/cic-ussd/config/test/phone.ini new file mode 100644 index 00000000..64e9ae50 --- /dev/null +++ b/apps/cic-ussd/config/test/phone.ini @@ -0,0 +1,5 @@ +[e164] +region=KE + +[office] +support_phone=0757628885 diff --git a/apps/cic-ussd/.config/test/pip.ini b/apps/cic-ussd/config/test/pip.ini similarity index 100% rename from apps/cic-ussd/.config/test/pip.ini rename to apps/cic-ussd/config/test/pip.ini diff --git a/apps/cic-ussd/config/test/redis.ini b/apps/cic-ussd/config/test/redis.ini new file mode 100644 index 00000000..1d5fb3b9 --- /dev/null +++ b/apps/cic-ussd/config/test/redis.ini @@ -0,0 +1,5 @@ +[redis] +host=localhost +database=0 +password= +port=6379 diff --git a/apps/cic-ussd/config/test/translations.ini b/apps/cic-ussd/config/test/translations.ini new file mode 100644 index 00000000..f32c64ed --- /dev/null +++ b/apps/cic-ussd/config/test/translations.ini @@ -0,0 +1,3 @@ +[locale] +fallback=sw +path=var/lib/locale/ diff --git a/apps/cic-ussd/config/test/ussd.ini b/apps/cic-ussd/config/test/ussd.ini new file mode 100644 index 00000000..cb8ba9e0 --- /dev/null +++ b/apps/cic-ussd/config/test/ussd.ini @@ -0,0 +1,5 @@ +[ussd] +menu_file=cic_ussd/db/ussd_menu.json +service_code=*483*46#,*483*061#,*384*96# +user = +pass = diff --git a/apps/cic-ussd/config/translations.ini b/apps/cic-ussd/config/translations.ini new file mode 100644 index 00000000..f32c64ed --- /dev/null +++ b/apps/cic-ussd/config/translations.ini @@ -0,0 +1,3 @@ +[locale] +fallback=sw +path=var/lib/locale/ diff --git a/apps/cic-ussd/config/ussd.ini b/apps/cic-ussd/config/ussd.ini new file mode 100644 index 00000000..d996d16d --- /dev/null +++ b/apps/cic-ussd/config/ussd.ini @@ -0,0 +1,5 @@ +[ussd] +menu_file=data/ussd_menu.json +service_code=*483*46#,*483*061#,*384*96# +user = +pass = diff --git a/apps/cic-ussd/docker/Dockerfile b/apps/cic-ussd/docker/Dockerfile index 81ceb733..8de82801 100644 --- a/apps/cic-ussd/docker/Dockerfile +++ b/apps/cic-ussd/docker/Dockerfile @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:1.2 FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev - +RUN apt-get install -y redis-server # create secrets directory RUN mkdir -vp pgp/keys @@ -26,7 +26,7 @@ COPY docker/*.sh . RUN chmod +x /root/*.sh # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts -COPY .config/ /usr/local/etc/cic-ussd/ +COPY config/ /usr/local/etc/cic-ussd/ COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic ENTRYPOINT [] diff --git a/apps/cic-ussd/docker/Dockerfile_ci b/apps/cic-ussd/docker/Dockerfile_ci index e431b264..9811f563 100644 --- a/apps/cic-ussd/docker/Dockerfile_ci +++ b/apps/cic-ussd/docker/Dockerfile_ci @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:1.2 FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev - +RUN apt-get install -y redis-server # create secrets directory RUN mkdir -vp pgp/keys @@ -26,7 +26,7 @@ COPY docker/*.sh . RUN chmod +x /root/*.sh # copy config and migration files to definitive file so they can be referenced in path definitions for running scripts -COPY .config/ /usr/local/etc/cic-ussd/ +COPY config/ /usr/local/etc/cic-ussd/ COPY cic_ussd/db/migrations/ /usr/local/share/cic-ussd/alembic ENTRYPOINT [] diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index 24944960..b0b81f95 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -1,17 +1,17 @@ -cic-eth~=0.12.2a3 +alembic==1.4.2 +bcrypt==3.2.0 +celery==4.4.7 +cic-eth[services]==0.12.2a3 cic-notify~=0.4.0a10 cic-types~=0.1.0a14 confini~=0.4.1a1 -semver==2.13.0 -alembic==1.4.2 -SQLAlchemy==1.3.20 -psycopg2==2.8.6 -tinydb==4.2.0 phonenumbers==8.12.12 -redis==3.5.3 -celery==4.4.7 +psycopg2==2.8.6 python-i18n[YAML]==0.3.9 pyxdg==0.27 -bcrypt==3.2.0 -uWSGI==2.0.19.1 +redis==3.5.3 +semver==2.13.0 +SQLAlchemy==1.3.20 +tinydb==4.2.0 transitions==0.8.4 +uWSGI==2.0.19.1 \ No newline at end of file diff --git a/apps/cic-ussd/scripts/migrate.py b/apps/cic-ussd/scripts/migrate.py index 0c4cb76a..d7321c0f 100644 --- a/apps/cic-ussd/scripts/migrate.py +++ b/apps/cic-ussd/scripts/migrate.py @@ -19,7 +19,7 @@ root_directory = os.path.dirname(os.path.dirname(__file__)) db_directory = os.path.join(root_directory, 'cic_ussd', 'db') migrationsdir = os.path.join(db_directory, 'migrations') -config_directory = os.path.join(root_directory, '.config') +config_directory = os.path.join(root_directory, 'config') arg_parser = argparse.ArgumentParser() arg_parser.add_argument('-c', type=str, default=config_directory, help='config file') @@ -51,4 +51,3 @@ ac.set_main_option('sqlalchemy.url', dsn) ac.set_main_option('script_location', migrations_dir) alembic.command.upgrade(ac, 'head') - diff --git a/apps/cic-ussd/setup.cfg b/apps/cic-ussd/setup.cfg index 29a5bf98..b95dae34 100644 --- a/apps/cic-ussd/setup.cfg +++ b/apps/cic-ussd/setup.cfg @@ -29,11 +29,14 @@ licence_files = python_requires = >= 3.6 packages = cic_ussd + cic_ussd.account cic_ussd.db cic_ussd.db.models cic_ussd.files + cic_ussd.http cic_ussd.menu cic_ussd.metadata + cic_ussd.processor cic_ussd.runnable cic_ussd.runnable.daemons cic_ussd.session diff --git a/apps/cic-ussd/states/signup_states.json b/apps/cic-ussd/states/signup_states.json index c2b2876f..1e6722d2 100644 --- a/apps/cic-ussd/states/signup_states.json +++ b/apps/cic-ussd/states/signup_states.json @@ -3,5 +3,6 @@ "scan_data", "initial_language_selection", "initial_pin_entry", - "initial_pin_confirmation" + "initial_pin_confirmation", + "change_preferred_language" ] diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_balance.py b/apps/cic-ussd/tests/cic_ussd/account/test_balance.py new file mode 100644 index 00000000..554f7fb6 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_balance.py @@ -0,0 +1,68 @@ +# standard imports + +# external imports +import pytest + +# local imports +from cic_ussd.account.balance import calculate_available_balance, get_balances, get_cached_available_balance +from cic_ussd.account.chain import Chain +from cic_ussd.error import CachedDataNotFoundError + +# test imports +from tests.helpers.accounts import blockchain_address + + +def test_async_get_balances(activated_account, + celery_session_worker, + load_chain_spec, + load_config, + mock_async_balance_api_query): + blockchain_address = activated_account.blockchain_address + chain_str = Chain.spec.__str__() + token_symbol = load_config.get('TEST_TOKEN_SYMBOL') + get_balances(blockchain_address, chain_str, token_symbol, asynchronous=True) + assert mock_async_balance_api_query.get('address') == blockchain_address + assert mock_async_balance_api_query.get('token_symbol') == token_symbol + + +def test_sync_get_balances(activated_account, + balances, + celery_session_worker, + load_chain_spec, + load_config, + mock_sync_balance_api_query): + blockchain_address = activated_account.blockchain_address + chain_str = Chain.spec.__str__() + token_symbol = load_config.get('TEST_TOKEN_SYMBOL') + res = get_balances(blockchain_address, chain_str, token_symbol, asynchronous=False) + assert res == balances + + +@pytest.mark.parametrize('balance_incoming, balance_network, balance_outgoing, available_balance', [ + (0, 50000000, 0, 50.00), + (5000000, 89000000, 67000000, 27.00) +]) +def test_calculate_available_balance(activated_account, + balance_incoming, + balance_network, + balance_outgoing, + available_balance): + balances = { + 'address': activated_account.blockchain_address, + 'converters': [], + 'balance_network': balance_network, + 'balance_outgoing': balance_outgoing, + 'balance_incoming': balance_incoming + } + assert calculate_available_balance(balances) == available_balance + + +def test_get_cached_available_balance(activated_account, cache_balances, balances): + cached_available_balance = get_cached_available_balance(activated_account.blockchain_address) + available_balance = calculate_available_balance(balances[0]) + assert cached_available_balance == available_balance + address = blockchain_address() + with pytest.raises(CachedDataNotFoundError) as error: + cached_available_balance = get_cached_available_balance(address) + assert cached_available_balance is None + assert str(error.value) == f'No cached available balance for address: {address}' diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_maps.py b/apps/cic-ussd/tests/cic_ussd/account/test_maps.py new file mode 100644 index 00000000..95323fc5 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_maps.py @@ -0,0 +1,28 @@ +# standard imports + +# external imports +import pytest + +# local imports +from cic_ussd.account.maps import gender, language + +# test imports + + +@pytest.mark.parametrize('key, expected_value', [ + ('1', 'male'), + ('2', 'female'), + ('3', 'other') +]) +def test_gender(key, expected_value): + g_map = gender() + assert g_map[key] == expected_value + + +@pytest.mark.parametrize('key, expected_value', [ + ('1', 'en'), + ('2', 'sw'), +]) +def test_language(key, expected_value): + l_map = language() + assert l_map[key] == expected_value diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_metadata.py b/apps/cic-ussd/tests/cic_ussd/account/test_metadata.py new file mode 100644 index 00000000..9622221a --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_metadata.py @@ -0,0 +1,28 @@ +# standard imports +import json + +# external imports +from cic_types.models.person import get_contact_data_from_vcard + +# local imports +from cic_ussd.account.metadata import get_cached_preferred_language, parse_account_metadata + +# test imports +from tests.helpers.accounts import blockchain_address + + +def test_get_cached_preferred_language(activated_account, cache_preferences, preferences): + cached_preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + assert cached_preferred_language == preferences.get('preferred_language') + cached_preferred_language = get_cached_preferred_language(blockchain_address()) + assert cached_preferred_language is None + + +def test_parse_account_metadata(person_metadata): + contact_information = get_contact_data_from_vcard(person_metadata.get('vcard')) + given_name = contact_information.get('given') + family_name = contact_information.get('family') + phone_number = contact_information.get('tel') + parsed_account_metadata = f'{given_name} {family_name} {phone_number}' + assert parse_account_metadata(person_metadata) == parsed_account_metadata + diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_statement.py b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py new file mode 100644 index 00000000..86d79b2e --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_statement.py @@ -0,0 +1,85 @@ +# standard imports +import json +import time + +# external imports +import pytest +import requests_mock +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.statement import (filter_statement_transactions, + generate, + get_cached_statement, + parse_statement_transactions, + query_statement, + statement_transaction_set) +from cic_ussd.account.transaction import transaction_actors +from cic_ussd.cache import cache_data_key, get_cached_data + +# test imports +from tests.helpers.accounts import blockchain_address + + +def test_filter_statement_transactions(transactions_list): + assert len(transactions_list) == 4 + assert len(filter_statement_transactions(transactions_list)) == 1 + + +def test_generate(activated_account, + cache_preferences, + celery_session_worker, + init_cache, + init_database, + set_locale_files, + preferences, + preferences_metadata_url, + transactions_list): + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('GET', preferences_metadata_url, status_code=200, reason='OK', json=preferences) + statement_transactions = filter_statement_transactions(transactions_list) + for transaction in statement_transactions: + querying_party = activated_account.blockchain_address + recipient_transaction, sender_transaction = transaction_actors(transaction) + if recipient_transaction.get('blockchain_address') == querying_party: + generate(querying_party, None, recipient_transaction) + if sender_transaction.get('blockchain_address') == querying_party: + generate(querying_party, None, sender_transaction) + time.sleep(2) + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + key = cache_data_key(identifier, ':cic.statement') + statement = get_cached_data(key) + statement = json.loads(statement) + assert len(statement) == 1 + + +def test_get_cached_statement(activated_account, cache_statement, statement): + cached_statement = get_cached_statement(activated_account.blockchain_address) + assert cached_statement is not None + cached_statement = json.loads(cached_statement) + assert cached_statement[0].get('blockchain_address') == statement[0].get('blockchain_address') + + +def test_parse_statement_transactions(statement): + parsed_transactions = parse_statement_transactions(statement) + parsed_transaction = parsed_transactions[0] + parsed_transaction.startswith('Sent') + + +@pytest.mark.parametrize('blockchain_address, limit', [ + (blockchain_address(), 10), + (blockchain_address(), 5) +]) +def test_query_statement(blockchain_address, limit, load_chain_spec, activated_account, mock_transaction_list_query): + query_statement(blockchain_address, limit) + assert mock_transaction_list_query.get('address') == blockchain_address + assert mock_transaction_list_query.get('limit') == limit + + +def test_statement_transaction_set(preferences, set_locale_files, statement): + parsed_transactions = parse_statement_transactions(statement) + preferred_language = preferences.get('preferred_language') + transaction_set = statement_transaction_set(preferred_language, parsed_transactions) + transaction_set.startswith('Sent') + transaction_set = statement_transaction_set(preferred_language, []) + transaction_set.startswith('No') diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_tokens.py b/apps/cic-ussd/tests/cic_ussd/account/test_tokens.py new file mode 100644 index 00000000..ca7e9cb5 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_tokens.py @@ -0,0 +1,42 @@ +# standard imports +import json + +# external imports +import pytest + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.account.tokens import get_cached_default_token, get_default_token_symbol, query_default_token + + +# test imports + + +def test_get_cached_default_token(cache_default_token_data, default_token_data, load_chain_spec): + chain_str = Chain.spec.__str__() + cached_default_token = get_cached_default_token(chain_str) + cached_default_token_data = json.loads(cached_default_token) + assert cached_default_token_data['symbol'] == default_token_data['symbol'] + assert cached_default_token_data['address'] == default_token_data['address'] + assert cached_default_token_data['name'] == default_token_data['name'] + assert cached_default_token_data['decimals'] == default_token_data['decimals'] + + +def test_get_default_token_symbol_from_api(default_token_data, load_chain_spec, mock_sync_default_token_api_query): + default_token_symbol = get_default_token_symbol() + assert default_token_symbol == default_token_data['symbol'] + + +def test_query_default_token(default_token_data, load_chain_spec, mock_sync_default_token_api_query): + chain_str = Chain.spec.__str__() + queried_default_token_data = query_default_token(chain_str) + assert queried_default_token_data['symbol'] == default_token_data['symbol'] + assert queried_default_token_data['address'] == default_token_data['address'] + assert queried_default_token_data['name'] == default_token_data['name'] + assert queried_default_token_data['decimals'] == default_token_data['decimals'] + + +def test_get_default_token_symbol_from_cache(cache_default_token_data, default_token_data, load_chain_spec): + default_token_symbol = get_default_token_symbol() + assert default_token_symbol is not None + assert default_token_symbol == default_token_data.get('symbol') diff --git a/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py b/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py new file mode 100644 index 00000000..aaf979e5 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/account/test_transaction.py @@ -0,0 +1,110 @@ +# standard imports +from decimal import Decimal + +# external imports +import pytest + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.account.transaction import (aux_transaction_data, + from_wei, + to_wei, + truncate, + transaction_actors, + validate_transaction_account, + OutgoingTransaction) +from cic_ussd.db.models.account import Account +from cic_ussd.error import UnknownUssdRecipient +from cic_ussd.translation import translation_for + + +# test imports + + +def check_aux_data(action_tag_key, direction_tag_key, preferred_language, transaction_with_aux_data): + assert transaction_with_aux_data.get('action_tag') == translation_for(action_tag_key, preferred_language) + assert transaction_with_aux_data.get('direction_tag') == translation_for(direction_tag_key, preferred_language) + + +def test_aux_transaction_data(preferences, set_locale_files, transactions_list): + sample_transaction = transactions_list[0] + preferred_language = preferences.get('preferred_language') + recipient_transaction, sender_transaction = transaction_actors(sample_transaction) + recipient_tx_aux_data = aux_transaction_data(preferred_language, recipient_transaction) + check_aux_data('helpers.received', 'helpers.from', preferred_language, recipient_tx_aux_data) + sender_tx_aux_data = aux_transaction_data(preferred_language, sender_transaction) + check_aux_data('helpers.sent', 'helpers.to', preferred_language, sender_tx_aux_data) + + +@pytest.mark.parametrize("wei, expected_result", [ + (50000000, Decimal('50.00')), + (100000, Decimal('0.10')) +]) +def test_from_wei(wei, expected_result): + assert from_wei(wei) == expected_result + + +@pytest.mark.parametrize("value, expected_result", [ + (50, 50000000), + (0.10, 100000) +]) +def test_to_wei(value, expected_result): + assert to_wei(value) == expected_result + + +@pytest.mark.parametrize("decimals, value, expected_result", [ + (3, 1234.32875, 1234.328), + (2, 98.998, 98.99) +]) +def test_truncate(decimals, value, expected_result): + assert truncate(value=value, decimals=decimals).__float__() == expected_result + + +def test_transaction_actors(activated_account, transaction_result, valid_recipient): + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + assert recipient_transaction.get('blockchain_address') == valid_recipient.blockchain_address + assert sender_transaction.get('blockchain_address') == activated_account.blockchain_address + assert recipient_transaction.get('role') == 'recipient' + assert sender_transaction.get('role') == 'sender' + assert recipient_transaction.get('token_symbol') == transaction_result.get('destination_token_symbol') + assert sender_transaction.get('token_symbol') == transaction_result.get('source_token_symbol') + assert recipient_transaction.get('token_value') == transaction_result.get('destination_token_value') + assert sender_transaction.get('token_value') == transaction_result.get('source_token_value') + + +def test_validate_transaction_account(activated_account, init_database, transactions_list): + sample_transaction = transactions_list[0] + recipient_transaction, sender_transaction = transaction_actors(sample_transaction) + recipient_account = validate_transaction_account(init_database, recipient_transaction) + sender_account = validate_transaction_account(init_database, sender_transaction) + assert isinstance(recipient_account, Account) + assert isinstance(sender_account, Account) + sample_transaction = transactions_list[1] + recipient_transaction, sender_transaction = transaction_actors(sample_transaction) + with pytest.raises(UnknownUssdRecipient) as error: + validate_transaction_account(init_database, recipient_transaction) + assert str( + error.value) == f'Tx for recipient: {recipient_transaction.get("blockchain_address")} has no matching account in the system.' + validate_transaction_account(init_database, sender_transaction) + assert f'Tx from sender: {sender_transaction.get("blockchain_address")} has no matching account in system.' + + +@pytest.mark.parametrize("amount", [50, 0.10]) +def test_outgoing_transaction_processor(activated_account, + amount, + celery_session_worker, + load_config, + load_chain_spec, + mock_transfer_api, + valid_recipient): + chain_str = Chain.spec.__str__() + token_symbol = load_config.get('TEST_TOKEN_SYMBOL') + outgoing_tx_processor = OutgoingTransaction(chain_str, + activated_account.blockchain_address, + valid_recipient.blockchain_address) + + outgoing_tx_processor.transfer(amount, token_symbol) + assert mock_transfer_api.get('from_address') == activated_account.blockchain_address + assert mock_transfer_api.get('to_address') == valid_recipient.blockchain_address + assert mock_transfer_api.get('value') == to_wei(amount) + assert mock_transfer_api.get('token_symbol') == token_symbol diff --git a/apps/cic-ussd/tests/cic_ussd/db/models/test_account.py b/apps/cic-ussd/tests/cic_ussd/db/models/test_account.py new file mode 100644 index 00000000..6baa9b31 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/db/models/test_account.py @@ -0,0 +1,96 @@ +# standard imports +import json + +# external imports +import pytest +from cic_types.models.person import get_contact_data_from_vcard + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.cache import get_cached_data +from cic_ussd.db.enum import AccountStatus +from cic_ussd.db.models.account import Account, create, cache_creation_task_uuid +from cic_ussd.db.models.task_tracker import TaskTracker + +# test imports +from tests.helpers.accounts import blockchain_address, phone_number + + +def test_account(init_database, set_fernet_key): + address = blockchain_address() + phone = phone_number() + account = Account(address, phone) + account.create_password('0000') + account.activate_account() + init_database.add(account) + init_database.commit() + + account = init_database.query(Account).get(1) + assert account.blockchain_address == address + assert account.phone_number == phone + assert account.failed_pin_attempts == 0 + assert account.verify_password('0000') is True + assert account.get_status(init_database) == AccountStatus.ACTIVE.name + + +def test_account_repr(activated_account): + assert repr(activated_account) == f'' + + +def test_account_statuses(init_database, pending_account): + assert pending_account.get_status(init_database) == AccountStatus.PENDING.name + pending_account.create_password('1111') + pending_account.activate_account() + init_database.add(pending_account) + init_database.commit() + assert pending_account.get_status(init_database) == AccountStatus.ACTIVE.name + pending_account.failed_pin_attempts = 3 + assert pending_account.get_status(init_database) == AccountStatus.LOCKED.name + pending_account.reset_pin(init_database) + assert pending_account.get_status(init_database) == AccountStatus.RESET.name + pending_account.activate_account() + assert pending_account.get_status(init_database) == AccountStatus.ACTIVE.name + + +def test_get_by_phone_number(activated_account, init_database): + account = Account.get_by_phone_number(activated_account.phone_number, init_database) + assert account == activated_account + + +def test_has_preferred_language(activated_account, cache_preferences): + assert activated_account.has_preferred_language() is True + + +def test_lacks_preferred_language(activated_account): + assert activated_account.has_preferred_language() is False + + +def test_has_valid_pin(activated_account, init_database, pending_account): + assert activated_account.has_valid_pin(init_database) is True + assert pending_account.has_valid_pin(init_database) is False + + +def test_pin_is_blocked(activated_account, init_database): + assert activated_account.pin_is_blocked(init_database) is False + activated_account.failed_pin_attempts = 3 + init_database.add(activated_account) + init_database.commit() + assert activated_account.pin_is_blocked(init_database) is True + + +def test_standard_metadata_id(activated_account, cache_person_metadata, pending_account, person_metadata): + contact_information = get_contact_data_from_vcard(person_metadata.get('vcard')) + given_name = contact_information.get('given') + family_name = contact_information.get('family') + phone_number = contact_information.get('tel') + parsed_account_metadata = f'{given_name} {family_name} {phone_number}' + assert activated_account.standard_metadata_id() == parsed_account_metadata + assert pending_account.standard_metadata_id() == pending_account.phone_number + + +def test_account_create(init_cache, init_database, load_chain_spec, mock_account_creation_task_result, task_uuid): + chain_str = Chain.spec.__str__() + create(chain_str, phone_number(), init_database) + assert len(init_database.query(TaskTracker).all()) == 1 + account_creation_data = get_cached_data(task_uuid) + assert json.loads(account_creation_data).get('status') == AccountStatus.PENDING.name diff --git a/apps/cic-ussd/tests/cic_ussd/db/models/test_task_tracker.py b/apps/cic-ussd/tests/cic_ussd/db/models/test_task_tracker.py index 936e6d5b..5f2bdc62 100644 --- a/apps/cic-ussd/tests/cic_ussd/db/models/test_task_tracker.py +++ b/apps/cic-ussd/tests/cic_ussd/db/models/test_task_tracker.py @@ -6,9 +6,8 @@ def test_task_tracker(init_database): task_uuid = '31e85315-feee-4b6d-995e-223569082cc4' task_in_tracker = TaskTracker(task_uuid=task_uuid) - session = init_database - session.add(task_in_tracker) - session.commit() + init_database.add(task_in_tracker) + init_database.commit() - queried_task = session.query(TaskTracker).get(1) + queried_task = init_database.query(TaskTracker).get(1) assert queried_task.task_uuid == task_uuid diff --git a/apps/cic-ussd/tests/cic_ussd/db/models/test_user.py b/apps/cic-ussd/tests/cic_ussd/db/models/test_user.py deleted file mode 100644 index 24f6fc99..00000000 --- a/apps/cic-ussd/tests/cic_ussd/db/models/test_user.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tests the persistence of the user record and associated functions to the user object""" - -# standard imports -import pytest - -# platform imports -from cic_ussd.db.models.account import Account - - -def test_user(init_database, set_fernet_key): - user = Account(blockchain_address='0x417f5962fc52dc33ff0689659b25848680dec6dcedc6785b03d1df60fc6d5c51', - phone_number='+254700000000') - user.create_password('0000') - - session = Account.session - session.add(user) - session.commit() - - queried_user = session.query(Account).get(1) - assert queried_user.blockchain_address == '0x417f5962fc52dc33ff0689659b25848680dec6dcedc6785b03d1df60fc6d5c51' - assert queried_user.phone_number == '+254700000000' - assert queried_user.failed_pin_attempts == 0 - assert queried_user.verify_password('0000') is True - - -def test_user_state_transition(create_pending_user): - user = create_pending_user - session = Account.session - - assert user.get_account_status() == 'PENDING' - user.activate_account() - assert user.get_account_status() == 'ACTIVE' - user.failed_pin_attempts = 3 - assert user.get_account_status() == 'LOCKED' - user.reset_account_pin() - assert user.get_account_status() == 'RESET' - user.activate_account() - assert user.get_account_status() == 'ACTIVE' - session.add(user) - session.commit() diff --git a/apps/cic-ussd/tests/cic_ussd/db/models/test_ussd_session.py b/apps/cic-ussd/tests/cic_ussd/db/models/test_ussd_session.py index ab7ed45c..20fd9408 100644 --- a/apps/cic-ussd/tests/cic_ussd/db/models/test_ussd_session.py +++ b/apps/cic-ussd/tests/cic_ussd/db/models/test_ussd_session.py @@ -1,54 +1,78 @@ +# standard imports +import os + # third party imports import pytest +from sqlalchemy import desc # local imports from cic_ussd.db.models.ussd_session import UssdSession from cic_ussd.error import VersionTooLowError -def test_ussd_session(init_database, create_in_redis_ussd_session, create_activated_user): - session = init_database - +def test_ussd_session(activated_account, init_database, init_cache, load_config): + valid_service_codes = load_config.get('USSD_SERVICE_CODE').split(",") ussd_session = UssdSession( - external_session_id='AT65423', - service_code='*123#', - msisdn=create_activated_user.phone_number, + external_session_id=os.urandom(20).hex(), + service_code=valid_service_codes[0], + msisdn=activated_account.phone_number, user_input='1', state='start', - session_data={}, + data={}, version=1, ) - session.add(ussd_session) - session.commit() + init_database.add(ussd_session) + init_database.commit() ussd_session.set_data(key='foo', session=init_database, value='bar') assert ussd_session.get_data('foo') == 'bar' - ussd_session.update( - session=init_database, - user_input='3', - state='next', - version=2 - ) + ussd_session.update('3', 'next', 2, init_database) assert ussd_session.version == 2 - session.add(ussd_session) - session.commit() + init_database.add(ussd_session) + init_database.commit() - assert UssdSession.have_session_for_phone(create_activated_user.phone_number) is True + assert UssdSession.has_record_for_phone_number(activated_account.phone_number, init_database) is True -def test_version_too_low_error(init_database, create_in_redis_ussd_session, create_activated_user): +def test_version_too_low_error(activated_account, init_database, init_cache): with pytest.raises(VersionTooLowError) as e: session = UssdSession( external_session_id='AT38745', service_code='*123#', - msisdn=create_activated_user.phone_number, + msisdn=activated_account.phone_number, user_input='1', state='start', - session_data={}, + data={}, version=3, ) assert session.check_version(1) assert session.check_version(3) assert str(e.value) == 'New session version number is not greater than last saved version!' + + +def test_set_data(init_database, ussd_session_data, persisted_ussd_session): + assert persisted_ussd_session.data == {} + for key, value in ussd_session_data.items(): + persisted_ussd_session.set_data(key, init_database, value) + init_database.commit() + + assert persisted_ussd_session.get_data('recipient') == ussd_session_data.get('recipient') + + +def test_has_record_for_phone_number(activated_account, init_database, persisted_ussd_session): + ussd_session = UssdSession.has_record_for_phone_number(activated_account.phone_number, init_database) + assert ussd_session is not None + + +def test_last_ussd_session(init_database, ussd_session_traffic): + assert len(init_database.query(UssdSession).all()) >= 5 + ussd_session = init_database.query(UssdSession).order_by(desc(UssdSession.created)).first() + phone_number = ussd_session.msisdn + assert UssdSession.last_ussd_session(phone_number, init_database).id == ussd_session.id + + +def test_persisted_to_json(persisted_ussd_session): + assert isinstance(persisted_ussd_session, UssdSession) + assert isinstance(persisted_ussd_session.to_json(), dict) \ No newline at end of file diff --git a/apps/cic-ussd/tests/cic_ussd/db/test_db.py b/apps/cic-ussd/tests/cic_ussd/db/test_db.py index c51937b4..1c80204d 100644 --- a/apps/cic-ussd/tests/cic_ussd/db/test_db.py +++ b/apps/cic-ussd/tests/cic_ussd/db/test_db.py @@ -10,11 +10,6 @@ from cic_ussd.db import dsn_from_config def test_dsn_from_config(load_config): """ """ - # test dsn for sqlite engine - dsn = dsn_from_config(load_config) - scheme = f'{load_config.get("DATABASE_ENGINE")}+{load_config.get("DATABASE_DRIVER")}' - assert dsn == f'{scheme}:///{load_config.get("DATABASE_NAME")}' - # test dsn for other db formats overrides = { 'DATABASE_PASSWORD': 'password', @@ -28,3 +23,16 @@ def test_dsn_from_config(load_config): dsn = dsn_from_config(load_config) assert dsn == f"{scheme}://{load_config.get('DATABASE_USER')}:{load_config.get('DATABASE_PASSWORD')}@{load_config.get('DATABASE_HOST')}:{load_config.get('DATABASE_PORT')}/{load_config.get('DATABASE_NAME')}" + # undoes overrides to revert engine and drivers to sqlite + overrides = { + 'DATABASE_PASSWORD': '', + 'DATABASE_DRIVER': 'pysqlite', + 'DATABASE_ENGINE': 'sqlite' + } + load_config.dict_override(dct=overrides, dct_description='Override values to test different db formats.') + + # test dsn for sqlite engine + dsn = dsn_from_config(load_config) + scheme = f'{load_config.get("DATABASE_ENGINE")}+{load_config.get("DATABASE_DRIVER")}' + assert dsn == f'{scheme}:///{load_config.get("DATABASE_NAME")}' + diff --git a/apps/cic-ussd/tests/cic_ussd/files/test_local_files.py b/apps/cic-ussd/tests/cic_ussd/files/test_local_files.py index 37d9feb1..ab44988a 100644 --- a/apps/cic-ussd/tests/cic_ussd/files/test_local_files.py +++ b/apps/cic-ussd/tests/cic_ussd/files/test_local_files.py @@ -36,7 +36,7 @@ def test_json_file_parser(load_config): WHEN the json_file_parser function is passed a directory path containing JSON files THEN it dynamically loads all the files and compiles their content into one python array """ - files_dir = load_config.get('STATEMACHINE_TRANSITIONS') + files_dir = load_config.get('MACHINE_TRANSITIONS') files_dir = os.path.join(root_directory, files_dir) # total files len @@ -46,10 +46,7 @@ def test_json_file_parser(load_config): # get path of data files filepath = os.path.join(files_dir, filepath) - # read file content - data_file = open(filepath) - data = json.load(data_file) - file_content_length += len(data) - data_file.close() + with open(filepath) as data_file: + data = json.load(data_file) + file_content_length += len(data) assert len(json_file_parser(filepath=files_dir)) == file_content_length - diff --git a/apps/cic-ussd/tests/cic_ussd/http/test_requests.py b/apps/cic-ussd/tests/cic_ussd/http/test_requests.py new file mode 100644 index 00000000..849639ff --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/http/test_requests.py @@ -0,0 +1,69 @@ +# standard imports +from urllib.parse import urlparse, parse_qs + +# external imports +import pytest +import requests +from requests.exceptions import HTTPError +import requests_mock + +# local imports +from cic_ussd.http.requests import (error_handler, + get_query_parameters, + get_request_endpoint, + get_request_method, + make_request) +from cic_ussd.error import UnsupportedMethodError +# test imports + + +@pytest.mark.parametrize('status_code, starts_with', [ + (102, 'Informational errors'), + (303, 'Redirect Issues'), + (406, 'Client Error'), + (500, 'Server Error') +]) +def test_error_handler(status_code, starts_with, mocker): + mock_result = mocker.patch('requests.Response') + mock_result.status_code = status_code + with pytest.raises(HTTPError) as error: + error_handler(mock_result) + assert str(error.value).startswith(starts_with) + + +def test_get_query_parameters(with_params_env): + assert get_query_parameters(with_params_env, 'phone') == with_params_env.get('REQUEST_URI')[8:] + parsed_url = urlparse(with_params_env.get('REQUEST_URI')) + params = parse_qs(parsed_url.query) + assert get_query_parameters(with_params_env) == params + + +def test_get_request_endpoint(with_params_env): + assert get_request_endpoint(with_params_env) == with_params_env.get('PATH_INFO') + + +def test_get_request_method(with_params_env): + assert get_request_method(with_params_env) == with_params_env.get('REQUEST_METHOD') + + +def test_make_request(mock_response, mock_url): + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('GET', mock_url, status_code=200, reason='OK', json=mock_response) + response = make_request(method='GET', url=mock_url) + assert response.json() == requests.get(mock_url).json() + + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('POST', mock_url, status_code=201, reason='CREATED', json=mock_response) + response = make_request('POST', mock_url, {'test': 'data'}) + assert response.content == requests.post(mock_url).content + + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('PUT', mock_url, status_code=200, reason='OK') + response = make_request('PUT', mock_url, data={'test': 'data'}) + assert response.content == requests.put(mock_url).content + + with pytest.raises(UnsupportedMethodError) as error: + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('DELETE', mock_url, status_code=200, reason='OK') + make_request('DELETE', mock_url) + assert str(error.value) == 'Unsupported method: DELETE' diff --git a/apps/cic-ussd/tests/cic_ussd/http/test_responses.py b/apps/cic-ussd/tests/cic_ussd/http/test_responses.py new file mode 100644 index 00000000..e48ec3ec --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/http/test_responses.py @@ -0,0 +1,18 @@ +# standard imports + +# external imports +import pytest + +# local imports +from cic_ussd.http.responses import with_content_headers + +# test imports + + +@pytest.mark.parametrize('headers, response, expected_result',[ + ([('Content-Type', 'text/plain')], 'some-text', (b'some-text', [('Content-Type', 'text/plain'), ('Content-Length', '9')])), + ([('Content-Type', 'text/plain'), ('Content-Length', '0')], 'some-text', (b'some-text', [('Content-Type', 'text/plain'), ('Content-Length', '9')])) +]) +def test_with_content_headers(headers, response, expected_result): + response_bytes, headers = with_content_headers(headers, response) + assert response_bytes, headers == expected_result diff --git a/apps/cic-ussd/tests/cic_ussd/http/test_routes.py b/apps/cic-ussd/tests/cic_ussd/http/test_routes.py new file mode 100644 index 00000000..b92f3ac7 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/http/test_routes.py @@ -0,0 +1,45 @@ +# standard imports +import json + +# external imports +import pytest + +# local imports +from cic_ussd.db.models.account import Account +from cic_ussd.http.routes import locked_accounts, pin_reset + +# test imports + + +@pytest.mark.parametrize('method, expected_response, expected_message', [ + ('GET', '{"status": "LOCKED"}', '200 OK'), + ('PUT', 'Pin reset successful.', '200 OK'), +]) +def test_pin_reset(method, expected_response, expected_message, init_database, pin_blocked_account, uwsgi_env): + uwsgi_env['REQUEST_METHOD'] = method + response, message = pin_reset(uwsgi_env, pin_blocked_account.phone_number, init_database) + assert response == expected_response + assert message == expected_message + + response, message = pin_reset(uwsgi_env, '070000000', init_database) + assert response == '' + assert message == '404 Not found' + + +def test_locked_accounts(init_database, locked_accounts_env, locked_accounts_traffic): + response, message = locked_accounts(locked_accounts_env, init_database) + assert message == '200 OK' + locked_account_addresses = json.loads(response) + assert len(locked_account_addresses) == 10 + account_1 = init_database.query(Account).filter_by(blockchain_address=locked_account_addresses[2]).first() + account_2 = init_database.query(Account).filter_by(blockchain_address=locked_account_addresses[7]).first() + assert account_1.updated > account_2.updated + locked_accounts_env['PATH_INFO'] = '/accounts/locked/10' + response, message = locked_accounts(locked_accounts_env, init_database) + assert message == '200 OK' + locked_account_addresses = json.loads(response) + assert len(locked_account_addresses) == 10 + locked_accounts_env['REQUEST_METHOD'] = 'POST' + response, message = locked_accounts(locked_accounts_env, init_database) + assert message == '405 Play by the rules' + assert response == '' diff --git a/apps/cic-ussd/tests/cic_ussd/menu/test_ussd_menu.py b/apps/cic-ussd/tests/cic_ussd/menu/test_ussd_menu.py index 28c9b834..693db700 100644 --- a/apps/cic-ussd/tests/cic_ussd/menu/test_ussd_menu.py +++ b/apps/cic-ussd/tests/cic_ussd/menu/test_ussd_menu.py @@ -1,12 +1,14 @@ # standard imports import os -# third party imports +# external imports import pytest # local imports from cic_ussd.files.local_files import create_local_file_data_stores from cic_ussd.menu.ussd_menu import UssdMenu + +# tests imports from tests.helpers.tmp_files import create_tmp_file @@ -49,4 +51,3 @@ def test_failed_create_ussd_menu(): assert str(error.value) == "Menu already exists!" os.close(descriptor) os.remove(tmp_file) - diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py new file mode 100644 index 00000000..f7633e04 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_base.py @@ -0,0 +1,44 @@ +# standard imports +import json +import os + +# external imports +import requests_mock +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata.base import MetadataRequestsHandler + + +# external imports + + +def test_metadata_requests_handler(activated_account, + init_cache, + load_config, + person_metadata, + setup_metadata_request_handler, + setup_metadata_signer): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + cic_type = ':cic.person' + metadata_client = MetadataRequestsHandler(cic_type, identifier) + assert metadata_client.cic_type == cic_type + assert metadata_client.engine == 'pgp' + assert metadata_client.identifier == identifier + assert metadata_client.metadata_pointer == generate_metadata_pointer(identifier, cic_type) + assert metadata_client.url == os.path.join(load_config.get('CIC_META_URL'), metadata_client.metadata_pointer) + + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('POST', metadata_client.url, status_code=200, reason='OK', json=person_metadata) + person_metadata['digest'] = os.urandom(20).hex() + request_mocker.register_uri('PUT', metadata_client.url, status_code=200, reason='OK', json=person_metadata) + result = metadata_client.create(person_metadata) + assert result.json() == person_metadata + assert result.status_code == 200 + person_metadata.pop('digest') + request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata) + result = metadata_client.query() + assert result == person_metadata + cached_metadata = metadata_client.get_cached_metadata() + assert json.loads(cached_metadata) == person_metadata diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py new file mode 100644 index 00000000..846fbca7 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py @@ -0,0 +1,22 @@ +# standard imports +import os +# external imports +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata import CustomMetadata + +# test imports + + +def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): + cic_type = ':cic.custom' + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + custom_metadata_client = CustomMetadata(identifier) + assert custom_metadata_client.cic_type == cic_type + assert custom_metadata_client.engine == 'pgp' + assert custom_metadata_client.identifier == identifier + assert custom_metadata_client.metadata_pointer == generate_metadata_pointer(identifier, cic_type) + assert custom_metadata_client.url == os.path.join( + load_config.get('CIC_META_URL'), custom_metadata_client.metadata_pointer) diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_metadata.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_metadata.py deleted file mode 100644 index faefe9e9..00000000 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_metadata.py +++ /dev/null @@ -1,80 +0,0 @@ -# standard imports -import json - -# third-party imports -import pytest -import requests -import requests_mock - -# local imports -from cic_ussd.error import UnsupportedMethodError -from cic_ussd.metadata import blockchain_address_to_metadata_pointer, make_request - - -def test_make_request(define_metadata_pointer_url, mock_meta_get_response, mock_meta_post_response, person_metadata): - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'GET', - define_metadata_pointer_url, - status_code=200, - reason='OK', - content=json.dumps(mock_meta_get_response).encode('utf-8') - ) - response = make_request(method='GET', url=define_metadata_pointer_url) - assert response.content == requests.get(define_metadata_pointer_url).content - - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'POST', - define_metadata_pointer_url, - status_code=201, - reason='CREATED', - content=json.dumps(mock_meta_post_response).encode('utf-8') - ) - response = make_request( - method='POST', - url=define_metadata_pointer_url, - data=json.dumps(person_metadata).encode('utf-8'), - headers={ - 'X-CIC-AUTOMERGE': 'server', - 'Content-Type': 'application/json' - } - ) - assert response.content == requests.post(define_metadata_pointer_url).content - - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'PUT', - define_metadata_pointer_url, - status_code=200, - reason='OK' - ) - response = make_request( - method='PUT', - url=define_metadata_pointer_url, - data=json.dumps(person_metadata).encode('utf-8'), - headers={ - 'X-CIC-AUTOMERGE': 'server', - 'Content-Type': 'application/json' - } - ) - assert response.content == requests.put(define_metadata_pointer_url).content - - with pytest.raises(UnsupportedMethodError) as error: - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'DELETE', - define_metadata_pointer_url, - status_code=200, - reason='OK' - ) - make_request( - method='DELETE', - url=define_metadata_pointer_url - ) - assert str(error.value) == 'Unsupported method: DELETE' - - -def test_blockchain_address_to_metadata_pointer(create_activated_user): - blockchain_address = create_activated_user.blockchain_address - assert type(blockchain_address_to_metadata_pointer(blockchain_address)) == bytes diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py new file mode 100644 index 00000000..6a21810e --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_person.py @@ -0,0 +1,22 @@ +# standard imports +import os +# external imports +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata import PersonMetadata + +# test imports + + +def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): + cic_type = ':cic.person' + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + person_metadata_client = PersonMetadata(identifier) + assert person_metadata_client.cic_type == cic_type + assert person_metadata_client.engine == 'pgp' + assert person_metadata_client.identifier == identifier + assert person_metadata_client.metadata_pointer == generate_metadata_pointer(identifier, cic_type) + assert person_metadata_client.url == os.path.join( + load_config.get('CIC_META_URL'), person_metadata_client.metadata_pointer) diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py new file mode 100644 index 00000000..3bee949a --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py @@ -0,0 +1,23 @@ +# standard imports +import os +# external imports +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata import PhonePointerMetadata + + +# test imports + + +def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): + cic_type = ':cic.phone' + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + phone_pointer_metadata = PhonePointerMetadata(identifier) + assert phone_pointer_metadata.cic_type == cic_type + assert phone_pointer_metadata.engine == 'pgp' + assert phone_pointer_metadata.identifier == identifier + assert phone_pointer_metadata.metadata_pointer == generate_metadata_pointer(identifier, cic_type) + assert phone_pointer_metadata.url == os.path.join( + load_config.get('CIC_META_URL'), phone_pointer_metadata.metadata_pointer) diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py new file mode 100644 index 00000000..2094f035 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py @@ -0,0 +1,22 @@ +# standard imports +import os +# external imports +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata import PreferencesMetadata + +# test imports + + +def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer): + cic_type = ':cic.preferences' + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + preferences_metadata_client = PreferencesMetadata(identifier) + assert preferences_metadata_client.cic_type == cic_type + assert preferences_metadata_client.engine == 'pgp' + assert preferences_metadata_client.identifier == identifier + assert preferences_metadata_client.metadata_pointer == generate_metadata_pointer(identifier, cic_type) + assert preferences_metadata_client.url == os.path.join( + load_config.get('CIC_META_URL'), preferences_metadata_client.metadata_pointer) diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py index 5f2c2508..88a5c9a9 100644 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py +++ b/apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py @@ -9,26 +9,9 @@ from cic_ussd.metadata.signer import Signer def test_client(load_config, setup_metadata_signer, person_metadata): signer = Signer() - # get gpg used - digest = 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' - person_metadata['digest'] = digest gpg = signer.gpg - - # check that key data was loaded assert signer.key_data is not None - - # check that correct operational key is returned gpg.import_keys(key_data=signer.key_data) gpg_keys = gpg.list_keys() assert signer.get_operational_key() == gpg_keys[0] - - # check that correct signature is returned - key_id = signer.get_operational_key().get('keyid') - signature = gpg.sign(message=digest, passphrase=load_config.get('KEYS_PASSPHRASE'), keyid=key_id) - assert str(signature) == signer.sign_digest(data=person_metadata) - - # remove tmp gpg file shutil.rmtree(Signer.gpg_path) - - - diff --git a/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py b/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py deleted file mode 100644 index 1338f78d..00000000 --- a/apps/cic-ussd/tests/cic_ussd/metadata/test_user_metadata.py +++ /dev/null @@ -1,123 +0,0 @@ -# standard imports -import json - -# third-party imports -import pytest -import requests_mock -from cic_types.models.person import generate_metadata_pointer - -# local imports -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.metadata.signer import Signer -from cic_ussd.metadata.person import PersonMetadata -from cic_ussd.redis import get_cached_data - - -def test_user_metadata(create_activated_user, define_metadata_pointer_url, load_config): - PersonMetadata.base_url = load_config.get('CIC_META_URL') - identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - person_metadata_client = PersonMetadata(identifier=identifier) - - assert person_metadata_client.url == define_metadata_pointer_url - - -def test_create_person_metadata(caplog, - create_activated_user, - define_metadata_pointer_url, - load_config, - mock_meta_post_response, - person_metadata): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - person_metadata_client = PersonMetadata(identifier=identifier) - - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'POST', - define_metadata_pointer_url, - status_code=201, - reason='CREATED', - content=json.dumps(mock_meta_post_response).encode('utf-8') - ) - person_metadata_client.create(data=person_metadata) - assert 'Get signed material response status: 201' in caplog.text - - with pytest.raises(RuntimeError) as error: - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'POST', - define_metadata_pointer_url, - status_code=400, - reason='BAD REQUEST' - ) - person_metadata_client.create(data=person_metadata) - assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}' - - -def test_edit_person_metadata(caplog, - create_activated_user, - define_metadata_pointer_url, - load_config, - person_metadata, - setup_metadata_signer): - Signer.gpg_passphrase = load_config.get('KEYS_PASSPHRASE') - identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - person_metadata_client = PersonMetadata(identifier=identifier) - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'PUT', - define_metadata_pointer_url, - status_code=200, - reason='OK' - ) - person_metadata_client.edit(data=person_metadata) - assert 'Signed content submission status: 200' in caplog.text - - with pytest.raises(RuntimeError) as error: - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'PUT', - define_metadata_pointer_url, - status_code=400, - reason='BAD REQUEST' - ) - person_metadata_client.edit(data=person_metadata) - assert str(error.value) == f'400 Client Error: BAD REQUEST for url: {define_metadata_pointer_url}' - - -def test_get_user_metadata(caplog, - create_activated_user, - define_metadata_pointer_url, - init_redis_cache, - load_config, - person_metadata, - setup_metadata_signer): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - person_metadata_client = PersonMetadata(identifier=identifier) - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'GET', - define_metadata_pointer_url, - status_code=200, - content=json.dumps(person_metadata).encode('utf-8'), - reason='OK' - ) - person_metadata_client.query() - assert 'Get latest data status: 200' in caplog.text - key = generate_metadata_pointer( - identifier=identifier, - cic_type=':cic.person' - ) - cached_user_metadata = get_cached_data(key=key) - assert cached_user_metadata - - with pytest.raises(RuntimeError) as error: - with requests_mock.Mocker(real_http=False) as request_mocker: - request_mocker.register_uri( - 'GET', - define_metadata_pointer_url, - status_code=404, - reason='NOT FOUND' - ) - person_metadata_client.query() - assert 'The data is not available and might need to be added.' in caplog.text - assert str(error.value) == f'400 Client Error: NOT FOUND for url: {define_metadata_pointer_url}' diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py new file mode 100644 index 00000000..3e9a698e --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_menu.py @@ -0,0 +1,183 @@ +# standard imports +import json + +# external imports +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.balance import get_cached_available_balance +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.account.statement import ( + get_cached_statement, + parse_statement_transactions, + statement_transaction_set +) +from cic_ussd.account.tokens import get_default_token_symbol +from cic_ussd.account.transaction import from_wei, to_wei +from cic_ussd.menu.ussd_menu import UssdMenu +from cic_ussd.metadata import PersonMetadata +from cic_ussd.phone_number import Support +from cic_ussd.processor.menu import response +from cic_ussd.processor.util import parse_person_metadata +from cic_ussd.translation import translation_for + + +# test imports + + +def test_menu_processor(activated_account, + balances, + cache_balances, + cache_default_token_data, + cache_preferences, + cache_person_metadata, + cache_statement, + celery_session_worker, + generic_ussd_session, + init_database, + load_chain_spec, + load_support_phone, + load_ussd_menu, + mock_sync_balance_api_query, + mock_transaction_list_query, + valid_recipient): + preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + available_balance = get_cached_available_balance(activated_account.blockchain_address) + token_symbol = get_default_token_symbol() + + tax = '' + bonus = '' + display_key = 'ussd.kenya.account_balances' + ussd_menu = UssdMenu.find_by_name('account_balances') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(display_key, + preferred_language, + available_balance=available_balance, + tax=tax, + bonus=bonus, + token_symbol=token_symbol) + + cached_statement = get_cached_statement(activated_account.blockchain_address) + statement = json.loads(cached_statement) + statement_transactions = parse_statement_transactions(statement) + transaction_sets = [statement_transactions[tx:tx + 3] for tx in range(0, len(statement_transactions), 3)] + first_transaction_set = [] + middle_transaction_set = [] + last_transaction_set = [] + if transaction_sets: + first_transaction_set = statement_transaction_set(preferred_language, transaction_sets[0]) + if len(transaction_sets) >= 2: + middle_transaction_set = statement_transaction_set(preferred_language, transaction_sets[1]) + if len(transaction_sets) >= 3: + last_transaction_set = statement_transaction_set(preferred_language, transaction_sets[2]) + + display_key = 'ussd.kenya.first_transaction_set' + ussd_menu = UssdMenu.find_by_name('first_transaction_set') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + + assert resp == translation_for(display_key, preferred_language, first_transaction_set=first_transaction_set) + + display_key = 'ussd.kenya.middle_transaction_set' + ussd_menu = UssdMenu.find_by_name('middle_transaction_set') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + + assert resp == translation_for(display_key, preferred_language, middle_transaction_set=middle_transaction_set) + + display_key = 'ussd.kenya.last_transaction_set' + ussd_menu = UssdMenu.find_by_name('last_transaction_set') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + + assert resp == translation_for(display_key, preferred_language, last_transaction_set=last_transaction_set) + + display_key = 'ussd.kenya.display_user_metadata' + ussd_menu = UssdMenu.find_by_name('display_user_metadata') + name = ussd_menu.get('name') + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + person_metadata = PersonMetadata(identifier) + cached_person_metadata = person_metadata.get_cached_metadata() + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == parse_person_metadata(cached_person_metadata, display_key, preferred_language) + + display_key = 'ussd.kenya.account_balances_pin_authorization' + ussd_menu = UssdMenu.find_by_name('account_balances_pin_authorization') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(f'{display_key}.first', preferred_language) + + activated_account.failed_pin_attempts = 1 + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + retry_pin_entry = translation_for('ussd.kenya.retry_pin_entry', preferred_language, remaining_attempts=2) + assert resp == translation_for(f'{display_key}.retry', preferred_language, retry_pin_entry=retry_pin_entry) + activated_account.failed_pin_attempts = 0 + + display_key = 'ussd.kenya.start' + ussd_menu = UssdMenu.find_by_name('start') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(display_key, + preferred_language, + account_balance=available_balance, + account_token_name=token_symbol) + + display_key = 'ussd.kenya.transaction_pin_authorization' + ussd_menu = UssdMenu.find_by_name('transaction_pin_authorization') + name = ussd_menu.get('name') + generic_ussd_session['data'] = { + 'recipient_phone_number': valid_recipient.phone_number, + 'transaction_amount': '15' + } + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + user_input = generic_ussd_session.get('data').get('transaction_amount') + transaction_amount = to_wei(value=int(user_input)) + tx_recipient_information = valid_recipient.standard_metadata_id() + tx_sender_information = activated_account.standard_metadata_id() + assert resp == translation_for(f'{display_key}.first', + preferred_language, + recipient_information=tx_recipient_information, + transaction_amount=from_wei(transaction_amount), + token_symbol=token_symbol, + sender_information=tx_sender_information) + + display_key = 'ussd.kenya.exit_insufficient_balance' + ussd_menu = UssdMenu.find_by_name('exit_insufficient_balance') + name = ussd_menu.get('name') + generic_ussd_session['data'] = { + 'recipient_phone_number': valid_recipient.phone_number, + 'transaction_amount': '85' + } + transaction_amount = generic_ussd_session.get('data').get('transaction_amount') + transaction_amount = to_wei(value=int(transaction_amount)) + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(display_key, + preferred_language, + amount=from_wei(transaction_amount), + token_symbol=token_symbol, + recipient_information=tx_recipient_information, + token_balance=available_balance) + + display_key = 'ussd.kenya.exit_invalid_menu_option' + ussd_menu = UssdMenu.find_by_name('exit_invalid_menu_option') + name = ussd_menu.get('name') + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(display_key, preferred_language, support_phone=Support.phone_number) + + display_key = 'ussd.kenya.exit_successful_transaction' + ussd_menu = UssdMenu.find_by_name('exit_successful_transaction') + name = ussd_menu.get('name') + generic_ussd_session['data'] = { + 'recipient_phone_number': valid_recipient.phone_number, + 'transaction_amount': '15' + } + transaction_amount = generic_ussd_session.get('data').get('transaction_amount') + transaction_amount = to_wei(value=int(transaction_amount)) + resp = response(activated_account, display_key, name, init_database, generic_ussd_session) + assert resp == translation_for(display_key, + preferred_language, + transaction_amount=from_wei(transaction_amount), + token_symbol=token_symbol, + recipient_information=tx_recipient_information, + sender_information=tx_sender_information) diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py b/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py new file mode 100644 index 00000000..25ccb8e6 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_ussd.py @@ -0,0 +1,121 @@ +# standard imports +import json +import os +import time + +# external imports +import i18n +import requests_mock +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.cache import cache_data, cache_data_key, get_cached_data +from cic_ussd.db.models.task_tracker import TaskTracker +from cic_ussd.menu.ussd_menu import UssdMenu +from cic_ussd.metadata import PersonMetadata +from cic_ussd.processor.ussd import get_menu, handle_menu, handle_menu_operations +from cic_ussd.translation import translation_for + +# test imports +from tests.helpers.accounts import phone_number + + +def test_handle_menu(activated_account, + init_cache, + init_database, + load_ussd_menu, + pending_account, + persisted_ussd_session, + pin_blocked_account, + preferences): + persisted_ussd_session.state = 'account_creation_prompt' + menu_resp = handle_menu(activated_account, init_database) + ussd_menu = UssdMenu.find_by_name('start') + assert menu_resp.get('name') == ussd_menu.get('name') + persisted_ussd_session.state = 'display_user_metadata' + menu_resp = handle_menu(activated_account, init_database) + ussd_menu = UssdMenu.find_by_name('display_user_metadata') + assert menu_resp.get('name') == ussd_menu.get('name') + menu_resp = handle_menu(pin_blocked_account, init_database) + ussd_menu = UssdMenu.find_by_name('exit_pin_blocked') + assert menu_resp.get('name') == ussd_menu.get('name') + menu_resp = handle_menu(pending_account, init_database) + ussd_menu = UssdMenu.find_by_name('initial_language_selection') + assert menu_resp.get('name') == ussd_menu.get('name') + identifier = bytes.fromhex(strip_0x(pending_account.blockchain_address)) + key = cache_data_key(identifier, ':cic.preferences') + cache_data(key, json.dumps(preferences)) + time.sleep(2) + menu_resp = handle_menu(pending_account, init_database) + ussd_menu = UssdMenu.find_by_name('initial_pin_entry') + assert menu_resp.get('name') == ussd_menu.get('name') + + +def test_get_menu(activated_account, + cache_preferences, + cache_person_metadata, + generic_ussd_session, + init_database, + init_state_machine, + persisted_ussd_session): + menu_resp = get_menu(activated_account, init_database, '', generic_ussd_session) + ussd_menu = UssdMenu.find_by_name(name='exit_invalid_input') + assert menu_resp.get('name') == ussd_menu.get('name') + + menu_resp = get_menu(activated_account, init_database, '1111', None) + ussd_menu = UssdMenu.find_by_name('initial_language_selection') + assert menu_resp.get('name') == ussd_menu.get('name') + + generic_ussd_session['state'] = 'start' + menu_resp = get_menu(activated_account, init_database, '1', generic_ussd_session) + ussd_menu = UssdMenu.find_by_name(name='enter_transaction_recipient') + assert menu_resp.get('name') == ussd_menu.get('name') + + +def test_handle_menu_operations(activated_account, + cache_preferences, + celery_session_worker, + generic_ussd_session, + init_database, + init_cache, + load_chain_spec, + load_config, + mock_account_creation_task_result, + persisted_ussd_session, + person_metadata, + set_locale_files, + setup_metadata_request_handler, + setup_metadata_signer, + task_uuid): + # sourcery skip: extract-duplicate-method + chain_str = Chain.spec.__str__() + phone = phone_number() + external_session_id = os.urandom(20).hex() + valid_service_codes = load_config.get('USSD_SERVICE_CODE').split(",") + preferred_language = i18n.config.get('fallback') + resp = handle_menu_operations(chain_str, external_session_id, phone, None, valid_service_codes[0], init_database, '4444') + assert resp == translation_for('ussd.kenya.account_creation_prompt', preferred_language) + cached_ussd_session = get_cached_data(external_session_id) + ussd_session = json.loads(cached_ussd_session) + assert ussd_session['msisdn'] == phone + task_tracker = init_database.query(TaskTracker).filter_by(task_uuid=task_uuid).first() + assert task_tracker.task_uuid == task_uuid + cached_creation_task_uuid = get_cached_data(task_uuid) + creation_task_uuid_data = json.loads(cached_creation_task_uuid) + assert creation_task_uuid_data['status'] == 'PENDING' + + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + person_metadata_client = PersonMetadata(identifier) + with requests_mock.Mocker(real_http=False) as request_mocker: + request_mocker.register_uri('GET', person_metadata_client.url, status_code=200, reason='OK', + json=person_metadata) + person_metadata_client.query() + external_session_id = os.urandom(20).hex() + phone = activated_account.phone_number + preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + persisted_ussd_session.state = 'enter_transaction_recipient' + resp = handle_menu_operations(chain_str, external_session_id, phone, None, valid_service_codes[0], init_database, '1') + assert resp == translation_for('ussd.kenya.enter_transaction_recipient', preferred_language) + diff --git a/apps/cic-ussd/tests/cic_ussd/processor/test_util.py b/apps/cic-ussd/tests/cic_ussd/processor/test_util.py new file mode 100644 index 00000000..038ad8ca --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/processor/test_util.py @@ -0,0 +1,62 @@ +# standard imports +import datetime +import json + +# external imports +import pytest +from chainlib.hash import strip_0x +from cic_types.models.person import get_contact_data_from_vcard + +# local imports +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.metadata import PersonMetadata +from cic_ussd.processor.util import latest_input, parse_person_metadata, resume_last_ussd_session +from cic_ussd.translation import translation_for + + +# test imports + + +@pytest.mark.parametrize('user_input, expected_value', [ + ('1*9*6*7', '7'), + ('1', '1'), + ('', '') +]) +def test_latest_input(user_input, expected_value): + assert latest_input(user_input) == expected_value + + +def test_parse_person_metadata(activated_account, cache_person_metadata, cache_preferences): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + person_metadata = PersonMetadata(identifier) + cached_person_metadata = person_metadata.get_cached_metadata() + person_metadata = json.loads(cached_person_metadata) + preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + display_key = 'ussd.kenya.display_person_metadata' + parsed_person_metadata = parse_person_metadata(cached_person_metadata, + display_key, + preferred_language) + contact_data = get_contact_data_from_vcard(person_metadata.get('vcard')) + full_name = f'{contact_data.get("given")} {contact_data.get("family")}' + date_of_birth = person_metadata.get('date_of_birth') + year_of_birth = date_of_birth.get('year') + present_year = datetime.datetime.now().year + age = present_year - year_of_birth + gender = person_metadata.get('gender') + products = ', '.join(person_metadata.get('products')) + location = person_metadata.get('location').get('area_name') + assert parsed_person_metadata == translation_for(key=display_key, + preferred_language=preferred_language, + full_name=full_name, + age=age, + gender=gender, + location=location, + products=products) + + +@pytest.mark.parametrize('last_state, expected_menu_name', [ + ('account_creation_prompt', 'start'), + ('enter_transaction_recipient', 'enter_transaction_recipient') +]) +def test_resume_last_ussd_session(expected_menu_name, last_state, load_ussd_menu): + assert resume_last_ussd_session(last_state).get('name') == expected_menu_name diff --git a/apps/cic-ussd/tests/cic_ussd/session/test_cached_ussd_session.py b/apps/cic-ussd/tests/cic_ussd/session/test_cached_ussd_session.py new file mode 100644 index 00000000..e6d8a77f --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/session/test_cached_ussd_session.py @@ -0,0 +1,72 @@ +# standard imports +import json + +# external imports + +# local imports +from cic_ussd.cache import get_cached_data +from cic_ussd.db.models.ussd_session import UssdSession as PersistedUssdSession +from cic_ussd.menu.ussd_menu import UssdMenu +from cic_ussd.session.ussd_session import (create_or_update_session, + create_ussd_session, + persist_ussd_session, + save_session_data, + update_ussd_session, + UssdSession) + + +# test imports + + +def test_ussd_session(cached_ussd_session, load_ussd_menu): + assert UssdMenu.find_by_name(name='initial_language_selection').get('name') == cached_ussd_session.state + cached_ussd_session.set_data('some_key', 'some_value') + assert cached_ussd_session.get_data('some_key') == 'some_value' + assert isinstance(cached_ussd_session, UssdSession) + assert isinstance(cached_ussd_session.to_json(), dict) + + +def test_create_or_update_session(activated_account_ussd_session, cached_ussd_session, init_cache, init_database): + external_session_id = activated_account_ussd_session.get('external_session_id') + ussd_session = create_or_update_session(external_session_id=external_session_id, + service_code=activated_account_ussd_session.get('service_code'), + msisdn=activated_account_ussd_session.get('msisdn'), + user_input=activated_account_ussd_session.get('user_input'), + state=activated_account_ussd_session.get('state'), + session=init_database) + cached_ussd_session = get_cached_data(external_session_id) + assert json.loads(cached_ussd_session).get('external_session_id') == ussd_session.external_session_id + + +def test_update_ussd_session(activated_account_ussd_session, cached_ussd_session, init_cache, load_ussd_menu): + ussd_session = create_ussd_session(external_session_id=activated_account_ussd_session.get('external_session_id'), + service_code=activated_account_ussd_session.get('service_code'), + msisdn=activated_account_ussd_session.get('msisdn'), + user_input=activated_account_ussd_session.get('user_input'), + state=activated_account_ussd_session.get('state')) + assert ussd_session.user_input == activated_account_ussd_session.get('user_input') + assert ussd_session.state == activated_account_ussd_session.get('state') + ussd_session = update_ussd_session(ussd_session=ussd_session, user_input='1*2', state='initial_pin_entry') + assert ussd_session.user_input == '1*2' + assert ussd_session.state == 'initial_pin_entry' + + +def test_save_session_data(activated_account_ussd_session, + cached_ussd_session, + celery_session_worker, + init_cache, + init_database, + ussd_session_data): + external_session_id = activated_account_ussd_session.get('external_session_id') + ussd_session = get_cached_data(external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('data') == {} + save_session_data( + queue='cic-ussd', + data=ussd_session_data, + ussd_session=cached_ussd_session.to_json(), + session=init_database + ) + ussd_session = get_cached_data(external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('data') == ussd_session_data diff --git a/apps/cic-ussd/tests/cic_ussd/session/test_in_memory_ussd_session.py b/apps/cic-ussd/tests/cic_ussd/session/test_in_memory_ussd_session.py deleted file mode 100644 index a3c49c26..00000000 --- a/apps/cic-ussd/tests/cic_ussd/session/test_in_memory_ussd_session.py +++ /dev/null @@ -1,11 +0,0 @@ -# standard imports -import json - -# local imports -from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.session.ussd_session import UssdSession - - -def test_ussd_session(load_ussd_menu, get_in_redis_ussd_session): - ussd_session = get_in_redis_ussd_session - assert UssdMenu.find_by_name(name='initial_language_selection').get('name') == ussd_session.state diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_account_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_account_logic.py new file mode 100644 index 00000000..9e8da5a6 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_account_logic.py @@ -0,0 +1,154 @@ +# standard imports +import json + +# external imports +import pytest +import requests_mock +from chainlib.hash import strip_0x +from cic_types.models.person import Person, get_contact_data_from_vcard + +# local imports +from cic_ussd.cache import get_cached_data +from cic_ussd.account.maps import gender +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.db.enum import AccountStatus +from cic_ussd.metadata import PreferencesMetadata +from cic_ussd.state_machine.logic.account import (change_preferred_language, + edit_user_metadata_attribute, + parse_gender, + parse_person_metadata, + save_complete_person_metadata, + save_metadata_attribute_to_session_data, + update_account_status_to_active) +from cic_ussd.translation import translation_for + + +# test imports + + +@pytest.mark.parametrize('user_input, expected_preferred_language', [ + ('1', 'en'), + ('2', 'sw') +]) +def test_change_preferred_language(activated_account, + celery_session_worker, + expected_preferred_language, + init_database, + generic_ussd_session, + mock_response, + preferences, + setup_metadata_request_handler, + user_input): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + preferences_metadata_client = PreferencesMetadata(identifier) + with requests_mock.Mocker(real_http=False) as requests_mocker: + requests_mocker.register_uri( + 'POST', preferences_metadata_client.url, status_code=200, reason='OK', json=mock_response + ) + state_machine_data = (user_input, generic_ussd_session, activated_account, init_database) + res = change_preferred_language(state_machine_data) + init_database.commit() + assert res.id is not None + assert activated_account.preferred_language == expected_preferred_language + + +@pytest.mark.parametrize('user_input', [ + '1', + '2', + '3' +]) +def test_parse_gender(activated_account, cache_preferences, user_input): + preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + parsed_gender = parse_gender(activated_account, user_input) + r_user_input = gender().get(user_input) + assert parsed_gender == translation_for(f'helpers.{r_user_input}', preferred_language) + + +def test_parse_person_metadata(activated_account, load_chain_spec, raw_person_metadata): + parsed_user_metadata = parse_person_metadata(activated_account, raw_person_metadata) + person = Person() + user_metadata = person.deserialize(parsed_user_metadata) + assert parsed_user_metadata == user_metadata.serialize() + + +@pytest.mark.parametrize("current_state, expected_key, expected_result, user_input", [ + ("enter_given_name", "given_name", "John", "John"), + ("enter_family_name", "family_name", "Doe", "Doe"), + ("enter_location", "location", "Kangemi", "Kangemi"), + ("enter_products", "products", "Mandazi", "Mandazi"), +]) +def test_save_metadata_attribute_to_session_data(activated_account, + cached_ussd_session, + celery_session_worker, + current_state, + expected_key, + expected_result, + init_cache, + init_database, + load_chain_spec, + set_locale_files, + persisted_ussd_session, + user_input): + persisted_ussd_session.state = current_state + ussd_session = persisted_ussd_session.to_json() + state_machine_data = (user_input, ussd_session, activated_account, init_database) + ussd_session_in_cache = get_cached_data(cached_ussd_session.external_session_id) + ussd_session_in_cache = json.loads(ussd_session_in_cache) + assert ussd_session_in_cache.get('data') == {} + ussd_session['state'] = current_state + save_metadata_attribute_to_session_data(state_machine_data) + cached_ussd_session = get_cached_data(cached_ussd_session.external_session_id) + cached_ussd_session = json.loads(cached_ussd_session) + assert cached_ussd_session.get('data')[expected_key] == expected_result + + +def test_update_account_status_to_active(generic_ussd_session, init_database, pending_account): + state_machine_data = ('', generic_ussd_session, pending_account, init_database) + assert pending_account.get_status(init_database) == AccountStatus.PENDING.name + update_account_status_to_active(state_machine_data) + assert pending_account.get_status(init_database) == AccountStatus.ACTIVE.name + + +def test_save_complete_person_metadata(activated_account, + cached_ussd_session, + celery_session_worker, + init_database, + load_chain_spec, + mocker, + person_metadata, + raw_person_metadata): + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + ussd_session['data'] = raw_person_metadata + metadata = parse_person_metadata(activated_account, raw_person_metadata) + state_machine_data = ('', ussd_session, activated_account, init_database) + mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_person_metadata.apply_async') + save_complete_person_metadata(state_machine_data=state_machine_data) + mocked_create_metadata_task.assert_called_with( + (activated_account.blockchain_address, metadata), {}, queue='cic-ussd') + + +def test_edit_user_metadata_attribute(activated_account, + cache_person_metadata, + cached_ussd_session, + celery_session_worker, + init_cache, + init_database, + load_chain_spec, + mocker, + person_metadata): + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert person_metadata['location']['area_name'] == 'kayaba' + ussd_session['data'] = {'location': 'nairobi'} + contact_data = get_contact_data_from_vcard(person_metadata.get('vcard')) + phone_number = contact_data.get('tel') + activated_account.phone_number = phone_number + state_machine_data = ('', ussd_session, activated_account, init_database) + mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.create_person_metadata.apply_async') + edit_user_metadata_attribute(state_machine_data) + person_metadata['date_registered'] = int(activated_account.created.replace().timestamp()) + person_metadata['location']['area_name'] = 'nairobi' + mocked_edit_metadata.assert_called_with( + (activated_account.blockchain_address, person_metadata), {}, queue='cic-ussd') + diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu.py deleted file mode 100644 index 9e8f8c49..00000000 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu.py +++ /dev/null @@ -1,21 +0,0 @@ -# local imports -from cic_ussd.state_machine.logic.menu import (menu_one_selected, - menu_two_selected, - menu_three_selected, - menu_four_selected) - - -def test_menu_selection(create_pending_user, create_in_db_ussd_session): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - assert menu_one_selected(('1', serialized_in_db_ussd_session, create_pending_user)) is True - assert menu_one_selected(('x', serialized_in_db_ussd_session, create_pending_user)) is False - - assert menu_two_selected(('2', serialized_in_db_ussd_session, create_pending_user)) is True - assert menu_two_selected(('1', serialized_in_db_ussd_session, create_pending_user)) is False - - assert menu_three_selected(('3', serialized_in_db_ussd_session, create_pending_user)) is True - assert menu_three_selected(('4', serialized_in_db_ussd_session, create_pending_user)) is False - - assert menu_four_selected(('4', serialized_in_db_ussd_session, create_pending_user)) is True - assert menu_four_selected(('d', serialized_in_db_ussd_session, create_pending_user)) is False - diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu_logic.py new file mode 100644 index 00000000..1cde862c --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_menu_logic.py @@ -0,0 +1,37 @@ +# standard imports + +# external imports + +# local imports +from cic_ussd.state_machine.logic.menu import (menu_one_selected, + menu_two_selected, + menu_three_selected, + menu_four_selected, + menu_five_selected, + menu_six_selected, + menu_zero_zero_selected, + menu_ninety_nine_selected) + +# test imports + + +def test_menu_selection(init_database, pending_account, persisted_ussd_session): + ussd_session = persisted_ussd_session.to_json() + assert menu_one_selected(('1', ussd_session, pending_account, init_database)) is True + assert menu_one_selected(('x', ussd_session, pending_account, init_database)) is False + assert menu_two_selected(('2', ussd_session, pending_account, init_database)) is True + assert menu_two_selected(('1', ussd_session, pending_account, init_database)) is False + assert menu_three_selected(('3', ussd_session, pending_account, init_database)) is True + assert menu_three_selected(('4', ussd_session, pending_account, init_database)) is False + assert menu_four_selected(('4', ussd_session, pending_account, init_database)) is True + assert menu_four_selected(('d', ussd_session, pending_account, init_database)) is False + assert menu_five_selected(('5', ussd_session, pending_account, init_database)) is True + assert menu_five_selected(('e', ussd_session, pending_account, init_database)) is False + assert menu_six_selected(('6', ussd_session, pending_account, init_database)) is True + assert menu_six_selected(('8', ussd_session, pending_account, init_database)) is False + assert menu_zero_zero_selected(('00', ussd_session, pending_account, init_database)) is True + assert menu_zero_zero_selected(('/', ussd_session, pending_account, init_database)) is False + assert menu_ninety_nine_selected(('99', ussd_session, pending_account, init_database)) is True + assert menu_ninety_nine_selected(('d', ussd_session, pending_account, init_database)) is False + + diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin.py deleted file mode 100644 index 1c1419fe..00000000 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin.py +++ /dev/null @@ -1,94 +0,0 @@ -# standards imports -import json - -# third party imports -import pytest - -# local imports -from cic_ussd.encoder import check_password_hash, create_password_hash -from cic_ussd.state_machine.logic.pin import (complete_pin_change, - is_valid_pin, - is_valid_new_pin, - is_authorized_pin, - is_blocked_pin, - pins_match, - save_initial_pin_to_session_data) - - -def test_complete_pin_change(init_database, create_pending_user, create_in_db_ussd_session): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('1212', serialized_in_db_ussd_session, create_pending_user) - assert create_pending_user.password_hash is None - create_in_db_ussd_session.set_data(key='initial_pin', session=init_database, value=create_password_hash('1212')) - complete_pin_change(state_machine_data) - assert create_pending_user.password_hash is not None - assert create_pending_user.verify_password(password='1212') is True - - -@pytest.mark.parametrize('user_input, expected', [ - ('4562', True), - ('jksu', False), - ('ij45', False), -]) -def test_is_valid_pin(create_pending_user, create_in_db_ussd_session, user_input, expected): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_pending_user) - assert is_valid_pin(state_machine_data) is expected - - -@pytest.mark.parametrize('user_input, expected', [ - ('1212', True), - ('0000', False) -]) -def test_pins_match(init_database, create_pending_user, create_in_db_ussd_session, user_input, expected): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_pending_user) - create_in_db_ussd_session.set_data(key='initial_pin', session=init_database, value=create_password_hash(user_input)) - assert pins_match(state_machine_data) is True - - -def test_save_initial_pin_to_session_data(create_pending_user, - create_in_redis_ussd_session, - create_in_db_ussd_session, - celery_session_worker): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('1212', serialized_in_db_ussd_session, create_pending_user) - save_initial_pin_to_session_data(state_machine_data) - external_session_id = create_in_db_ussd_session.external_session_id - in_memory_ussd_session = create_in_redis_ussd_session.get(external_session_id) - in_memory_ussd_session = json.loads(in_memory_ussd_session) - assert check_password_hash( - password='1212', hashed_password=in_memory_ussd_session.get('session_data')['initial_pin']) - - -@pytest.mark.parametrize('user_input, expected_result', [ - ('1212', False), - ('0000', True) -]) -def test_is_authorized_pin(create_activated_user, create_in_db_ussd_session, expected_result, user_input): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_activated_user) - assert is_authorized_pin(state_machine_data=state_machine_data) is expected_result - - -def test_is_not_blocked_pin(create_activated_user, create_in_db_ussd_session): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('', serialized_in_db_ussd_session, create_activated_user) - assert is_blocked_pin(state_machine_data=state_machine_data) is False - - -def test_is_blocked_pin(create_pin_blocked_user, create_in_db_ussd_session): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - alt_state_machine_data = ('user_input', serialized_in_db_ussd_session, create_pin_blocked_user) - assert is_blocked_pin(state_machine_data=alt_state_machine_data) is True - - -@pytest.mark.parametrize('user_input, expected_result', [ - ('1212', True), - ('0000', False) -]) -def test_is_valid_new_pin(create_activated_user, create_in_db_ussd_session, expected_result, user_input): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_activated_user) - assert is_valid_new_pin(state_machine_data=state_machine_data) is expected_result - diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin_logic.py new file mode 100644 index 00000000..6921a034 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_pin_logic.py @@ -0,0 +1,101 @@ +# standard imports +import json + +# external imports +import pytest + +# local imports +from cic_ussd.cache import get_cached_data +from cic_ussd.encoder import check_password_hash, create_password_hash +from cic_ussd.state_machine.logic.pin import (complete_pin_change, + is_valid_pin, + is_valid_new_pin, + is_authorized_pin, + is_blocked_pin, + is_locked_account, + pins_match, + save_initial_pin_to_session_data) + + +def test_complete_pin_change(activated_account, cached_ussd_session, init_database): + state_machine_data = ('1212', cached_ussd_session.to_json(), activated_account, init_database) + assert activated_account.password_hash is not None + cached_ussd_session.set_data('initial_pin', create_password_hash('1212')) + complete_pin_change(state_machine_data) + assert activated_account.verify_password('1212') is True + + +@pytest.mark.parametrize('user_input, expected', [ + ('4562', True), + ('jksu', False), + ('ij45', False), +]) +def test_is_valid_pin(activated_account, expected, generic_ussd_session, init_database, user_input): + state_machine_data = (user_input, generic_ussd_session, activated_account, init_database) + assert is_valid_pin(state_machine_data) is expected + + +@pytest.mark.parametrize('user_input', [ + '1212', + '0000' +]) +def test_pins_match(activated_account, cached_ussd_session, init_cache, init_database, user_input): + state_machine_data = (user_input, cached_ussd_session.to_json(), activated_account, init_database) + cached_ussd_session.set_data('initial_pin', create_password_hash(user_input)) + assert pins_match(state_machine_data) is True + + +def test_save_initial_pin_to_session_data(activated_account, + cached_ussd_session, + celery_session_worker, + init_cache, + init_database, + persisted_ussd_session, + set_fernet_key): + state_machine_data = ('1212', cached_ussd_session.to_json(), activated_account, init_database) + save_initial_pin_to_session_data(state_machine_data) + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert check_password_hash('1212', ussd_session.get('data')['initial_pin']) + cached_ussd_session.set_data('some_key', 'some_value') + state_machine_data = ('1212', cached_ussd_session.to_json(), activated_account, init_database) + save_initial_pin_to_session_data(state_machine_data) + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('data')['some_key'] == 'some_value' + + +@pytest.mark.parametrize('user_input, expected_result', [ + ('1212', False), + ('0000', True) +]) +def test_is_authorized_pin(activated_account, cached_ussd_session, expected_result, init_database, user_input): + state_machine_data = (user_input, cached_ussd_session.to_json(), activated_account, init_database) + assert is_authorized_pin(state_machine_data) is expected_result + + +def test_is_not_blocked_pin(activated_account, cached_ussd_session, init_database): + state_machine_data = ('', cached_ussd_session.to_json(), activated_account, init_database) + assert is_blocked_pin(state_machine_data) is False + + +def test_is_blocked_pin(cached_ussd_session, init_database, pin_blocked_account): + state_machine_data = ('user_input', cached_ussd_session, pin_blocked_account, init_database) + assert is_blocked_pin(state_machine_data) is True + + +def test_is_locked_account(activated_account, generic_ussd_session, init_database, pin_blocked_account): + state_machine_data = ('', generic_ussd_session, activated_account, init_database) + assert is_locked_account(state_machine_data) is False + state_machine_data = ('', generic_ussd_session, pin_blocked_account, init_database) + assert is_locked_account(state_machine_data) is True + + +@pytest.mark.parametrize('user_input, expected_result', [ + ('1212', True), + ('0000', False) +]) +def test_is_valid_new_pin(activated_account, cached_ussd_session, expected_result, init_database, user_input): + state_machine_data = (user_input, cached_ussd_session.to_json(), activated_account, init_database) + assert is_valid_new_pin(state_machine_data) is expected_result + diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms.py deleted file mode 100644 index 28923b59..00000000 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms.py +++ /dev/null @@ -1,36 +0,0 @@ -# standard imports - -# third-party imports -import pytest - -# local imports -from cic_ussd.state_machine.logic.sms import (send_terms_to_user_if_required, - process_mini_statement_request, - upsell_unregistered_recipient) - - -def test_send_terms_to_user_if_required(caplog, - create_in_db_ussd_session, - create_activated_user): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('', serialized_in_db_ussd_session, create_activated_user) - send_terms_to_user_if_required(state_machine_data=state_machine_data) - assert 'Requires integration to cic-notify.' in caplog.text - - -def test_process_mini_statement_request(caplog, - create_in_db_ussd_session, - create_activated_user): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('', serialized_in_db_ussd_session, create_activated_user) - process_mini_statement_request(state_machine_data=state_machine_data) - assert 'Requires integration to cic-notify.' in caplog.text - - -def test_upsell_unregistered_recipient(caplog, - create_in_db_ussd_session, - create_activated_user): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('', serialized_in_db_ussd_session, create_activated_user) - upsell_unregistered_recipient(state_machine_data=state_machine_data) - assert 'Requires integration to cic-notify.' in caplog.text diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms_logic.py new file mode 100644 index 00000000..9d35ce29 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_sms_logic.py @@ -0,0 +1,42 @@ +# standard imports +import json + +# external imports + +# local imports +from cic_ussd.account.metadata import get_cached_preferred_language +from cic_ussd.account.tokens import get_default_token_symbol +from cic_ussd.cache import get_cached_data +from cic_ussd.phone_number import Support +from cic_ussd.state_machine.logic.sms import upsell_unregistered_recipient +from cic_ussd.translation import translation_for + + +# tests imports + + +def test_upsell_unregistered_recipient(activated_account, + cache_default_token_data, + cache_preferences, + cached_ussd_session, + init_database, + load_support_phone, + mock_notifier_api, + set_locale_files, + valid_recipient): + cached_ussd_session.set_data('recipient_phone_number', valid_recipient.phone_number) + state_machine_data = ('', cached_ussd_session.to_json(), activated_account, init_database) + upsell_unregistered_recipient(state_machine_data) + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + phone_number = ussd_session.get('data')['recipient_phone_number'] + preferred_language = get_cached_preferred_language(activated_account.blockchain_address) + token_symbol = get_default_token_symbol() + tx_sender_information = activated_account.standard_metadata_id() + message = translation_for('sms.upsell_unregistered_recipient', + preferred_language, + tx_sender_information=tx_sender_information, + token_symbol=token_symbol, + support_phone=Support.phone_number) + assert mock_notifier_api.get('message') == message + assert mock_notifier_api.get('recipient') == phone_number diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction.py deleted file mode 100644 index e7be07c1..00000000 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction.py +++ /dev/null @@ -1,111 +0,0 @@ -# standard imports -import json - -# third-party imports -import pytest - -# local imports -from cic_ussd.state_machine.logic.transaction import (has_sufficient_balance, - is_valid_recipient, - is_valid_transaction_amount, - process_transaction_request, - save_recipient_phone_to_session_data, - save_transaction_amount_to_session_data) -from cic_ussd.redis import InMemoryStore - - -@pytest.mark.parametrize("amount, expected_result", [ - ('50', True), - ('', False) -]) -def test_is_valid_transaction_amount(create_activated_user, create_in_db_ussd_session, amount, expected_result): - state_machine_data = (amount, create_in_db_ussd_session, create_activated_user) - validity = is_valid_transaction_amount(state_machine_data=state_machine_data) - assert validity == expected_result - - -def test_save_recipient_phone_to_session_data(create_activated_user, - create_in_db_ussd_session, - celery_session_worker, - create_in_redis_ussd_session, - init_database): - phone_number = '+254712345678' - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data') == {} - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (phone_number, serialized_in_db_ussd_session, create_activated_user) - save_recipient_phone_to_session_data(state_machine_data=state_machine_data) - - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data')['recipient_phone_number'] == phone_number - - -def test_save_transaction_amount_to_session_data(create_activated_user, - create_in_db_ussd_session, - celery_session_worker, - create_in_redis_ussd_session, - init_database): - transaction_amount = '100' - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data') == {} - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (transaction_amount, serialized_in_db_ussd_session, create_activated_user) - save_transaction_amount_to_session_data(state_machine_data=state_machine_data) - - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data')['transaction_amount'] == transaction_amount - - -@pytest.mark.parametrize("test_value, expected_result", [ - ('45', True), - ('75', False) -]) -def test_has_sufficient_balance(mock_balance, - create_in_db_ussd_session, - create_valid_tx_sender, - expected_result, - test_value): - mock_balance(60) - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (test_value, serialized_in_db_ussd_session, create_valid_tx_sender) - result = has_sufficient_balance(state_machine_data=state_machine_data) - assert result == expected_result - - -@pytest.mark.parametrize("test_value, expected_result", [ - ('+25498765432', True), - ('+25498765433', False) -]) -def test_is_valid_recipient(create_in_db_ussd_session, - create_valid_tx_recipient, - create_valid_tx_sender, - expected_result, - test_value): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (test_value, serialized_in_db_ussd_session, create_valid_tx_sender) - result = is_valid_recipient(state_machine_data=state_machine_data) - assert result == expected_result - - -def test_process_transaction_request(create_valid_tx_recipient, - create_valid_tx_sender, - load_config, - mock_outgoing_transactions, - setup_chain_spec, - ussd_session_data): - ussd_session_data['session_data'] = { - 'recipient_phone_number': create_valid_tx_recipient.phone_number, - 'transaction_amount': '50' - } - state_machine_data = ('', ussd_session_data, create_valid_tx_sender) - process_transaction_request(state_machine_data=state_machine_data) - assert mock_outgoing_transactions[0].get('amount') == 50.0 - assert mock_outgoing_transactions[0].get('token_symbol') == 'SRF' diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py new file mode 100644 index 00000000..f2acf054 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_transaction_logic.py @@ -0,0 +1,112 @@ +# standard imports +import json + +# external imports +import pytest +import requests_mock +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.transaction import to_wei +from cic_ussd.cache import get_cached_data +from cic_ussd.metadata import PersonMetadata +from cic_ussd.state_machine.logic.transaction import (is_valid_recipient, + is_valid_transaction_amount, + has_sufficient_balance, + process_transaction_request, + retrieve_recipient_metadata, + save_recipient_phone_to_session_data, + save_transaction_amount_to_session_data) + + +# test imports + + +def test_is_valid_recipient(activated_account, + generic_ussd_session, + init_database, + load_e164_region, + pending_account, + valid_recipient): + state_machine = ('0112365478', generic_ussd_session, valid_recipient, init_database) + assert is_valid_recipient(state_machine) is False + state_machine = (pending_account.phone_number, generic_ussd_session, valid_recipient, init_database) + assert is_valid_recipient(state_machine) is False + state_machine = (valid_recipient.phone_number, generic_ussd_session, activated_account, init_database) + assert is_valid_recipient(state_machine) is True + + +@pytest.mark.parametrize("amount, expected_result", [ + ('50', True), + ('', False) +]) +def test_is_valid_transaction_amount(activated_account, amount, expected_result, generic_ussd_session, init_database): + state_machine_data = (amount, generic_ussd_session, activated_account, init_database) + assert is_valid_transaction_amount(state_machine_data) is expected_result + + +@pytest.mark.parametrize("value, expected_result", [ + ('45', True), + ('75', False) +]) +def test_has_sufficient_balance(activated_account, + cache_balances, + expected_result, + generic_ussd_session, + init_database, + value): + state_machine_data = (value, generic_ussd_session, activated_account, init_database) + assert has_sufficient_balance(state_machine_data=state_machine_data) == expected_result + + +def test_process_transaction_request(activated_account, + cache_default_token_data, + cached_ussd_session, + celery_session_worker, + init_cache, + init_database, + load_chain_spec, + load_config, + mock_transfer_api, + valid_recipient): + cached_ussd_session.set_data('recipient_phone_number', valid_recipient.phone_number) + cached_ussd_session.set_data('transaction_amount', '50') + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + state_machine_data = ('', ussd_session, activated_account, init_database) + process_transaction_request(state_machine_data) + assert mock_transfer_api['from_address'] == activated_account.blockchain_address + assert mock_transfer_api['to_address'] == valid_recipient.blockchain_address + assert mock_transfer_api['value'] == to_wei(50) + assert mock_transfer_api['token_symbol'] == load_config.get('TEST_TOKEN_SYMBOL') + + +def test_retrieve_recipient_metadata(activated_account, + generic_ussd_session, + init_database, + load_chain_spec, + mocker, + valid_recipient): + state_machine_data = (valid_recipient.phone_number, generic_ussd_session, activated_account, init_database) + mocked_query_metadata = mocker.patch('cic_ussd.tasks.metadata.query_person_metadata.apply_async') + retrieve_recipient_metadata(state_machine_data) + mocked_query_metadata.assert_called_with((valid_recipient.blockchain_address, ), {}, queue='cic-ussd') + + +def test_transaction_information_to_session_data(activated_account, + cached_ussd_session, + init_cache, + init_database, + load_e164_region, + valid_recipient): + assert cached_ussd_session.to_json()['data'] == {} + state_machine_data = (valid_recipient.phone_number, cached_ussd_session.to_json(), activated_account, init_database) + save_recipient_phone_to_session_data(state_machine_data) + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('data')['recipient_phone_number'] == valid_recipient.phone_number + state_machine_data = ('25', cached_ussd_session.to_json(), activated_account, init_database) + save_transaction_amount_to_session_data(state_machine_data) + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('data')['transaction_amount'] == '25' diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py deleted file mode 100644 index 97f1327f..00000000 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_user_logic.py +++ /dev/null @@ -1,155 +0,0 @@ -# standard imports -import json - -# third-party-imports -import pytest - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.redis import InMemoryStore -from cic_ussd.state_machine.logic.user import ( - change_preferred_language_to_en, - change_preferred_language_to_sw, - edit_user_metadata_attribute, - format_user_metadata, - get_user_metadata, - save_complete_user_metadata, - process_gender_user_input, - save_metadata_attribute_to_session_data, - update_account_status_to_active) - - -def test_change_preferred_language(create_pending_user, create_in_db_ussd_session): - state_machine_data = ('', create_in_db_ussd_session, create_pending_user) - assert create_pending_user.preferred_language is None - change_preferred_language_to_en(state_machine_data) - assert create_pending_user.preferred_language == 'en' - change_preferred_language_to_sw(state_machine_data) - assert create_pending_user.preferred_language == 'sw' - - -def test_update_account_status_to_active(create_pending_user, create_in_db_ussd_session): - state_machine_data = ('', create_in_db_ussd_session, create_pending_user) - update_account_status_to_active(state_machine_data) - assert create_pending_user.get_account_status() == 'ACTIVE' - - -@pytest.mark.parametrize("current_state, expected_key, expected_result, user_input", [ - ("enter_given_name", "given_name", "John", "John"), - ("enter_family_name", "family_name", "Doe", "Doe"), - ("enter_gender", "gender", "Male", "1"), - ("enter_location", "location", "Kangemi", "Kangemi"), - ("enter_products", "products", "Mandazi", "Mandazi"), -]) -def test_save_metadata_attribute_to_session_data(current_state, - expected_key, - expected_result, - user_input, - celery_session_worker, - create_activated_user, - create_in_db_ussd_session, - create_in_redis_ussd_session): - create_in_db_ussd_session.state = current_state - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_activated_user) - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - assert in_memory_ussd_session.get('session_data') == {} - serialized_in_db_ussd_session['state'] = current_state - save_metadata_attribute_to_session_data(state_machine_data=state_machine_data) - - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data')[expected_key] == expected_result - - -@pytest.mark.parametrize("preferred_language, user_input, expected_gender_value", [ - ("en", "1", "Male"), - ("en", "2", "Female"), - ("sw", "1", "Mwanaume"), - ("sw", "2", "Mwanamke"), -]) -def test_process_gender_user_input(create_activated_user, expected_gender_value, preferred_language, user_input): - create_activated_user.preferred_language = preferred_language - gender = process_gender_user_input(user=create_activated_user, user_input=user_input) - assert gender == expected_gender_value - - -def test_format_user_metadata(create_activated_user, - complete_user_metadata, - setup_chain_spec): - from cic_types.models.person import Person - formatted_user_metadata = format_user_metadata(metadata=complete_user_metadata, user=create_activated_user) - person = Person() - user_metadata = person.deserialize(person_data=formatted_user_metadata) - assert formatted_user_metadata == user_metadata.serialize() - - -def test_save_complete_user_metadata(celery_session_worker, - complete_user_metadata, - create_activated_user, - create_in_redis_ussd_session, - mocker, - setup_chain_spec, - ussd_session_data): - ussd_session = create_in_redis_ussd_session.get(ussd_session_data.get('external_session_id')) - ussd_session = json.loads(ussd_session) - ussd_session['session_data'] = complete_user_metadata - user_metadata = format_user_metadata(metadata=ussd_session.get('session_data'), user=create_activated_user) - state_machine_data = ('', ussd_session, create_activated_user) - mocked_create_metadata_task = mocker.patch('cic_ussd.tasks.metadata.create_person_metadata.apply_async') - save_complete_user_metadata(state_machine_data=state_machine_data) - mocked_create_metadata_task.assert_called_with( - (user_metadata, create_activated_user.blockchain_address), - {}, - queue='cic-ussd' - ) - - -def test_edit_user_metadata_attribute(celery_session_worker, - cached_user_metadata, - create_activated_user, - create_in_redis_ussd_session, - init_redis_cache, - mocker, - person_metadata, - setup_chain_spec, - ussd_session_data): - ussd_session = create_in_redis_ussd_session.get(ussd_session_data.get('external_session_id')) - ussd_session = json.loads(ussd_session) - - assert person_metadata['location']['area_name'] == 'kayaba' - - # appropriately format session - ussd_session['session_data'] = { - 'location': 'nairobi' - } - state_machine_data = ('', ussd_session, create_activated_user) - - mocked_edit_metadata = mocker.patch('cic_ussd.tasks.metadata.edit_person_metadata.apply_async') - edit_user_metadata_attribute(state_machine_data=state_machine_data) - person_metadata['location']['area_name'] = 'nairobi' - mocked_edit_metadata.assert_called_with( - (create_activated_user.blockchain_address, person_metadata, Chain.spec.engine()), - {}, - queue='cic-ussd' - ) - - -def test_get_user_metadata_attribute(celery_session_worker, - create_activated_user, - create_in_redis_ussd_session, - mocker, - ussd_session_data): - ussd_session = create_in_redis_ussd_session.get(ussd_session_data.get('external_session_id')) - ussd_session = json.loads(ussd_session) - state_machine_data = ('', ussd_session, create_activated_user) - - mocked_get_metadata = mocker.patch('cic_ussd.tasks.metadata.query_person_metadata.apply_async') - get_user_metadata(state_machine_data=state_machine_data) - mocked_get_metadata.assert_called_with( - (create_activated_user.blockchain_address,), - {}, - queue='cic-ussd' - ) diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py index 8e91d37d..044256c3 100644 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/logic/test_validator_logic.py @@ -1,55 +1,54 @@ # standard imports -import json -# third-party imports +# external imports import pytest -from cic_types.models.person import generate_metadata_pointer # local imports -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.redis import cache_data -from cic_ussd.state_machine.logic.validator import (is_valid_name, +from cic_ussd.state_machine.logic.validator import (has_cached_person_metadata, + is_valid_name, is_valid_gender_selection, - has_cached_user_metadata) + is_valid_date) + +# test imports + + +def test_has_cached_person_metadata(activated_account, + cache_person_metadata, + generic_ussd_session, + init_database, + pending_account): + state_machine_data = ('', generic_ussd_session, activated_account, init_database) + assert has_cached_person_metadata(state_machine_data) is True + state_machine_data = ('', generic_ussd_session, pending_account, init_database) + assert has_cached_person_metadata(state_machine_data) is False @pytest.mark.parametrize("user_input, expected_result", [ ("Arya", True), ("1234", False) ]) -def test_is_valid_name(create_in_db_ussd_session, create_pending_user, user_input, expected_result): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_pending_user) - result = is_valid_name(state_machine_data=state_machine_data) - assert result is expected_result +def test_is_valid_name(expected_result, generic_ussd_session, init_database, pending_account, user_input): - -def test_has_cached_user_metadata(create_in_db_ussd_session, - create_activated_user, - init_redis_cache, - person_metadata): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = ('', serialized_in_db_ussd_session, create_activated_user) - result = has_cached_user_metadata(state_machine_data=state_machine_data) - assert result is False - # cache metadata - user = create_activated_user - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=user.blockchain_address), - cic_type=':cic.person' - ) - cache_data(key=key, data=json.dumps(person_metadata)) - result = has_cached_user_metadata(state_machine_data=state_machine_data) - assert result + state_machine_data = (user_input, generic_ussd_session, pending_account, init_database) + assert is_valid_name(state_machine_data) is expected_result @pytest.mark.parametrize("user_input, expected_result", [ ("1", True), ("2", True), + ("3", True), + ("4", False) +]) +def test_is_valid_gender_selection(expected_result, generic_ussd_session, init_database, pending_account, user_input): + state_machine_data = (user_input, generic_ussd_session, pending_account, init_database) + assert is_valid_gender_selection(state_machine_data) is expected_result + + +@pytest.mark.parametrize("user_input, expected_result", [ + ("1935", True), + ("1825", False), ("3", False) ]) -def test_is_valid_gender_selection(create_in_db_ussd_session, create_pending_user, user_input, expected_result): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine_data = (user_input, serialized_in_db_ussd_session, create_pending_user) - result = is_valid_gender_selection(state_machine_data=state_machine_data) - assert result is expected_result +def test_is_valid_date(expected_result, generic_ussd_session, init_database, pending_account, user_input): + state_machine_data = (user_input, generic_ussd_session, pending_account, init_database) + assert is_valid_date(state_machine_data) is expected_result diff --git a/apps/cic-ussd/tests/cic_ussd/state_machine/test_state_machine.py b/apps/cic-ussd/tests/cic_ussd/state_machine/test_state_machine.py index 979cd9f4..ee853735 100644 --- a/apps/cic-ussd/tests/cic_ussd/state_machine/test_state_machine.py +++ b/apps/cic-ussd/tests/cic_ussd/state_machine/test_state_machine.py @@ -2,11 +2,12 @@ from cic_ussd.state_machine import UssdStateMachine -def test_state_machine(create_in_db_ussd_session, - get_in_redis_ussd_session, - load_data_into_state_machine, - create_pending_user): - serialized_in_db_ussd_session = create_in_db_ussd_session.to_json() - state_machine = UssdStateMachine(ussd_session=get_in_redis_ussd_session.to_json()) - state_machine.scan_data(('1', serialized_in_db_ussd_session, create_pending_user)) +def test_state_machine(activated_account_ussd_session, + celery_session_worker, + init_database, + init_state_machine, + pending_account): + state_machine = UssdStateMachine(activated_account_ussd_session) + state_machine.scan_data(('1', activated_account_ussd_session, pending_account, init_database)) + assert state_machine.__repr__() == f'' assert state_machine.state == 'initial_pin_entry' diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py new file mode 100644 index 00000000..f70e220c --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler.py @@ -0,0 +1,178 @@ +# standard imports +import json +from decimal import Decimal + +# external imports +import celery +import pytest +import requests_mock +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.statement import generate, filter_statement_transactions +from cic_ussd.account.transaction import transaction_actors +from cic_ussd.cache import cache_data_key, get_cached_data +from cic_ussd.db.models.account import Account +from cic_ussd.error import AccountCreationDataNotFound +from cic_ussd.metadata import PreferencesMetadata + + +# test imports +from tests.helpers.accounts import blockchain_address + + +def test_account_creation_callback(account_creation_data, + cache_account_creation_data, + celery_session_worker, + custom_metadata, + init_cache, + init_database, + load_chain_spec, + mocker, + setup_metadata_request_handler, + setup_metadata_signer): + phone_number = account_creation_data.get('phone_number') + result = blockchain_address() + task_uuid = account_creation_data.get('task_uuid') + + mock_task = mocker.patch('celery.app.task.Task.request') + mock_task.root_id = task_uuid + mock_task.delivery_info = {'routing_key': 'cic-ussd'} + + status_code = 1 + with pytest.raises(ValueError) as error: + s_account_creation_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.account_creation_callback', [task_uuid, '', status_code] + ) + s_account_creation_callback.apply_async().get() + assert str(error.value) == f'Unexpected status code: {status_code}' + + cached_account_creation_data = get_cached_data(task_uuid) + cached_account_creation_data = json.loads(cached_account_creation_data) + assert cached_account_creation_data.get('status') == account_creation_data.get('status') + mock_add_phone_pointer = mocker.patch('cic_ussd.tasks.metadata.add_phone_pointer.apply_async') + mock_add_custom_metadata = mocker.patch('cic_ussd.tasks.metadata.add_custom_metadata.apply_async') + s_account_creation_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.account_creation_callback', [result, '', 0] + ) + s_account_creation_callback.apply_async().get() + account = init_database.query(Account).filter_by(phone_number=phone_number).first() + assert account.blockchain_address == result + cached_account_creation_data = get_cached_data(task_uuid) + cached_account_creation_data = json.loads(cached_account_creation_data) + assert cached_account_creation_data.get('status') == 'CREATED' + mock_add_phone_pointer.assert_called_with((result, phone_number), {}, queue='cic-ussd') + mock_add_custom_metadata.assert_called_with((result, custom_metadata), {}, queue='cic-ussd') + + task_uuid = celery.uuid() + mock_task.root_id = task_uuid + with pytest.raises(AccountCreationDataNotFound) as error: + s_account_creation_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.account_creation_callback', [task_uuid, '', 0] + ) + s_account_creation_callback.apply_async().get() + assert str(error.value) == f'No account creation data found for task id: {task_uuid}' + + +def test_balances_callback(activated_account, balances, celery_session_worker): + status_code = 1 + with pytest.raises(ValueError) as error: + s_balances_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.balances_callback', + [balances, activated_account.blockchain_address, status_code]) + s_balances_callback.apply_async().get() + assert str(error.value) == f'Unexpected status code: {status_code}.' + + status_code = 0 + s_balances_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.balances_callback', + [balances, activated_account.blockchain_address, status_code]) + s_balances_callback.apply_async().get() + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + key = cache_data_key(identifier, ':cic.balances') + cached_balances = get_cached_data(key) + cached_balances = json.loads(cached_balances) + assert cached_balances == balances[0] + + +def test_statement_callback(activated_account, mocker, transactions_list): + status_code = 1 + with pytest.raises(ValueError) as error: + s_statement_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.statement_callback', + [transactions_list, activated_account.blockchain_address, status_code]) + s_statement_callback.apply_async().get() + assert str(error.value) == f'Unexpected status code: {status_code}.' + + mock_task = mocker.patch('celery.app.task.Task.request') + mock_task.delivery_info = {'routing_key': 'cic-ussd'} + mock_statement_generate = mocker.patch('cic_ussd.tasks.processor.generate_statement.apply_async') + status_code = 0 + s_statement_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.statement_callback', + [transactions_list, activated_account.blockchain_address, status_code]) + s_statement_callback.apply_async().get() + statement_transactions = filter_statement_transactions(transactions_list) + recipient_transaction, sender_transaction = transaction_actors(statement_transactions[0]) + mock_statement_generate.assert_called_with( + (activated_account.blockchain_address, sender_transaction), {}, queue='cic-ussd') + + +def test_transaction_balances_callback(activated_account, + balances, + cache_balances, + cache_person_metadata, + cache_preferences, + load_chain_spec, + mocker, + preferences, + setup_metadata_signer, + setup_metadata_request_handler, + set_locale_files, + transaction_result): + status_code = 1 + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + with pytest.raises(ValueError) as error: + s_transaction_balances_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.transaction_balances_callback', + [balances, sender_transaction, status_code]) + s_transaction_balances_callback.apply_async().get() + assert str(error.value) == f'Unexpected status code: {status_code}.' + mocked_chain = mocker.patch('celery.chain') + mock_task_request = mocker.patch('celery.app.task.Task.request') + mock_task_request.delivery_info = {'routing_key': 'cic-ussd'} + sender_transaction['transaction_type'] = 'transfer' + status_code = 0 + s_transaction_balances_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.transaction_balances_callback', + [balances, sender_transaction, status_code]) + s_transaction_balances_callback.apply_async().get() + mocked_chain.assert_called() + sender_transaction['transaction_type'] = 'tokengift' + status_code = 0 + s_transaction_balances_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.transaction_balances_callback', + [balances, sender_transaction, status_code]) + s_transaction_balances_callback.apply_async().get() + mocked_chain.assert_called() + + +def test_transaction_callback(load_chain_spec, mock_async_balance_api_query, transaction_result): + status_code = 1 + with pytest.raises(ValueError) as error: + s_transaction_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.transaction_callback', + [transaction_result, 'transfer', status_code]) + s_transaction_callback.apply_async().get() + assert str(error.value) == f'Unexpected status code: {status_code}.' + + status_code = 0 + s_transaction_callback = celery.signature( + 'cic_ussd.tasks.callback_handler.transaction_callback', + [transaction_result, 'transfer', status_code]) + s_transaction_callback.apply_async().get() + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + assert mock_async_balance_api_query.get('address') == recipient_transaction.get('blockchain_address') or sender_transaction.get('blockchain_address') + assert mock_async_balance_api_query.get('token_symbol') == recipient_transaction.get('token_symbol') or sender_transaction.get('token_symbol') + + diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler_tasks.py deleted file mode 100644 index 43df49cc..00000000 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_callback_handler_tasks.py +++ /dev/null @@ -1,203 +0,0 @@ -# standard imports -import json -import logging -from datetime import datetime - -# third party imports -import celery -import pytest - -# local imports -from cic_ussd.db.models.account import Account -from cic_ussd.error import ActionDataNotFoundError -from cic_ussd.conversions import from_wei - -logg = logging.getLogger() - - -def test_successful_process_account_creation_callback_task(account_creation_action_data, - celery_session_worker, - init_database, - init_redis_cache, - mocker, - set_account_creation_action_data): - phone_number = account_creation_action_data.get('phone_number') - task_id = account_creation_action_data.get('task_id') - - mocked_task_request = mocker.patch('celery.app.task.Task.request') - - # WARNING: [THE SETTING OF THE ROOT ID IS A HACK AND SHOULD BE REVIEWED OR IMPROVED] - mocked_task_request.root_id = task_id - - user = init_database.query(Account).filter_by(phone_number=phone_number).first() - assert user is None - - redis_cache = init_redis_cache - action_data = redis_cache.get(task_id) - action_data = json.loads(action_data) - - assert action_data.get('status') == 'PENDING' - - status_code = 0 - result = '0x6315c185fd23bDbbba058E2a504197915aCC5065' - url = '' - - s_process_callback_request = celery.signature( - 'cic_ussd.tasks.callback_handler.process_account_creation_callback', - [result, url, status_code] - ) - s_process_callback_request.apply_async().get() - - user = init_database.query(Account).filter_by(phone_number=phone_number).first() - assert user.blockchain_address == result - - action_data = redis_cache.get(task_id) - action_data = json.loads(action_data) - - assert action_data.get('status') == 'CREATED' - - -def test_unsuccessful_process_account_creation_callback_task(init_database, - init_redis_cache, - celery_session_worker): - with pytest.raises(ActionDataNotFoundError) as error: - status_code = 0 - result = '0x6315c185fd23bDbbba058E2a504197915aCC5065' - url = '' - - s_process_callback_request = celery.signature( - 'cic_ussd.tasks.callback_handler.process_account_creation_callback', - [result, url, status_code] - ) - result = s_process_callback_request.apply_async() - task_id = result.get() - - assert str(error.value) == f'Account creation task: {task_id}, returned unexpected response: {status_code}' - - -def test_successful_token_gift_incoming_transaction(celery_session_worker, - create_activated_user, - mock_notifier_api, - set_locale_files, - successful_incoming_token_gift_callback): - result = successful_incoming_token_gift_callback.get('RESULT') - param = successful_incoming_token_gift_callback.get('PARAM') - status_code = successful_incoming_token_gift_callback.get('STATUS_CODE') - - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - balance = from_wei(result.get('destination_value')) - token_symbol = result.get('token_symbol') - - messages = mock_notifier_api - - assert messages[0].get('recipient') == create_activated_user.phone_number - assert messages[0].get( - 'message') == f'Hello {""} you have been registered on Sarafu Network! Your balance is {balance} {token_symbol}. To use dial *483*46#. For help 0757628885.' - - -def test_successful_transfer_incoming_transaction(celery_session_worker, - create_valid_tx_sender, - create_valid_tx_recipient, - mock_notifier_api, - set_locale_files, - successful_incoming_transfer_callback): - result = successful_incoming_transfer_callback.get('RESULT') - param = successful_incoming_transfer_callback.get('PARAM') - status_code = successful_incoming_transfer_callback.get('STATUS_CODE') - - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - value = result.get('destination_value') - balance = '' - token_symbol = result.get('token_symbol') - - sender_first_name = '' - sender_last_name = '' - phone_number = create_valid_tx_sender.phone_number - tx_sender_information = f'{phone_number}, {sender_first_name}, {sender_last_name}' - amount = from_wei(value=value) - timestamp = datetime.now().strftime('%d-%m-%y, %H:%M %p') - - messages = mock_notifier_api - - assert messages[0].get('recipient') == create_valid_tx_recipient.phone_number - assert messages[0].get( - 'message') == f'Successfully received {amount} {token_symbol} from {tx_sender_information} {timestamp}. New balance is {balance} {token_symbol}.' - - -def test_unsuccessful_incoming_transaction_recipient_not_found(celery_session_worker, - create_valid_tx_sender, - successful_incoming_transfer_callback): - result = successful_incoming_transfer_callback.get('RESULT') - param = successful_incoming_transfer_callback.get('PARAM') - status_code = successful_incoming_transfer_callback.get('STATUS_CODE') - - with pytest.raises(ValueError) as error: - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - recipient_blockchain_address = result.get('recipient') - assert str(error.value) == f'Tx for recipient: {recipient_blockchain_address} was received but has no matching user in the system.' - - -def test_successful_incoming_transaction_sender_not_found(caplog, - celery_session_worker, - create_valid_tx_recipient, - mock_notifier_api, - successful_incoming_transfer_callback): - result = successful_incoming_transfer_callback.get('RESULT') - param = successful_incoming_transfer_callback.get('PARAM') - status_code = successful_incoming_transfer_callback.get('STATUS_CODE') - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - sender_blockchain_address = result.get('sender') - assert 'Balance requires implementation of cic-eth integration with balance.' in caplog.text - # assert f'Tx with sender: {sender_blockchain_address} was received but has no matching user in the system.\n' in caplog.text - - -def test_unsuccessful_incoming_transaction_invalid_status_code(celery_session_worker, - incoming_transfer_callback_invalid_tx_status_code): - result = incoming_transfer_callback_invalid_tx_status_code.get('RESULT') - param = incoming_transfer_callback_invalid_tx_status_code.get('PARAM') - status_code = incoming_transfer_callback_invalid_tx_status_code.get('STATUS_CODE') - - with pytest.raises(ValueError) as error: - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - assert str(error.value) == f'Unexpected status code: {status_code}' - - -def test_unsuccessful_incoming_transaction_invalid_param(celery_session_worker, - incoming_transfer_callback_invalid_tx_param): - result = incoming_transfer_callback_invalid_tx_param.get('RESULT') - param = incoming_transfer_callback_invalid_tx_param.get('PARAM') - status_code = incoming_transfer_callback_invalid_tx_param.get('STATUS_CODE') - - with pytest.raises(ValueError) as error: - s_process_token_gift = celery.signature( - 'cic_ussd.tasks.callback_handler.process_incoming_transfer_callback', - [result, param, status_code] - ) - s_process_token_gift.apply_async().get() - - assert str(error.value) == f'Unexpected transaction: param {status_code}' diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py new file mode 100644 index 00000000..a00c82c7 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_metadata_tasks.py @@ -0,0 +1,52 @@ +# standard imports +import json +import os + +# external imports +import celery +import requests_mock +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.cache import cache_data_key, get_cached_data +from cic_ussd.metadata import PersonMetadata, PreferencesMetadata + +# tests imports + + +def test_query_person_metadata(activated_account, + celery_session_worker, + init_cache, + person_metadata, + setup_metadata_request_handler, + setup_metadata_signer): + with requests_mock.Mocker(real_http=False) as request_mocker: + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + metadata_client = PersonMetadata(identifier) + request_mocker.register_uri('GET', metadata_client.url, json=person_metadata, reason='OK', status_code=200) + s_query_person_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_person_metadata', [activated_account.blockchain_address]) + s_query_person_metadata.apply().get() + key = cache_data_key(identifier, ':cic.person') + cached_person_metadata = get_cached_data(key) + cached_person_metadata = json.loads(cached_person_metadata) + assert cached_person_metadata == person_metadata + + +def test_query_preferences_metadata(activated_account, + celery_session_worker, + init_cache, + preferences, + setup_metadata_request_handler, + setup_metadata_signer): + with requests_mock.Mocker(real_http=False) as request_mocker: + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + metadata_client = PreferencesMetadata(identifier) + request_mocker.register_uri('GET', metadata_client.url, json=preferences, reason='OK', status_code=200) + query_preferences_metadata = celery.signature( + 'cic_ussd.tasks.metadata.query_preferences_metadata', [activated_account.blockchain_address]) + query_preferences_metadata.apply().get() + key = cache_data_key(identifier, ':cic.preferences') + cached_preferences_metadata = get_cached_data(key) + cached_preferences_metadata = json.loads(cached_preferences_metadata) + assert cached_preferences_metadata == preferences diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py new file mode 100644 index 00000000..61e19761 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_notifications_tasks.py @@ -0,0 +1,71 @@ +# standard imports +import datetime + +# external imports +import celery + +# local imports +from cic_ussd.account.transaction import from_wei +from cic_ussd.phone_number import Support +from cic_ussd.translation import translation_for + + +# tests imports + + +def test_transaction(celery_session_worker, + load_support_phone, + mock_notifier_api, + notification_data, + set_locale_files): + notification_data['transaction_type'] = 'transfer' + amount = from_wei(notification_data.get('token_value')) + balance = notification_data.get('available_balance') + phone_number = notification_data.get('phone_number') + preferred_language = notification_data.get('preferred_language') + token_symbol = notification_data.get('token_symbol') + transaction_account_metadata = notification_data.get('metadata_id') + timestamp = datetime.datetime.now().strftime('%d-%m-%y, %H:%M %p') + s_transaction = celery.signature( + 'cic_ussd.tasks.notifications.transaction', [notification_data] + ) + s_transaction.apply_async().get() + assert mock_notifier_api.get('recipient') == phone_number + message = translation_for(key='sms.sent_tokens', + phone_number=phone_number, + preferred_language=preferred_language, + amount=amount, + token_symbol=token_symbol, + tx_recipient_information=transaction_account_metadata, + timestamp=timestamp, + balance=balance) + assert mock_notifier_api.get('message') == message + + notification_data['role'] = 'recipient' + notification_data['direction_tag'] = 'From' + s_transaction = celery.signature( + 'cic_ussd.tasks.notifications.transaction', [notification_data] + ) + s_transaction.apply_async().get() + message = translation_for(key='sms.received_tokens', + phone_number=phone_number, + preferred_language=preferred_language, + amount=amount, + token_symbol=token_symbol, + tx_sender_information=transaction_account_metadata, + timestamp=timestamp, + balance=balance) + assert mock_notifier_api.get('message') == message + + notification_data['transaction_type'] = 'tokengift' + s_transaction = celery.signature( + 'cic_ussd.tasks.notifications.transaction', [notification_data] + ) + s_transaction.apply_async().get() + support_phone = Support.phone_number + message = translation_for(key='sms.account_successfully_created', + preferred_language=preferred_language, + balance=balance, + support_phone=support_phone, + token_symbol=token_symbol) + assert mock_notifier_api.get('message') == message diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_persist_session_to_db_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_persist_session_to_db_tasks.py deleted file mode 100644 index b82f2355..00000000 --- a/apps/cic-ussd/tests/cic_ussd/tasks/test_persist_session_to_db_tasks.py +++ /dev/null @@ -1,46 +0,0 @@ -# third party imports -import celery -import pytest - -# local imports -from cic_ussd.db.models.ussd_session import UssdSession -from cic_ussd.error import SessionNotFoundError - - -def test_persist_session_to_db_task( - init_database, - create_activated_user, - ussd_session_data, - celery_session_worker, - create_in_redis_ussd_session): - external_session_id = ussd_session_data.get('external_session_id') - s_persist_session_to_db = celery.signature( - 'cic_ussd.tasks.ussd_session.persist_session_to_db', - [external_session_id] - ) - result = s_persist_session_to_db.apply_async() - result.get() - db_session = init_database.query(UssdSession).filter_by(external_session_id=external_session_id).first() - assert db_session.external_session_id == 'AT974186' - assert db_session.service_code == '*483*46#' - assert db_session.msisdn == '+25498765432' - assert db_session.user_input == '1' - assert db_session.state == 'initial_language_selection' - assert db_session.session_data is None - assert db_session.version == 2 - - assert UssdSession.have_session_for_phone(create_activated_user.phone_number) - - -def test_session_not_found_error( - celery_session_worker, - create_in_redis_ussd_session): - with pytest.raises(SessionNotFoundError) as error: - external_session_id = 'SomeRandomValue' - s_persist_session_to_db = celery.signature( - 'cic_ussd.tasks.ussd_session.persist_session_to_db', - [external_session_id] - ) - result = s_persist_session_to_db.apply_async() - result.get() - assert str(error.value) == "Session does not exist!" diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py new file mode 100644 index 00000000..999bd4fb --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_processor_tasks.py @@ -0,0 +1,75 @@ +# standard imports +import json + +# external imports +import celery +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.transaction import transaction_actors +from cic_ussd.cache import cache_data_key, get_cached_data + + +# test imports + + +def test_generate_statement(activated_account, + celery_session_worker, + cache_preferences, + mocker, + transaction_result): + mock_task = mocker.patch('celery.app.task.Task.request') + mock_task.delivery_info = {'routing_key': 'cic-ussd'} + mock_chain = mocker.patch('celery.chain') + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + s_generate_statement = celery.signature( + 'cic_ussd.tasks.processor.generate_statement', [activated_account.blockchain_address, sender_transaction] + ) + result = s_generate_statement.apply_async().get() + mock_chain.assert_called_once() + + +def test_cache_statement(activated_account, + cache_person_metadata, + celery_session_worker, + init_database, + preferences, + transaction_result): + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + key = cache_data_key(identifier, ':cic.statement') + cached_statement = get_cached_data(key) + assert cached_statement is None + s_parse_transaction = celery.signature( + 'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction]) + result = s_parse_transaction.apply_async().get() + s_cache_statement = celery.signature( + 'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address] + ) + s_cache_statement.apply_async().get() + cached_statement = get_cached_data(key) + cached_statement = json.loads(cached_statement) + assert len(cached_statement) == 1 + s_cache_statement = celery.signature( + 'cic_ussd.tasks.processor.cache_statement', [result, activated_account.blockchain_address] + ) + s_cache_statement.apply_async().get() + cached_statement = get_cached_data(key) + cached_statement = json.loads(cached_statement) + assert len(cached_statement) == 2 + + +def test_parse_transaction(activated_account, + cache_person_metadata, + celery_session_worker, + init_database, + preferences, + transaction_result): + recipient_transaction, sender_transaction = transaction_actors(transaction_result) + assert sender_transaction.get('metadata_id') is None + assert sender_transaction.get('phone_number') is None + s_parse_transaction = celery.signature( + 'cic_ussd.tasks.processor.parse_transaction', [preferences, sender_transaction]) + result = s_parse_transaction.apply_async().get() + assert result.get('metadata_id') == activated_account.standard_metadata_id() + assert result.get('phone_number') == activated_account.phone_number diff --git a/apps/cic-ussd/tests/cic_ussd/tasks/test_ussd_session_tasks.py b/apps/cic-ussd/tests/cic_ussd/tasks/test_ussd_session_tasks.py new file mode 100644 index 00000000..e1187fcd --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/tasks/test_ussd_session_tasks.py @@ -0,0 +1,33 @@ +# standard imports + +# external imports +import celery +import pytest + +# local imports +from cic_ussd.db.models.ussd_session import UssdSession +from cic_ussd.error import SessionNotFoundError + +# tests imports + + +def test_persist_session_to_db(cached_ussd_session, celery_session_worker, init_cache, init_database): + external_session_id = cached_ussd_session.external_session_id + s_persist_session_to_db = celery.signature( + 'cic_ussd.tasks.ussd_session.persist_session_to_db', [external_session_id]) + s_persist_session_to_db.apply_async().get() + ussd_session = init_database.query(UssdSession).filter_by(external_session_id=external_session_id).first() + assert ussd_session.external_session_id == cached_ussd_session.external_session_id + assert ussd_session.service_code == cached_ussd_session.service_code + assert ussd_session.msisdn == cached_ussd_session.msisdn + assert ussd_session.user_input == cached_ussd_session.user_input + assert ussd_session.state == cached_ussd_session.state + assert ussd_session.data is None + assert ussd_session.version == cached_ussd_session.version + assert UssdSession.has_record_for_phone_number(ussd_session.msisdn, init_database) + with pytest.raises(SessionNotFoundError) as error: + external_session_id = 'SomeRandomValue' + s_persist_session_to_db = celery.signature( + 'cic_ussd.tasks.ussd_session.persist_session_to_db', [external_session_id]) + result = s_persist_session_to_db.apply_async().get() + assert str(error.value) == "Session does not exist!" diff --git a/apps/cic-ussd/tests/cic_ussd/test_accounts.py b/apps/cic-ussd/tests/cic_ussd/test_accounts.py deleted file mode 100644 index 46822186..00000000 --- a/apps/cic-ussd/tests/cic_ussd/test_accounts.py +++ /dev/null @@ -1,20 +0,0 @@ -# standard imports - -# third-party imports - -# local imports -from cic_ussd.balance import BalanceManager -from cic_ussd.chain import Chain - - -def test_balance_manager(create_valid_tx_recipient, load_config, mocker, setup_chain_spec): - chain_str = Chain.spec.__str__() - balance_manager = BalanceManager( - address=create_valid_tx_recipient.blockchain_address, - chain_str=chain_str, - token_symbol='SRF' - ) - balance_manager.get_balances = mocker.MagicMock() - balance_manager.get_balances() - - balance_manager.get_balances.assert_called_once() diff --git a/apps/cic-ussd/tests/cic_ussd/test_cache.py b/apps/cic-ussd/tests/cic_ussd/test_cache.py new file mode 100644 index 00000000..6e7625b4 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/test_cache.py @@ -0,0 +1,33 @@ +# standard imports +import hashlib +import json + +# external imports + +# local imports +from cic_ussd.cache import cache_data, cache_data_key, get_cached_data + +# test imports + + +def test_cache_data(init_cache): + identifier = 'some_key'.encode() + key = cache_data_key(identifier, ':testing') + assert get_cached_data(key) is None + cache_data(key, json.dumps('some_value')) + assert get_cached_data(key) is not None + + +def test_cache_data_key(): + identifier = 'some_key'.encode() + key = cache_data_key(identifier, ':testing') + hash_object = hashlib.new("sha256") + hash_object.update(identifier) + hash_object.update(':testing'.encode(encoding="utf-8")) + assert hash_object.digest().hex() == key + + +def test_get_cached_data(cached_ussd_session): + ussd_session = get_cached_data(cached_ussd_session.external_session_id) + ussd_session = json.loads(ussd_session) + assert ussd_session.get('msisdn') == cached_ussd_session.msisdn diff --git a/apps/cic-ussd/tests/cic_ussd/test_encoder.py b/apps/cic-ussd/tests/cic_ussd/test_encode.py similarity index 100% rename from apps/cic-ussd/tests/cic_ussd/test_encoder.py rename to apps/cic-ussd/tests/cic_ussd/test_encode.py diff --git a/apps/cic-ussd/tests/cic_ussd/test_notifications.py b/apps/cic-ussd/tests/cic_ussd/test_notifications.py index 927dad1f..bfd0b1f6 100644 --- a/apps/cic-ussd/tests/cic_ussd/test_notifications.py +++ b/apps/cic-ussd/tests/cic_ussd/test_notifications.py @@ -1,6 +1,6 @@ # standard imports -# third-party imports +# external imports import pytest # local imports @@ -14,19 +14,15 @@ from cic_ussd.notifications import Notifier def test_send_sms_notification(celery_session_worker, expected_message, key, + mock_notifier_api, preferred_language, recipient, - set_locale_files, - mock_notifier_api): - + set_locale_files): notifier = Notifier() notifier.queue = None - notifier.send_sms_notification(key=key, phone_number=recipient, preferred_language=preferred_language) - messages = mock_notifier_api - - assert messages[0].get('message') == expected_message - assert messages[0].get('recipient') == recipient + assert mock_notifier_api.get('message') == expected_message + assert mock_notifier_api.get('recipient') == recipient diff --git a/apps/cic-ussd/tests/cic_ussd/test_operations.py b/apps/cic-ussd/tests/cic_ussd/test_operations.py deleted file mode 100644 index 6285ed07..00000000 --- a/apps/cic-ussd/tests/cic_ussd/test_operations.py +++ /dev/null @@ -1,243 +0,0 @@ -# standard imports -import json -import uuid - -# third party imports -import pytest - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.db.models.task_tracker import TaskTracker -from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.operations import (add_tasks_to_tracker, - create_ussd_session, - create_or_update_session, - define_response_with_content, - define_multilingual_responses, - get_account_status, - get_latest_input, - initiate_account_creation_request, - process_current_menu, - process_menu_interaction_requests, - cache_account_creation_task_id, - reset_pin, - update_ussd_session, - save_to_in_memory_ussd_session_data) -from cic_ussd.phone_number import get_user_by_phone_number,process_phone_number -from cic_ussd.transactions import truncate -from cic_ussd.redis import InMemoryStore -from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession - - -def test_add_tasks_to_tracker(init_database): - task_uuid = '31e85315-feee-4b6d-995e-223569082cc4' - session = init_database - assert len(session.query(TaskTracker).all()) == 0 - - add_tasks_to_tracker(task_uuid=task_uuid) - task_in_tracker = session.query(TaskTracker).filter_by(task_uuid=task_uuid).first() - assert task_in_tracker.id == 1 - assert task_in_tracker.task_uuid == task_uuid - - -def test_create_ussd_session(create_in_redis_ussd_session, ussd_session_data): - external_session_id = ussd_session_data.get('external_session_id') - ussd_session = create_ussd_session( - external_session_id=external_session_id, - service_code=ussd_session_data.get('service_code'), - phone=ussd_session_data.get('msisdn'), - user_input=ussd_session_data.get('user_input'), - current_menu=ussd_session_data.get('state') - ) - in_memory_ussd_session = create_in_redis_ussd_session.get(external_session_id) - assert json.loads(in_memory_ussd_session).get('external_session_id') == ussd_session.external_session_id - - -def test_create_or_update_session(init_database, create_in_redis_ussd_session, ussd_session_data): - external_session_id = ussd_session_data.get('external_session_id') - ussd_session = create_or_update_session(external_session_id=external_session_id, - service_code=ussd_session_data.get('service_code'), - phone=ussd_session_data.get('msisdn'), - user_input=ussd_session_data.get('user_input'), - current_menu=ussd_session_data.get('state')) - in_memory_ussd_session = create_in_redis_ussd_session.get(external_session_id) - assert json.loads(in_memory_ussd_session).get('external_session_id') == ussd_session.external_session_id - - -@pytest.mark.parametrize('headers, response, expected_result',[ - ([('Content-Type', 'text/plain')], 'some-text', (b'some-text', [('Content-Type', 'text/plain'), ('Content-Length', '9')])), - ([('Content-Type', 'text/plain'), ('Content-Length', '0')], 'some-text', (b'some-text', [('Content-Type', 'text/plain'), ('Content-Length', '9')])) -]) -def test_define_response_with_content(headers, response, expected_result): - response_bytes, headers = define_response_with_content(headers=headers, response=response) - assert response_bytes, headers == expected_result - - -def test_define_multilingual_responses(load_ussd_menu, set_locale_files): - response = define_multilingual_responses( - key='ussd.kenya.account_creation_prompt', locales=['en', 'sw'], prefix='END') - assert response == "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n" - - -def test_get_account_status(create_pending_user): - user = create_pending_user - assert get_account_status(user.phone_number) == 'PENDING' - - -@pytest.mark.parametrize('user_input, expected_value', [ - ('1*9*6*7', '7'), - ('1', '1'), - ('', '') -]) -def test_get_latest_input(user_input, expected_value): - assert get_latest_input(user_input=user_input) == expected_value - - -def test_initiate_account_creation_request(account_creation_action_data, - create_in_redis_ussd_session, - init_database, - load_config, - load_ussd_menu, - mocker, - setup_chain_spec, - set_locale_files, - ussd_session_data): - external_session_id = ussd_session_data.get('external_session_id') - phone_number = account_creation_action_data.get('phone_number') - task_id = account_creation_action_data.get('task_id') - - class Callable: - id = task_id - - mocker.patch('cic_eth.api.api_task.Api.create_account', return_value=Callable) - mocked_cache_function = mocker.patch('cic_ussd.operations.cache_account_creation_task_id') - mocked_cache_function(phone_number, task_id) - - chain_str = Chain.spec.__str__() - response = initiate_account_creation_request(chain_str=chain_str, - external_session_id=external_session_id, - phone_number=ussd_session_data.get('msisdn'), - service_code=ussd_session_data.get('service_code'), - user_input=ussd_session_data.get('user_input')) - in_memory_ussd_session = InMemoryUssdSession.redis_cache.get(external_session_id) - - # check that ussd session was created - assert json.loads(in_memory_ussd_session).get('external_session_id') == external_session_id - assert response == "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n" - - -def test_reset_pin(create_pin_blocked_user): - user = create_pin_blocked_user - assert user.get_account_status() == 'LOCKED' - reset_pin(user.phone_number) - assert user.get_account_status() == 'RESET' - - -def test_update_ussd_session(create_in_redis_ussd_session, load_ussd_menu, ussd_session_data): - external_session_id = ussd_session_data.get('external_session_id') - ussd_session = create_ussd_session(external_session_id=external_session_id, - service_code=ussd_session_data.get('service_code'), - phone=ussd_session_data.get('msisdn'), - user_input=ussd_session_data.get('user_input'), - current_menu=ussd_session_data.get('state') - ) - assert ussd_session.user_input == ussd_session_data.get('user_input') - assert ussd_session.state == ussd_session_data.get('state') - ussd_session = update_ussd_session(ussd_session=ussd_session, user_input='1*2', current_menu='initial_pin_entry') - assert ussd_session.user_input == '1*2' - assert ussd_session.state == 'initial_pin_entry' - - -def test_process_current_menu(create_activated_user, create_in_db_ussd_session): - ussd_session = create_in_db_ussd_session - current_menu = process_current_menu(ussd_session=ussd_session, user=create_activated_user, user_input="") - assert current_menu == UssdMenu.find_by_name(name='exit_invalid_input') - current_menu = process_current_menu(ussd_session=None, user=create_activated_user, user_input="1*0000") - assert current_menu == UssdMenu.find_by_name(name='start') - - -def test_cache_account_creation_task_id(init_redis_cache): - phone_number = '+25412345678' - task_id = str(uuid.uuid4()) - cache_account_creation_task_id(phone_number=phone_number, task_id=task_id) - - redis_cache = init_redis_cache - action_data = redis_cache.get(task_id) - action_data = json.loads(action_data) - - assert action_data.get('phone_number') == phone_number - assert action_data.get('sms_notification_sent') is False - assert action_data.get('status') == 'PENDING' - assert action_data.get('task_id') == task_id - - -def test_save_to_in_memory_ussd_session_data(celery_session_worker, - create_in_db_ussd_session, - create_in_redis_ussd_session, - init_database): - - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data') == {} - - session_data = { - 'some_test_key': 'some_test_value' - } - save_to_in_memory_ussd_session_data( - queue='cic-ussd', - session_data=session_data, - ussd_session=create_in_db_ussd_session.to_json() - ) - - in_memory_ussd_session = InMemoryStore.cache.get('AT974186') - in_memory_ussd_session = json.loads(in_memory_ussd_session) - - assert in_memory_ussd_session.get('session_data') == session_data - - -@pytest.mark.parametrize("external_session_id, phone_number, expected_response", [ - ("AT123456789", "+254700000000", "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n"), - ("AT974186", "+25498765432", "CON Please enter a PIN to manage your account.\n0. Back") -]) -def test_process_menu_interaction_requests(external_session_id, - phone_number, - expected_response, - load_ussd_menu, - load_data_into_state_machine, - load_config, - setup_chain_spec, - celery_session_worker, - create_activated_user, - create_in_db_ussd_session): - chain_str = Chain.spec.__str__() - response = process_menu_interaction_requests( - chain_str=chain_str, - external_session_id=external_session_id, - phone_number=phone_number, - queue='cic-ussd', - service_code=load_config.get('APP_SERVICE_CODE'), - user_input='1' - ) - - assert response == expected_response - - -@pytest.mark.parametrize("phone_number, region, expected_result", [ - ("0712345678", "KE", "+254712345678"), - ("+254787654321", "KE", "+254787654321") -]) -def test_process_phone_number(expected_result, phone_number, region): - processed_phone_number = process_phone_number(phone_number=phone_number, region=region) - assert processed_phone_number == expected_result - - -def test_get_user_by_phone_number(create_activated_user): - known_phone_number = create_activated_user.phone_number - user = get_user_by_phone_number(phone_number=known_phone_number) - assert user is not None - assert create_activated_user.blockchain_address == user.blockchain_address - - unknown_phone_number = '+254700000000' - user = get_user_by_phone_number(phone_number=unknown_phone_number) - assert user is None diff --git a/apps/cic-ussd/tests/cic_ussd/test_phone_number.py b/apps/cic-ussd/tests/cic_ussd/test_phone_number.py new file mode 100644 index 00000000..cf7c6966 --- /dev/null +++ b/apps/cic-ussd/tests/cic_ussd/test_phone_number.py @@ -0,0 +1,18 @@ +# standard imports + +# external imports +import pytest + +# local imports +from cic_ussd.phone_number import process_phone_number + +# tests imports + + +@pytest.mark.parametrize("phone_number, region, expected_result", [ + ("0712345678", "KE", "+254712345678"), + ("+254787654321", "KE", "+254787654321") +]) +def test_process_phone_number(expected_result, phone_number, region): + processed_phone_number = process_phone_number(phone_number=phone_number, region=region) + assert processed_phone_number == expected_result diff --git a/apps/cic-ussd/tests/cic_ussd/test_processor.py b/apps/cic-ussd/tests/cic_ussd/test_processor.py deleted file mode 100644 index 1437d998..00000000 --- a/apps/cic-ussd/tests/cic_ussd/test_processor.py +++ /dev/null @@ -1,130 +0,0 @@ -# local imports -from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.processor import (custom_display_text, - next_state, - process_request, - process_pin_authorization, - process_transaction_pin_authorization, - process_exit_insufficient_balance, - process_exit_successful_transaction) - - -def test_process_pin_authorization(create_activated_user, - load_ussd_menu, - set_locale_files): - ussd_menu = UssdMenu.find_by_name(name='standard_pin_authorization') - response = process_pin_authorization( - display_key=ussd_menu.get('display_key'), - user=create_activated_user - ) - assert response == 'CON Please enter your PIN.\n0. Back' - - user_with_one_failed_pin_attempt = create_activated_user - user_with_one_failed_pin_attempt.failed_pin_attempts = 1 - alt_response = process_pin_authorization( - display_key=ussd_menu.get('display_key'), - user=user_with_one_failed_pin_attempt, - ) - assert alt_response == 'CON Please enter your PIN. You have 2 attempts remaining.\n0. Back' - - -def test_process_transaction_pin_authorization(create_activated_user, - create_in_db_ussd_session, - load_ussd_menu, - set_locale_files): - session_data = { - 'recipient_phone_number': '+254700000000', - } - ussd_session = create_in_db_ussd_session.to_json() - ussd_session['session_data'] = session_data - ussd_session['user_input'] = '1*0700000000*120' - ussd_menu = UssdMenu.find_by_name(name='transaction_pin_authorization') - response = process_transaction_pin_authorization( - display_key=ussd_menu.get('display_key'), - user=create_activated_user, - ussd_session=ussd_session - ) - assert response == 'CON +254700000000 will receive 120.00 SRF from +25498765432.\nPlease enter your PIN to confirm.\n0. Back' - - -def test_process_request_for_pending_user(load_ussd_menu, create_pending_user): - expected_menu = process_request(user_input="", user=create_pending_user) - assert expected_menu == UssdMenu.find_by_name(name='initial_language_selection') - - -def test_processor_request_for_activated_user(load_ussd_menu, create_activated_user): - expected_menu = process_request(user_input="", user=create_activated_user) - assert expected_menu == UssdMenu.find_by_name(name="start") - - -def test_next_state(load_data_into_state_machine, load_ussd_menu, create_in_db_ussd_session, create_pending_user): - assert create_in_db_ussd_session.state == "initial_language_selection" - successive_state = next_state( - ussd_session=create_in_db_ussd_session.to_json(), - user=create_pending_user, - user_input="1" - ) - assert successive_state == "initial_pin_entry" - - -def test_custom_display_text(create_activated_user, - get_in_redis_ussd_session, - load_ussd_menu, - set_locale_files): - ussd_session = get_in_redis_ussd_session - user = create_activated_user - ussd_menu = UssdMenu.find_by_name(name='exit_invalid_request') - english_translation = custom_display_text( - display_key=ussd_menu.get('display_key'), - menu_name=ussd_menu.get('name'), - user=user, - ussd_session=ussd_session - ) - user.preferred_language = 'sw' - swahili_translation = custom_display_text( - display_key=ussd_menu.get('display_key'), - menu_name=ussd_menu.get('name'), - user=user, - ussd_session=ussd_session - ) - assert swahili_translation == 'END Chaguo si sahihi.' - assert english_translation == 'END Invalid request.' - - -def test_process_exit_insufficient_balance( - create_valid_tx_recipient, - load_ussd_menu, - mock_balance, - set_locale_files, - ussd_session_data): - mock_balance(50) - ussd_session_data['user_input'] = f'1*{create_valid_tx_recipient.phone_number}*75' - ussd_session_data['session_data'] = {'recipient_phone_number': create_valid_tx_recipient.phone_number} - ussd_session_data['display_key'] = 'exit_insufficient_balance' - ussd_menu = UssdMenu.find_by_name(name='exit_insufficient_balance') - response = process_exit_insufficient_balance( - display_key=ussd_menu.get('display_key'), - user=create_valid_tx_recipient, - ussd_session=ussd_session_data - ) - assert response == 'CON Payment of 75.00 SRF to +25498765432 has failed due to insufficent balance.\nYour Sarafu-Network balances is: 50.00\n00. Back\n99. Exit' - - -def test_process_exit_successful_transaction( - create_valid_tx_recipient, - create_valid_tx_sender, - load_ussd_menu, - set_locale_files, - ussd_session_data): - ussd_session_data['session_data'] = { - 'recipient_phone_number': create_valid_tx_recipient.phone_number, - 'transaction_amount': 75 - } - ussd_session_data['display_key'] = 'exit_successful_transaction' - ussd_menu = UssdMenu.find_by_name(name='exit_successful_transaction') - response = process_exit_successful_transaction( - display_key=ussd_menu.get('display_key'), - user=create_valid_tx_sender, - ussd_session=ussd_session_data - ) - assert response == 'CON Your request has been sent. +25498765432 will receive 75.00 SRF from +25498765433.\n00. Back\n99. Exit' diff --git a/apps/cic-ussd/tests/cic_ussd/test_requests.py b/apps/cic-ussd/tests/cic_ussd/test_requests.py deleted file mode 100644 index 93c9e93e..00000000 --- a/apps/cic-ussd/tests/cic_ussd/test_requests.py +++ /dev/null @@ -1,65 +0,0 @@ -# standard imports -import json - -# local imports -from cic_ussd.db.models.account import Account -from cic_ussd.requests import (get_query_parameters, - get_request_endpoint, - get_request_method, - process_pin_reset_requests, - process_locked_accounts_requests) - - -def test_get_query_parameters(get_request_with_params_env): - param = get_query_parameters(env=get_request_with_params_env, query_name='phone') - assert param == '0700000000' - - -def test_get_request_endpoint(valid_locked_accounts_env): - param = get_request_endpoint(env=valid_locked_accounts_env) - assert param == '/accounts/locked/10/10' - - -def test_get_request_method(valid_locked_accounts_env): - param = get_request_method(env=valid_locked_accounts_env) - assert param == 'GET' - - -def test_process_pin_reset_requests(uwsgi_env, create_pin_blocked_user): - env = uwsgi_env - env['REQUEST_METHOD'] = 'GET' - message, status = process_pin_reset_requests(env=env, phone_number='070000000') - assert message == 'No user matching 070000000 was found.' - assert status == '404 Not Found' - - env['REQUEST_METHOD'] = 'GET' - message, status = process_pin_reset_requests(env=env, phone_number=create_pin_blocked_user.phone_number) - assert message == '{"status": "LOCKED"}' - assert status == '200 OK' - - env['REQUEST_METHOD'] = 'GET' - message, status = process_pin_reset_requests(env=env, phone_number=create_pin_blocked_user.phone_number) - assert message == '{"status": "LOCKED"}' - assert status == '200 OK' - - env['REQUEST_METHOD'] = 'PUT' - message, status = process_pin_reset_requests(env=env, phone_number=create_pin_blocked_user.phone_number) - assert message == f'Pin reset for user {create_pin_blocked_user.phone_number} is successful!' - assert status == '200 OK' - assert create_pin_blocked_user.get_account_status() == 'RESET' - - -def test_process_locked_accounts_requests(create_locked_accounts, valid_locked_accounts_env): - - response, message = process_locked_accounts_requests(env=valid_locked_accounts_env) - - assert message == '200 OK' - locked_account_addresses = json.loads(response) - assert len(locked_account_addresses) == 10 - - # check that blockchain addresses are ordered by most recently accessed - user_1 = Account.session.query(Account).filter_by(blockchain_address=locked_account_addresses[2]).first() - user_2 = Account.session.query(Account).filter_by(blockchain_address=locked_account_addresses[7]).first() - - assert user_1.updated > user_2.updated - diff --git a/apps/cic-ussd/tests/cic_ussd/test_transactions.py b/apps/cic-ussd/tests/cic_ussd/test_transactions.py deleted file mode 100644 index b16264be..00000000 --- a/apps/cic-ussd/tests/cic_ussd/test_transactions.py +++ /dev/null @@ -1,35 +0,0 @@ -# standard imports - -# third-party imports -import pytest - -# local imports -from cic_ussd.chain import Chain -from cic_ussd.transactions import OutgoingTransactionProcessor, truncate - - -def test_outgoing_transaction_processor(load_config, - create_valid_tx_recipient, - create_valid_tx_sender, - mock_outgoing_transactions): - chain_str = Chain.spec.__str__() - outgoing_tx_processor = OutgoingTransactionProcessor( - chain_str=chain_str, - from_address=create_valid_tx_sender.blockchain_address, - to_address=create_valid_tx_recipient.blockchain_address - ) - - outgoing_tx_processor.process_outgoing_transfer_transaction( - amount=120, - token_symbol='SRF' - ) - assert mock_outgoing_transactions[0].get('amount') == 120.0 - assert mock_outgoing_transactions[0].get('token_symbol') == 'SRF' - - -@pytest.mark.parametrize("decimals, value, expected_result",[ - (3, 1234.32875, 1234.328), - (2, 98.998, 98.99) -]) -def test_truncate(decimals, value, expected_result): - assert truncate(value=value, decimals=decimals).__float__() == expected_result diff --git a/apps/cic-ussd/tests/cic_ussd/test_translation.py b/apps/cic-ussd/tests/cic_ussd/test_translation.py index 69a56ec8..a6448dbb 100644 --- a/apps/cic-ussd/tests/cic_ussd/test_translation.py +++ b/apps/cic-ussd/tests/cic_ussd/test_translation.py @@ -1,6 +1,12 @@ +# standard imports + +# external imports + # local imports from cic_ussd.translation import translation_for +# tests imports + def test_translation_for(set_locale_files): english_translation = translation_for( diff --git a/apps/cic-ussd/tests/cic_ussd/test_validator.py b/apps/cic-ussd/tests/cic_ussd/test_validator.py index c90e56db..b6702276 100644 --- a/apps/cic-ussd/tests/cic_ussd/test_validator.py +++ b/apps/cic-ussd/tests/cic_ussd/test_validator.py @@ -1,14 +1,14 @@ -# third party imports +# standard imports + +# external imports import pytest # local imports from cic_ussd.validator import (check_ip, check_request_content_length, - check_service_code, - check_known_user, check_request_method, - validate_phone_number, - validate_response_type) + is_valid_response, + validate_phone_number) def test_check_ip(load_config, uwsgi_env): @@ -19,15 +19,6 @@ def test_check_request_content_length(load_config, uwsgi_env): assert check_request_content_length(config=load_config, env=uwsgi_env) is True -def test_check_service_code(load_config): - assert check_service_code(code='*483*46#', config=load_config) is True - - -def test_check_known_user(create_pending_user): - user = create_pending_user - assert check_known_user(phone=user.phone_number) is True - - def test_check_request_method(uwsgi_env): assert check_request_method(env=uwsgi_env) is True @@ -49,5 +40,5 @@ def test_validate_phone_number(phone, expected_value): ('BIO testing', False) ]) def test_validate_response_type(response, expected_value): - assert validate_response_type(response) is expected_value + assert is_valid_response(response) is expected_value diff --git a/apps/cic-ussd/tests/conftest.py b/apps/cic-ussd/tests/conftest.py index fa9389e3..8af8ec1f 100644 --- a/apps/cic-ussd/tests/conftest.py +++ b/apps/cic-ussd/tests/conftest.py @@ -1,15 +1,18 @@ -# third-party imports +# standard imports +from logging import config + +# external imports from cic_types.pytest import * - -# local imports -from tests.fixtures.config import * -from tests.fixtures.db import * -from tests.fixtures.celery import * -from tests.fixtures.integration import * -from tests.fixtures.user import * -from tests.fixtures.ussd_session import * -from tests.fixtures.redis import * -from tests.fixtures.callback import * -from tests.fixtures.requests import * -from tests.fixtures.mocks import * +# test imports +from .fixtures.account import * +from .fixtures.config import * +from .fixtures.db import * +from .fixtures.cache import * +from .fixtures.integration import * +from .fixtures.metadata import * +from .fixtures.patches.account import * +from .fixtures.tasker import * +from .fixtures.transaction import * +from .fixtures.ussd_session import * +from .fixtures.util import * diff --git a/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc b/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc new file mode 100644 index 00000000..3b2fd80c --- /dev/null +++ b/apps/cic-ussd/tests/data/pgp/privatekeys_meta.asc @@ -0,0 +1,241 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lQWGBF+hSOgBDACpkPQEjADjnQtjmAsdPYpx5N+OMJBYj1DAoIYsDtV6vbcBJQt9 +4Om3xl7RBhv9m2oLgzPsiRwjCEFRWyNSu0BUp5CFjcXfm0S4K2egx4erFnTnSSC9 +S6tmVNrVNEXvScE6sKAnmJ7JNX1ExJuEiWPbUDRWJ1hoI9+AR+8EONeJRLo/j0Np ++S4IFDn0PsxdT+SB0GY0z2cEgjvjoPr4lW9IAb8Ft9TDYp+mOzejn1Fg7CuIrlBR +SAv+sj7bVQw15dh1SpbwtS5xxubCa8ExEGI4ByXmeXdR0KZJ+EA5ksO0iSsQ/6ip +SOdSg+i0niOClFNm1P/OhbUsYAxCUfiX654FMn2zoxVBEjJ3e7l0pH7ktodaxEct +PofQLBA9LSDUIejqJsU0npw/DHDD2uvxG+/A6lgV9L8ETlvgp8RzeOCf2bHuiKYY +z87txvkFwsXgU1+TZxbk+mtCBbngsVPLNarY/KGkVJL+yhcHRD0Pl4wXUd6auQuY +6vQ9AuKiCT1We2sAEQEAAf4HAwK2fexgxtQ8CfgdeIlzdeY9K+HZL18brETddoya +3BeC1MSH7gxXqtCVQ5qdBk27J4wlGl0H83kYSCeVQs6hmrSrv8JCErguIdpZIJ/D +kcjGlGrOELfnXeif0VfUZN3LWxJZizCIS8I9F8VKD9c57nZEcbWcKTLizV0j1BeT +sdrumt/3UDhpCJTj1q3biRsiUbpmX+jPlRWN3OeSZJaRRyy4FnzTs3bndBYmkOsk +ZNKRk7jRNEU/LItbABStuP2zDrZsampVntKcNRXBVE2170t4T/Q4Gc0ckz4ohprY +lGykE2DdwapCdcKWccVXhM+svDwoLf+g4kjKuCE7R11v6rZlRxYrfquZXwtUx0DB +17x+JqyBaacyWm3Vq65DcNyiQw2NqCPdJU+iZoOGaermKIz3BqwxY+WE0HyjxQkH +P5KUpKQTmsTIlwHWFOVDYMRUUvD7P7XiElOBECDb3bJL9+z2SHZWTE6OaZKnmBFf +ZTdXgtGe/Ctx97PgWZOwM500Q45QC+D59NCYtXRtqi9WCLGsZpQbSZmojIUOJRuD +s5un+8lA3T0BhlJS4DC9CgN8Lxs4kT/XV/LYiXU4Z8MWEahurEbpDwH6YNzGktUR +zuE9HOe0fesdrOV0Sk2aol5CCRj3vTcsROTFUcT6UYPq28vy0U3zUJVvyNa0swk7 +PUiB+xhCi24Z9dy+0F1q1L20tJ/YCjC/tyLI36Rkl85PnoviwOOOll0+/claf8BV +e9x43voYe0o8Q7ttU0aFxVH/lGaTRyVMcXJFw0EPLuwIrcGrcauatcbO7lI2nVww +kBZFepWh7JBRl2x5SXnvTqLnWp2D5w1viUPcBN5xAj9IKOWrRr2kIRLiOVIGh9ta +Hiio2+vg/ZmhsmMzA36xYkH6NvyjNAeLUgTVfEAtsCrRXdW8FYTTGOKDmw55Ma+P +Ej1QWWzbwqPU+h+AOyklVZ1xGncxTkyad5niXYEzBJbbA01QoAtZeY7kSg0ae6uD +YPRQGf+0G6YlCKPOZjBH8AvbedhyjIKZhBT8M2sHIKSESPP0Vs8yS16rYzy8o6+e +7uYsIST+PMWXxDpJHmN2Ks5uo789+TiHfffHzbsTuevNIwk9FbMA6gpDdtMCaFZX +abZxz6sxLv9MoWjIKR2vDZKHjK5DVlJv4V1De3gTsCmfQhhToPzNGGFEI00aBki6 +IJIyisOuZtQiXhHy1vN499evLDwkc8u1S6ex6Q7blp75IQmJJ4/WG+XA55D+Mfnd +QSbV+zP9WQu66RR+RDsx+c7L7Bg58bqXE3bPcoLzaHOmDwpw74BGmNu84dfmyKbI +FocSAWP+Oe3sBxcdE7aVS+FB+B30It25LbQeTWVyIE1hbiA8bWVybWFuQGdyZXlz +a3VsbC5jb20+iQHUBBMBCAA+FiEE8/r2aOgu9RJNUYe67yb0aCND9pIFAl+hSOgC +GwMFCQPCZwAFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ7yb0aCND9pLwiwwA +hFJbAyUK05TJKfDz81757N472STtB8sfr0auwmRr8Zs1utHRVM0b/jkjTuo4uJNr +7YVVKTKgE7+rJ+pwhm3wlTQ44LVLjByWAi/7NWg3E9b2elm+qkfgm/RfFt3vkuOx +GSyZyIFFh+/twv6iABPvr6w7MZwrFaS0UP3g1VGa5TFqg6KNxod9H/gPLxv45lut +Xf3VvBZTJpr1pxn7aLHlFzEyIgNZbP/N1QF44GSrN/k0DfL631sZjauUXaZXbi5x +GsKKCYwJ1g3q587pi6mTdTV3n0hKgVuipO8hGy5++YeOv+hXsCxDwyZ+Shv+qavd +/SapxYgCdEueuwONIFfsIsWCd3SCcjKXicTTEFMu8nvBmf7xuo2hv6vEOxoijlXV ++4LkGrskdB8ZMg8PywEx6DLmDokgnAhTLrTc1ShbkOtQ3yNjjyFK7BDpqobsJal6 +d8SpbhccUJLepaSmsk0CgJsTjhAl6EwX0EYgTo3kP5fScqrbD8VwQaT8CcE4rCV4 +nQWGBF+hSOgBDADHtpTT1k4x+6FN5OeURpKAaIsoPHghkJ2lb6yWmESCa+DaR6GX +AKlbd0L9UMcXLqnaCn4SpZvbf8hP4fJRgWdRl5uVN/rmyVbZLUVjM8NcVdFRIrTs +Nyu4mLBmydc3iA/90sCTEOj9e7DSvxLmmLFjpwM5xXLd6z0l6+9G+woNmARXVS3V +/RryFntyKC3ATCqVlJoQBG45Tj2gMIunpadTJXWmdioooeGW3sLeUv5MM98mSB4S +jKRlJqGPNjx5lO6MmJbZeXZ/L/aO6EsXUQD2h82Wphll4rpGYWPiHTCYqZYiqNYr +6E3xUpzcvWVp3uCYVJWP6Ds117p7BoyKVz00yxC9ledF3eppktZWqFVowCMihQE3 +676L3DDTZsnJf1/8xKUh5U2Mj3lBvjlvCECKi00qo8b1mn/OklQjJ5T4WzTrH6X+ +/zpez8ZkmtcOayHdUKD/64roZ9dXbXG/hp5A+UWj8oSVYKg2QNAwAnZ+aiZ2KVRE +/Y61DCgFg6Ccx/cAEQEAAf4HAwLvYCWT4e84+PjE5pF2+FQAEMmVwTUm5pv9XhBd +Lnw68o0N/OGhi8LLMuhiI22u60W+//6Pknws1FfHI6zVeHZ1V4DcE8JtJcbSqGk4 +X1IFSXB60kduyCDLxq7PgqlLac2vr8jOsZAGTM8okJ3jrCrXd0oEPMIPQzo4RKZJ +PeBwUyzTU1+jA5pZjpj+DgpBoC5uZTeGLB2ftbN/w3wBUsZZR3q7WiM7p34+xvST +Obe1u5PerN5BH6zizvCWr2yRGF0RdUYz6q0kQdUorDjqrowYlNi5Em3RIyK1IoFR +MpcZPf9zMODMPZ2VlBruDQu40thr/Ho/5w15QmJ/7SmstGreKerI2jUziHPa4XMo +pUS+jGpIC3pZRa2Y+4UpgtYciuc5CusxzAOYbSh+py1kLuL/tkI54QsLYG2gDcd5 +dGz/jxun4irlZ/Iy1GtGM5+SrREktwRD2lIou295XqWOHwJPahPG7xb172VeUfoK +AObWonSJ9uWcsG/FKNo1at9ENA1x+zUV6s+F8B78snQJ96iFIHtz+5NAXQR0pEnD +i7DIHSSGaeZdj2NcbmM6t5/dyN40KHwymYxrItHGL19uRUJiJfgGeI9+dNCRfMOU +4YK+/kiGqH4Yr4WNBmF8zeP51gWDCspCzMKp+Z3wtGXx7j+147iWqW/6ARZ5krJa +oWF+gmesFYFWz54Lr/IuA4usaRSbt+ZnXpJTQip74NOrKF7JpXeVMWY7BN5wcnyO +SXrJrg3xKupq/oZlHnpGiL/UGrr9NZmT/ajg1xjVArkWD0YkwnTRP+CBXLNyrhtd +eLzClaDiv8wXMIm1uWImX7zVv+H7ngfU2aQOMQiU1BbV+pU69bAVdD73glniID1R +HYJHFhOxyF9nFTfBkPM/3rNuJDURLyMhkIyZ3OhIOiDv+5W2Q1swhlfLI5Tf7eCv +wxMGBM508I7TuemCuUk0oqsDnm1Z+oCEWqEI06qvMpGPPO9HU90kELdGDVlnVo6J +wP9UOgXa9LsywaFO+otV/spEpntQXXmHgzLgESyCxe0iHSSv9GxBLk1lTTCgi2qW +B8KI60TJiK3+jTiBR422XMQs5mkvDqOBLuX4dpOuosewPwAEfrl9ZF6z1f2TVVwk +piuHzNcz0NaLWkIrfDb2wIEPEzdCU+pVSfrh3g4S8dMiAK0IMWTYvye5xZ1bd9tN +vwI7ottJiJDk97ScnBU6b//Pb8QbQjjtXbssrfkBaJH/e0cE2WGkUzIQd6sJ8qnq +7mofMB7zU9iD0C3B5BCSnh36vKtGecosrpUmRNfGm79DattdQqAzZSY8rBHvJ+22 +KWF1VcqZVxYk5B33jc0p7tXjix2xyMc9IYkBvAQYAQgAJhYhBPP69mjoLvUSTVGH +uu8m9GgjQ/aSBQJfoUjoAhsMBQkDwmcAAAoJEO8m9GgjQ/aSIPcL/3jqL2A2SmC+ +s0BO4vMPEfCpa2gZ/vo1azzjUieZu5WhIxb5ik0V6T75EW5F0OeZj9qXI06gW+IM +8+C6ImUgaR3l47UjBiBPq+uKO9QuT/nOtbSs2dXoTNCLMQN7MlrdUBix+lnqZZGS +Dgh6n/uVyAYw8Sh4c3/3thHUiR7xzVKGxAKDT8LoVjhHshTzYuQq8MqlfvwVI4eE +SLaryQ+Y+j5+VLDzSLgPAnnIqF/ui2JQjefJxm/VLoYNaPAGdqoz/u/R0Tmz94bZ +UfLjgQaDoUpnxYywK2JGlf3mPZ3PNWjxJzuQTF5Ge5bz/TylnRYIyBT7KD7oaKHO +62fhDbYPJ4f94iZN4B6nnTAeP34zFDlkUbX4AHudXU7bvxT5OUk9x9c2tj7xwxQH +aEhq2+JsYW0EVw27RLhbymnBfLjVVUktNF0nQGvU2TEocw4pr2ZkDHQkSnlbNa4k +ujlL7VzbpnEgyOmi5er9GaIuVSVADovBu+pz/Ov1y/3jUe8hZ/KleZUFhgRfoUka +AQwA2r2HiLvpnclyZMoeck1LFoVyEU/CjPcYWF1B76ekO9mrlYvbKsnsyL0WcuEq +wCmHdLk70i743Fn21WQK4uvvlvrEpev9aj9DihyLctv4qrPm6wAU/Xibf75tg1iR +L+muMQfv6hQhjdhwkYFx/7XQ6UWkEibqFS7xJwrhz9lHL4KTA4sO5PeW713+mpz7 +tM5RmGV6NOQAyEEfAv6OawlWk0f5o8xngIoyo2BS5qIeEBO+iz45+GG8GQC6XufO +Ix7VVl++ZpsxZKtDq/AXfAskxfLRwZMqH9Db5pPMzrL1bPV16AwoWqhAGd2HIMkO +DLEC5XTGIKCqO5+n288rHhAJTqFmE7TpAo+Eb0Tkk4jfm6LyRonmQGpu/Zxa53n5 +D6d+AgYWAMeHkEthWJkES4mKpZu4nV21+n9mynnPg8wzthL705Q6IBjtlxX8EP6e +eRFE1BUCNp2RZttTSdI+8iwzYsGOJdJeeXeLOGhvU9/PLkRj9jgZLgCLAo1QGo2o +xetZABEBAAH+BwMCYxRGMNwlr/T4SMsvXNo05Y9gvmJ/vNY89nIF3J/WsBcBChWT +MAls+3BDxHbEjjXb4sWQeGE5IxNUv1TMjZ1CLDAzga5Rm/KICYl3Yo6hWKRWk6qx +fdacQ8Z5aHXtQQ8qJxX2dIPbZtTkmhlCIj3B1H7xThFF/b+oh1+hV8F6kWuKZ2jJ +3cm+hy/sBpnENU0EOMvDAcQZ5QikmyyYPe03MMMEhl4Q51NbwFZi1Fnb1qYieGdh +lRX+92/+V5okFj0zTKLTtglwBcYobAs8Vlwa0bC9Bw5U21U1b4uU0wVrOHsdnGZp +LLZFXxON1t8ZSdNixus2kuUZDuCX2xGKestufSL+6rgf2pQAcoHI61uwwQT1LZGf +wmAieWHy6v+KWBODmTO6P6a3w1mCfI9gVATfWSuhbuIbqgUMLBWsimUB0pdWTwX9 +oVKMS+OxL+ZHPoaixFwkFz6GqCJVRJ+rKafmjgOfmCWwCl3VoqGp/fkZKKgrBprP +HB1aIUkiiOvgPOW3ZbjG5SwBFSdjKt+KiWVEAVKnl9XAtzB+SS4fk2aKvezNB3Yf +LW6wmq4U+OkXEfGLpk1KJ81wb/D/ULAI3FRauxB5drlTwJ2mrWqeuJsR4A2sy49t +LKapWlbDlOvFtXtynultB/mc8mhphiaJdMoSKuOspiSSXNk/On9UdSVn0tDDlZEh +QU6iYwtvATo3Q1/RWZI74V4IZqt8R1d0Y+HIn8SNfUp3Zcs5vcqb+YvUGzqveLnl +Dn6ndCrLq7spDAFk9WVObApFYtnuEt9pKmrluQczckXwb7yH6CFCgoF+DjMdYi+L +En869iCp+jW2SVjo0q6SOODrIB2aiIEW8PoRIC/vFSTVgv528s7A6DjXeh0c/hkb +Ud3b5KNCJosz3RArv7ljiYq58Kj4scFr45orj80XulsLbr+tFaN3VNKgEsBDp0ZD +wgISoJr6fzAttqTKsPdzHGh3lNY5RNuP4r3VTgu3dN2ZxIDXxhiIWhbWiXmBz2p0 +Y+TRwtgoUluDnMJhFDx8m1w07AqrLT7ivISgHrHwcDZgDGZ8l6rviDk3b8AsKtqY +r//yTXMpTC0kgEb89oHqRd2NiCS4R+2bjWZG+2CtQ7TpCYscbdNdYucEhQGiAUMk +7MJISwC0VSw3xesuHcF8Nx+5vY+GlTrZDIkrS0qKkmOvwSWP0xtSWa1jvIvsd4UK +yoHgDCdvME9UBeIrfqa9JfKAPFE1iGN3uXmq04hwnWwu/vybFA6IjeA2tfbFWWaO +oh2YyXDqhuL8HbUMESiyPOybFXm3aw6HRgIr3OM/R4O6Hv02zNeWJXnkATTKgTje +1xkJuQNXY5N6bpBPkw01Kr20IkJlYXN0IE1hbiA8YmVhc3RtYW5AZ3JleXNrdWxs +LmNvbT6JAdQEEwEIAD4WIQT2ReBH7lvE4oJMlNtC3JHPqKugKwUCX6FJGgIbAwUJ +A8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBC3JHPqKugK25hC/9VF1fe +kj0IKnrOJRUcK/Cv4RBowl60V91w27ApsoP2awEJiFhY7qRijtkA3NKrT3tke7aT +nC3yAJ8SFOmvIAC94ijb7Iv97xkG+1IIz8pvru9y+dzd2NnvCkts8gFF0CI/xtEM +E90rU3Pay9B5IyrpP++UdmSmnp3Neuwi94BZDfMlqkeiYOzWWSeYbmSSVfKTXeBd +UuTyfRI4m/bPbh6gegOB/XdgSIrNY74D0nR3np0I+s0IGZepK24kgBKfUPwRDk7f +98PXCh29iL3xH+TBxu30WHq7xKmPoXxCRyFLtnKF0MN5Ib276fHnJZM+hXf5i/1E +Pi4NLnk86e7fNI69hwiUd1msEt3VmZWe7anJe/1p3sSXwbQGhhGWM5K41/rQ1CZ9 +qD95d6wkHRSc0n4z78qxgYV73yJHinN8xIFnPWbopPPIJbELSoM3IEpHobsj95pH +4hzZAPSmDfOfLzV1G2ec1QPfWnTqUriUt7edDs4//7Cczj6sRh2B6ax2diCdBYYE +X6FJGgEMAMqxn5io6fWKnMz8h5THqp4gEzDuoImapfMKbAKcxEtJmcLkvn+4ufEP +/hcll66InqJHsqMOrdb+zbduCruYWpizhqCIGSsuRu7+ZEEkQFmF5juCOV/5qKQJ +gZZmxSKbRtboapMRR/jmg1pvhnUG7wJOGWi7qv+iRdsWKskDO7tUQE34+ID7IwfD +Ze2fbFKxf66nPlUunF8aMglsvGmtCEzm/xwjunHnmoqZBQIzTdEXIaEwhVosbgY7 +A1iwOJ/gT2dcF2KJa7tygrtcbgdVzYCibynwtlvDGXukweuYLQFsObyBG3UHRhJg +61p7n344sy1U9uwCP3/pVCr9bNY9mLZpCgHFkqxErmB8cWouQkbwnqxQFm21KtGF +zjUawuKBXVtDEeA8C5Ha0sx7lw5JrX8GD3EL60qKWjqujJsR1kyijXx1No7Xr9NW +WuPoIDYH06ZoYE+j065VTRqZIGr3NjUZnqT7s9M41roQMnKAzRBXousRXRW9dXfS +5YIG4nWTlwARAQAB/gcDAsDkrCv+8rcr+OKtXIf6oDyx2tbPr+tpZJII4Lqchego +FTB0/GoqHF+iu+uYDCuzkwXBSIAPTCudjhZ+0cwvO4WgjdqGC3zqCc4bCP68cItN +fcLsof5L7rJ8BXX/0YXhua3gFtWGw/EtGpO4tqFCrzkpgEvovP/N1CLFaHnRzWSN +AE0ebsdfTCRYjWuZiAKlWjKCMNmHrE7AB5TraGqclP5GlY28lm7T9KXnNXixFaaR +pLDaLFyGZDEilEjkCKx1cyg3oBNeqUP/Ra6DYEF3PWTGpX8PxBF4lA2qnq+XuUK8 +30Nz2upz38Hb1jG1sdNlYEWLv05bFc0vMLWmzwAd6Ij0I4C6WdsakT213frlFw4w ++hoilBcrW5+UOBc1dbU3UFh72khzLdKz1aUVC2N5HN4gS7WTSw3of0sOy+LR4JaH +O8kSlZAIMXCooBDKr/R6x97A5sq8zMQ0vI0LSN2FpfwgdApwWLJYFBAy7ZJU9efO +0f79yEqk7d9xFEsIIpn2R+zVcUdAvj9/EDnbu/QaEj9jl+2PH6arqp1AGurF0Dp/ +cB4T7ZCuaunIet5MqEN6Ac4WdjpEcC7tKjB4cQ53Y2f9zCSouXa4JUypsgm4AQZX +O6hejChIiFC31T2x0a3M6eD6+XNw64ShdyX6i153xOe07d78Zq5qnhw+Vz28FQjZ +Lmvbm1sj26WaZmLH6LzyjAJjjV4YH7ijwLjUMdeKeuato0fsCff/cVO7MKDj0aPe +zQCWSbqcnsxl1Agsop82k6Y3W9gco0tVhqYgIwmCjsWvCXAWILk2yIuxIxINRNqX +Pt+TYZR0BuDAUK4x15fUT5tXuu7DmmnmZlWlaba44dtJPB3SFtEM7jJ7xNF0zie3 +G+6hxtZSmrjfYHBK2ZD+2veP1j0P/tYD/8n/rZx7u4pHuJiiZksj6ZwGF61HP03Y +zOu0LhhtrTQ1yYaE52mUdkLlQRNwD5b+qDkqN9/PeSzIoDesRVuVE2rbr7sIJ3GS +jxZHin4kHsxzqFQmecm1ctPgx+BpksMPok+MJGzSZ3OwE5tuK0VLvEIcMritgfM7 +BixcNPyDv+RkwEguSMlsHq7Lrr+LXtR2XmeFBMgDiulZ5FUxoGiGBfyyG3bsAug9 +D4ceg0yUh5HjTTjqoQ1Qo95Wi/yF64WPcZDllJ2BaBznDxlESNR/jmVhwQsrqOdn +NF54SCpU0e3N+XBsdHd4cRsS2lxemn5boK67/FVKdUmVgxo9VnAlreoeB+cVFHap +gfPSXD5V/MuFMiKSFuF63s61EP1T1Okl1cvrE2oAsJTXMgCIVSZdyLUwYKmjsMVD +rtfQXcoOQgFbA90qj7uOOtZId04NiQG8BBgBCAAmFiEE9kXgR+5bxOKCTJTbQtyR +z6iroCsFAl+hSRoCGwwFCQPCZwAACgkQQtyRz6iroCt8igwAgopqy+UgxJ7oTL2z +vOgL1ez7bv+E/U1/7Rdy5MHwr4WF6oZRpIBlgv3GXXeIFH9bFdDhgyPKgh+Tz24J +BL+7YjUtWGe/G/pmmNK1YazB/OxrwiGFpTCyk1zhxEkhMu7Hu3LgD571K+4TUUpa +PCqEeoBBg6O3T29DH1AxpWpEPGXlOrRDHYgVziEpLdUNahAjF53auNWvya+Vc2qZ +wM4NFt608LLf7J5yIA2vbsvf6+gVopPE3whXESKXo08B2hC1f3Pr9/Tgt6oIvy9/ +dAcTMalxRyyc42E2wX5kyzDlfhY9kqaNNfaGMZJO5g//gB7BdtrAfo/LhWtary/Y +fAOtbbnMYkf+HODAPZItaIjMZngBM0c0m78YoCetAQE8uBFK6aXmht3BZGPOwgyZ +pK5QT6ClYst2N9ca3tPUEfnddotKySmCEk/JWtu5/0lFl75WzHulc7iUNGJmnUff +VZyH12CjBWsTtqombHDkdEKFocavqpVcCCbKbtW5GZhuZC65lQWGBF+hSUIBDADS +tlWquV7SdREZtxXBVVzdCkV1xkeHYfo2Z244W0LTwmvpbO+o6P5GCAW2c336qWEl +sMO9ujeV2nuUZy3k3AtJLx19iWC+ywYVzJ8f878XAxq0ya1VBBnfsBc7iRI3umf2 +JSi+fHXf9l+rJ8Zr5AkLrUo3tQoxX8xWQIfUVY481nlkOvuMtxEI6h1t+z7PWjAJ +sdKKdevRPApPIBGXX0iGE/98ATsLYtvh9ln26j1SrSdtKpPktuYve3zkphlZAdf5 +ReViicik6gpEdyEfIxNab6nyV8LTbSeCHe+6/cz+AEqA+cr3K3MwriaapPzNhRV8 +izzGnIWChIZptGBKH5nLivfIAB/hbOgU6tM+YgUKrpJCXXA1My2q68o2kARJxh6s +0tuuT6pFEAG9RmzS3ywrPz4PAgkwrJA1uUa9fy9ngkOnQN3CEeVQTUU55b+6zVhW +1Qq8PII6AGqj1lSY9jLpjxEr3q227OlTaxfgg19x5o9rcyccAZlQqzL2p3Z7HZ0A +EQEAAf4HAwKyxiOcJwMkLPhbpalG07ErjqLt73SKDP3Qv5zzkUnBcqE0TbyFtFlp +HFf/Lv60X1m1OBgP4htz+JfikL5XVbWiGEBWvWPJP6VBBLJm+vjENjfKXzrRpcR6 +zhpfmJXm3BSXSpRg746AVW5Tjt2Z+dG7leTL+bddgu321OLYrpghyOUblKnRJZ0g +0+vLByFbLWlgtFs3VxPQJw6FmdN9+m748xeVbqzxXwEzScpBZhcGrjHUgnYL4/XY +PxzmZpUZ3qFW/P0uPZ8PdzI9MjEXaDhdxxOj/TP3cc2+XnrpBeWAGajMtulMvt+O +PA/jisn0ZViy7q1fNz+B3j/V++l3UxRAHnI5yaRY91pPlOmnaG4ScCP2o7NAUIiA +/Q3O13hIvVB+iIt9Y3p5WQSBbppHURVlhOOxDkSpXuxe5OhXqFYuDjGyx6hId4xL +b/IP4Gs29ZSG4+6nOZa7GWl4M21Zcw0AX1Gs0+6PPuqlIecW+6e28xxwFQjj7IKt +OvHq6zI810ReWdw9qVp3g9mzqI8x9KcFGdDvZmd4sA0R9GYR6UhvTKTIhdV4wrdO +w2oBe3CpmEnrggtsTrUFykAfjuYRS6aYUjRVv6rdeiWFKyQzBqqboLO9si2RkuNK +H8P2G6BdsLMax/kZKoXuuQ39xq/Li8NJjAoWEMz8iiZ2Io7MGPZobXssoA18Q3dn +tNRPM06cojXoDkXxc5jkQMwJUpuAaa59Zcsgp0sFv7/8nez9ejCaEBTqm1pkEQtd +b6178ld2T6q1jMb/tHWl8CjhH1sZVX2DdEk4SraIFdtGD5vUXo9SkI/QiY0RYwtY +t3tzNnlWMPAWmC+GaZ9QjmPYwEGCXvaGZ4rB2iRPQ+wAvHV6b49txRckLSGm56jb +8WMY7hSC0q3Bj4vFSx78Ytn/H2xwEh/XUiXe1rZhFRXxf7ocLoVS8xx+FL94kzaC +pLKTKoX1udNmYtebkO9llpkW/Z4KeQWJ73uSsjTZhMXVr3fJRanH2twjY8XodG+J +KXuERxMkM2sqnhecsAS8yCLndFLGSDSWyNIA7o6VAtCOpS4TKlYRNmv2XOaxrGgT +7y4hZQBJT8CwP8QuZl9R4WBtNQLPOn9LJCYtKtOznpb2v27BEyeAk4DlKgHJAfcr +kT2xBj/UoJsD72iJMh7SOhmWr7T+gbwAnDlhM1TmtMfWUCAG9Y3fz/metPlMHCKv +ttrgTfyLyvQHgiDTh3iqNEZ7rGK46on72/YYS+DXA3uSNckCaNQXupoIrxqpDjfA +WrrmoRqL4IWMxHzF1RVKAsjXWaYtpbvlMOSNtyMff29WM+MkZqG3IdbkokKJdJf4 +/+iQU+738VD43SdPurQcSGUgTWFuIDxoZW1hbkBncmV5c2t1bGwuY29tPokB1AQT +AQgAPhYhBIYPcR68MZb6cOhv9wDz8yhlQWZrBQJfoUlCAhsDBQkDwmcABQsJCAcC +BhUKCQgLAgQWAgMBAh4BAheAAAoJEADz8yhlQWZrD0YMAJp6WkrSzghIgrGmEquh +UPu4n8dnaGraGxu1Om9Z6HrUvphBvm/yZMlZxYbsQRvd8DUCuQD7fScBS12WX3AY +e001REfAbj0kDAdDQ0Z8sFCeCDSBJ9ulX07FzTHH0qROcSv6NONjGYVeTFicL2W0 +rATygnFzzjjSGboMq1qA8u6/5JNM7MAxJcIS0Dr8Fhdwv8TwTJrVg6ZzJDHN8OVA +UkPaciQI5lDDP5+kOVqbZZ92Ua8byxKtNACCdSsWZr2OvYyjUz4JKMp5X6yHbDQB +3vlwRkRS7Voo3pUGsdLwiBWiryklSa++DIbBemrALFLc5YnLgfCV0frPOEqsdDwW +ECRxwN4r+2DjY6TYCEEDfhM2Hm7MoMx/jM4uhI4KwPdOKmHsBPVBeXqBRXz32NMM +Zg6to0HRjDapR8AkbfdC5vjiuwnDA6llmxnVtx2oPX3g8RVOIw65f8KfWzWSfzEq +hoKTccsHMMza8J1ax6T6HXkqa/Tt/B/3d7nUzp53V3luG50FhgRfoUlCAQwA4rFx +mKwr4RAoVEqhDDWl8ecd/KQXEg0iCpkkmED6mEpPE9qAi8ORNId66E+rveS1Ssbm +bqVlrN9iHphtvYqvlwwb2IkgPaFpmVSqWrQ3yzEPrL5CLAWiiEq7M4ux7pueYKcO +mv3wQSta9eMgy9jaGUXrxFl4qotCevcEsLzkKC045OdVxkL++NFsiQUSfMYOtgGK +XuBh0ycI/pOb66lY186zPT0tR+QA18uzeCizEjhCZmPIlPHjN8NOEM7ZLU4UQrLd +Srm1quhO6DvGEoO5FulvGtp5hVHdJL5oB7svzNurXB3WVjdXCnRijoaCR07A/X9J +VZY2+kRxdl6ZkuLZxb5UE6usW7pTA5DKiuFG/w6CSGZA1Dv3+yoZnjN8KhnGmIWm +EJgvddWWoaJ3wFvSAGkYa3qBLX3noV3ZCm0c/r2LBcyFGyuyddEhg9wrqWU9vM7W +/4BkTqSJdeMRlS9FD803V9GqxAJBJ1KOSFt2s6b+ekYCI/d+Buso8GPp8eUHABEB +AAH+BwMCnL1QLv+DJ3P4dP2//f1cC7xTsDp9/ogeuz8gxIm6aWtNBhgWgRVgXnma +HsmQeEm7c70Vvt+Kjo9DbKUQbo32pc1Gwd8wvnNZUKtj+9E71hDd05f/SiA2ZTck +8AIRgRUV30Nj2qEgg0nFCWDNfMf0Lx7XH5APMJEZ2GXioiUdUInFlfXBvK6zv4wO +0jIyB/lRO5sCLcC8jNsNfe5oQVcoizziMxaAK91Fv93DeVa2hwqTK3VqBPXa/uyz +6iRMYe//nYIJCNllEsY8whKKfsskIOk1Dwofyuh2IYP6dv3SXhTj+l+qp1uqsg2r +JiThiyNXs0+zeVRxURBSZJrxMLHAs4tdcyckt0fCvM6bUCcDRo9+6w52GSMtb1w3 +08oJ+4YOLilJIR041x+Jzs1oTMhAWI5XH02x0mEFKADg/iSexOFSKIfT5RvFYFEj +Fpil+RalypUWzoxjaHFrSV9gxXpdys/qlHb4dr/nMTc/42x2d1xH6HTmlLtTp3rl +vM8/6tmeIhdTkfPtWIyrSfmi61ZCTJ21tKgDNj8r79lxkB6vqX+c86X/ug1tv3Ma +BN96f4QJOGHIhefImnStAw7OyQn3F9qnkj3x7u2f9f1XyDJO4T+WaQcYf67DEDy0 +KLpxwjEuT63BYIiWcQHli27lGOj8gAalTnDaWOWRckw7KAemL6cMGwZAHT91aHVH +IKd+dwl3gbArYJxWQ9Fc7lF0Nv4BfEghCOssrQuli9jKkhok61pQrx6L/ekkfeRt +mBjDtZOCO5NOTeeAZlc23TpZ/yjSBY/GPY0jXfnZ40Vm/Kl5VHW3Poc/rJDI67/5 +Zz+mL/sTOh2SRKUzsDGQqoQeq0ud1o8LQNf9m/3R+qxII3UsaRxKPDM4O2z7uLqG +v6DG9WVO/6nvoEMrItyh01BfU8l4zLkvXpkcrgbRT4D62w5BgoYpAfHprdqwxCDr +gRIiRKgNy2+kfxv6MVaTlmO8Fa9CR5wxeynx+YvtlIjVEF9SXQaXyb1g/zmnimn2 +kAjp/zdPQDLtZRW/cR6EEP6h8zW2jg3p+Owh9tVaZ1WoDfuelRMCuFFoiJ0RHXRQ +ocSzXfw7cB0YCpWR8Rrr0QlQYh/GEbQahTjjk+x0FXmEiCGkvOQeBFY2KUG/597g +maYHwRqfP2LjprG1mFgk0wUz6Juf86RZYD1XszIQPAL1CXf8kSuh49t7MRSgCiSo +qfMfZsMZgftjld5pD0lEqbpohHw/qpZdEklqUpkNUxJbBCWr9lPKirKadeLiXLKP +JI6Q0UEKkdw6lRLrg7UoDtr0vx/Izb3QB1jpKX7m1E/YZhTeVgYnLjrHCBjhJ8cE +kFmM7YC0iuh1TduJAbwEGAEIACYWIQSGD3EevDGW+nDob/cA8/MoZUFmawUCX6FJ +QgIbDAUJA8JnAAAKCRAA8/MoZUFma/gCC/9xkH8EF1Ka3TUa1kiBdcII4dyoX7gs +/dA/os0+fLb/iZZcG+bJZcKLma7DRiyDGXYc7nG3uPvho7/cOCUUg5P/EG5z0CDX +zLbmBrk2WlRnREmK/5NTcisCyezRMXHOxpya4pmExVMqSPGA0QbKGwdHqfbHQv2O +yI3PYBKvlN+eu6e5SEbT76AQijj5RSPcgbko24/sSqJylD1lnRocQK1p4XelosBr +aty4wzYSvQY9dRD4nafxPHI3YjKiAG0I7nJDQ0d1jDaW5FP0BkMvn51SmfGsuSg1 +s46h9JlGRZvS0enjBb1Ic9oBmHAWGQhlD1hvILlqIZOCdj8oWVjwmpZ7BK3/82wO +dVkUxy09IdIot+AIH+F/LA3KKgfDmjldyhXjI/HDrpmXwSUkJOBHebNLz5t1Edau +F+4DY5BHMsgtyyiYJBzRGT5pgrXMt4yCqZP+0jZwKt1Ech/Q6djIKjt+9wOGe9UB +1VrzRbOS5ymseDJcjejtMxuCOuSTN9R5KuQ= +=VqO+ +-----END PGP PRIVATE KEY BLOCK----- diff --git a/apps/cic-ussd/tests/fixtures/account.py b/apps/cic-ussd/tests/fixtures/account.py new file mode 100644 index 00000000..e2703877 --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/account.py @@ -0,0 +1,169 @@ +# standard imports +import json +import random + +# external accounts +import pytest +from chainlib.hash import strip_0x + +# local imports +from cic_ussd.account.chain import Chain +from cic_ussd.cache import cache_data, cache_data_key +from cic_ussd.db.enum import AccountStatus +from cic_ussd.db.models.account import Account + +# test imports +from tests.helpers.accounts import blockchain_address, phone_number + + +@pytest.fixture(scope='function') +def account_creation_data(task_uuid): + return { + 'phone_number': phone_number(), + 'sms_notification_sent': False, + 'status': 'PENDING', + 'task_uuid': task_uuid + } + + +@pytest.fixture(scope='function') +def activated_account(init_database, set_fernet_key): + account = Account(blockchain_address(), phone_number()) + account.create_password('0000') + account.activate_account() + init_database.add(account) + init_database.commit() + return account + + +@pytest.fixture(scope='function') +def balances(): + return [{ + 'address': blockchain_address(), + 'converters': [], + 'balance_network': 50000000, + 'balance_outgoing': 0, + 'balance_incoming': 0 + }] + + +@pytest.fixture(scope='function') +def cache_account_creation_data(init_cache, account_creation_data): + cache_data(account_creation_data.get('task_uuid'), json.dumps(account_creation_data)) + + +@pytest.fixture(scope='function') +def cache_balances(activated_account, balances, init_cache): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + balances = json.dumps(balances[0]) + key = cache_data_key(identifier, ':cic.balances') + cache_data(key, balances) + + +@pytest.fixture(scope='function') +def cache_default_token_data(default_token_data, init_cache, load_chain_spec): + chain_str = Chain.spec.__str__() + data = json.dumps(default_token_data) + key = cache_data_key(chain_str.encode('utf-8'), ':cic.default_token_data') + cache_data(key, data) + + +@pytest.fixture(scope='function') +def cache_person_metadata(activated_account, init_cache, person_metadata): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + person = json.dumps(person_metadata) + key = cache_data_key(identifier, ':cic.person') + cache_data(key, person) + + +@pytest.fixture(scope='function') +def cache_preferences(activated_account, init_cache, preferences): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + preferences = json.dumps(preferences) + key = cache_data_key(identifier, ':cic.preferences') + cache_data(key, preferences) + + +@pytest.fixture(scope='function') +def cache_statement(activated_account, init_cache, statement): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + preferences = json.dumps(statement) + key = cache_data_key(identifier, ':cic.statement') + cache_data(key, preferences) + + +@pytest.fixture(scope='function') +def custom_metadata(): + return {"tags": ["ussd", "individual"]} + + +@pytest.fixture(scope='function') +def default_token_data(token_symbol): + return { + 'symbol': token_symbol, + 'address': blockchain_address(), + 'name': 'Giftable', + 'decimals': 6 + } + + +@pytest.fixture(scope='function') +def locked_accounts_traffic(init_database, set_fernet_key): + for _ in range(20): + address = blockchain_address() + phone = phone_number() + account = Account(address, phone) + account.create_password(str(random.randint(1000, 9999))) + account.failed_pin_attempts = 3 + account.status = AccountStatus.LOCKED.value + init_database.add(account) + init_database.commit() + + +@pytest.fixture(scope='function') +def pending_account(init_database, set_fernet_key): + account = Account(blockchain_address(), phone_number()) + init_database.add(account) + init_database.commit() + return account + + +@pytest.fixture(scope='function') +def pin_blocked_account(init_database, set_fernet_key): + account = Account(blockchain_address(), phone_number()) + account.create_password('3333') + account.failed_pin_attempts = 3 + init_database.add(account) + init_database.commit() + return account + + +@pytest.fixture(scope='function') +def preferences(): + return { + 'preferred_language': random.sample(['en', 'sw'], 1)[0] + } + + +@pytest.fixture(scope='function') +def raw_person_metadata(): + return { + "date_of_birth": { + 'year': 1998 + }, + "family_name": "Snow", + "given_name": "Name", + "gender": 'Male', + "location": "Kangemi", + "products": "Mandazi" + } + + +@pytest.fixture(scope='function') +def valid_recipient(init_database, set_fernet_key): + account = Account(blockchain_address(), phone_number()) + account.create_password('2222') + account.activate_account() + init_database.add(account) + init_database.commit() + return account diff --git a/apps/cic-ussd/tests/fixtures/cache.py b/apps/cic-ussd/tests/fixtures/cache.py new file mode 100644 index 00000000..f522689d --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/cache.py @@ -0,0 +1,15 @@ +# standard imports + +# external imports +import pytest + +# local imports +from cic_ussd.cache import Cache +from cic_ussd.session.ussd_session import UssdSession + + +@pytest.fixture(scope='function') +def init_cache(redisdb): + Cache.store = redisdb + UssdSession.store = redisdb + return redisdb diff --git a/apps/cic-ussd/tests/fixtures/callback.py b/apps/cic-ussd/tests/fixtures/callback.py deleted file mode 100644 index 03e6016b..00000000 --- a/apps/cic-ussd/tests/fixtures/callback.py +++ /dev/null @@ -1,115 +0,0 @@ -# standard imports -import json - -# third party imports -import pytest - -# local imports -from cic_ussd.redis import InMemoryStore - - -@pytest.fixture(scope='function') -def account_creation_action_data(): - return { - 'phone_number': '+254712345678', - 'sms_notification_sent': False, - 'status': 'PENDING', - 'task_id': '31e85315-feee-4b6d-995e-223569082cc4' - } - - -@pytest.fixture(scope='function') -def set_account_creation_action_data(init_redis_cache, account_creation_action_data): - redis_cache = init_redis_cache - action_data = account_creation_action_data - task_id = action_data.get('task_id') - redis_cache.set(task_id, json.dumps(action_data)) - redis_cache.persist(task_id) - - -@pytest.fixture(scope='function') -def successful_incoming_token_gift_callback(): - return { - 'RESULT': { - 'hash': '0xb469fb2ebacc9574afb7b51d44e174fba7129fde71bf757fd39784363270832b', - 'sender': '0xd6204101012270Bf2558EDcFEd595938d1847bf0', - 'recipient': '0xFD9c5aD15C72C6F60f1a119A608931226674243f', - 'source_value': 1048576, - 'destination_value': 1048576, - 'source_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'destination_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'source_token_symbol': 'SRF', - 'destination_token_symbol': 'SRF', - 'source_token_decimals': 18, - 'destination_token_decimals': 18, - 'chain': 'Bloxberg:8996' - }, - 'PARAM': 'tokengift', - 'STATUS_CODE': 0, - } - - -@pytest.fixture(scope='function') -def successful_incoming_transfer_callback(): - return { - 'RESULT': { - 'hash': '0x8b0ed32533164d010afc46c0011fbcb58b0198e03c05b96e2791555746bd3606', - 'sender': '0xd6204101012270Bf2558EDcFEd595938d1847bf1', - 'recipient': '0xd6204101012270Bf2558EDcFEd595938d1847bf0', - 'source_value': 10000000000000000000000, - 'destination_value': 10000000000000000000000, - 'source_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'destination_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'source_token_symbol': 'SRF', - 'destination_token_symbol': 'SRF', - 'source_token_decimals': 18, - 'destination_token_decimals': 18, - 'chain': 'Bloxberg:8996' - }, - 'PARAM': 'transfer', - 'STATUS_CODE': 0 - } - - -@pytest.fixture(scope='function') -def incoming_transfer_callback_invalid_tx_status_code(): - return { - 'RESULT': { - 'hash': '0x8b0ed32533164d010afc46c0011fbcb58b0198e03c05b96e2791555746bd3606', - 'sender': '0xd6204101012270Bf2558EDcFEd595938d1847bf1', - 'recipient': '0xd6204101012270Bf2558EDcFEd595938d1847bf0', - 'source_value': 10000000000000000000000, - 'destination_value': 10000000000000000000000, - 'source_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'destination_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'source_token_symbol': 'SRF', - 'destination_token_symbol': 'SRF', - 'source_token_decimals': 18, - 'destination_token_decimals': 18, - 'chain': 'Bloxberg:8996' - }, - 'PARAM': 'transfer', - 'STATUS_CODE': 1 - } - - -@pytest.fixture(scope='function') -def incoming_transfer_callback_invalid_tx_param(): - return { - 'RESULT': { - 'hash': '0x8b0ed32533164d010afc46c0011fbcb58b0198e03c05b96e2791555746bd3606', - 'sender': '0xd6204101012270Bf2558EDcFEd595938d1847bf1', - 'recipient': '0xd6204101012270Bf2558EDcFEd595938d1847bf0', - 'source_value': 10000000000000000000000, - 'destination_value': 10000000000000000000000, - 'source_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'destination_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', - 'source_token_symbol': 'SRF', - 'destination_token_symbol': 'SRF', - 'source_token_decimals': 18, - 'destination_token_decimals': 18, - 'chain': 'Bloxberg:8996' - }, - 'PARAM': 'erroneousparam', - 'STATUS_CODE': 0 - } diff --git a/apps/cic-ussd/tests/fixtures/config.py b/apps/cic-ussd/tests/fixtures/config.py index 9dd18c37..ed33dd50 100644 --- a/apps/cic-ussd/tests/fixtures/config.py +++ b/apps/cic-ussd/tests/fixtures/config.py @@ -1,42 +1,28 @@ # standard imports -import i18n import logging import os -import tempfile # third party imports +import i18n import pytest from chainlib.chain import ChainSpec from confini import Config -from sqlalchemy import create_engine # local imports -from cic_ussd.chain import Chain -from cic_ussd.db import dsn_from_config +from cic_ussd.account.chain import Chain from cic_ussd.encoder import PasswordEncoder from cic_ussd.files.local_files import create_local_file_data_stores, json_file_parser from cic_ussd.menu.ussd_menu import UssdMenu -from cic_ussd.metadata import blockchain_address_to_metadata_pointer -from cic_ussd.metadata.signer import Signer -from cic_ussd.metadata.person import PersonMetadata +from cic_ussd.phone_number import E164Format, Support from cic_ussd.state_machine import UssdStateMachine +from cic_ussd.validator import validate_presence - -logg = logging.getLogger() +logg = logging.getLogger(__name__) fixtures_dir = os.path.dirname(__file__) root_directory = os.path.dirname(os.path.dirname(fixtures_dir)) -@pytest.fixture(scope='session') -def load_config(): - config_directory = os.path.join(root_directory, '.config/test') - config = Config(config_dir=config_directory) - config.process(set_as_current=True) - logg.debug('config loaded\n{}'.format(config)) - return config - - @pytest.fixture(scope='session') def alembic_config(): migrations_directory = os.path.join(root_directory, 'cic_ussd', 'db', 'migrations', 'default') @@ -47,22 +33,39 @@ def alembic_config(): } -@pytest.fixture(scope='session') -def alembic_engine(load_config): - data_source_name = dsn_from_config(load_config) - database_engine = create_engine(data_source_name) - return database_engine +@pytest.fixture(scope='function') +def init_state_machine(load_config): + UssdStateMachine.states = json_file_parser(filepath=load_config.get('MACHINE_STATES')) + UssdStateMachine.transitions = json_file_parser(filepath=load_config.get('MACHINE_TRANSITIONS')) @pytest.fixture(scope='function') -def set_fernet_key(load_config): - PasswordEncoder.set_key(load_config.get('APP_PASSWORD_PEPPER')) +def load_chain_spec(load_config): + chain_spec = ChainSpec( + common_name=load_config.get('CIC_COMMON_NAME'), + engine=load_config.get('CIC_ENGINE'), + network_id=load_config.get('CIC_NETWORK_ID') + ) + Chain.spec = chain_spec -@pytest.fixture -def set_locale_files(load_config): - i18n.load_path.append(load_config.get('APP_LOCALE_PATH')) - i18n.set('fallback', load_config.get('APP_LOCALE_FALLBACK')) +@pytest.fixture(scope='session') +def load_config(): + config_directory = os.path.join(root_directory, 'config/test') + config = Config(default_dir=config_directory) + config.process() + logg.debug('config loaded\n{}'.format(config)) + return config + + +@pytest.fixture(scope='function') +def load_e164_region(load_config): + E164Format.region = load_config.get('E164_REGION') + + +@pytest.fixture(scope='session') +def load_support_phone(load_config): + Support.phone_number = load_config.get('OFFICE_SUPPORT_PHONE') @pytest.fixture @@ -72,65 +75,12 @@ def load_ussd_menu(load_config): @pytest.fixture(scope='function') -def load_data_into_state_machine(load_config): - UssdStateMachine.states = json_file_parser(filepath=load_config.get('STATEMACHINE_STATES')) - UssdStateMachine.transitions = json_file_parser(filepath=load_config.get('STATEMACHINE_TRANSITIONS')) +def set_fernet_key(load_config): + PasswordEncoder.set_key(load_config.get('APP_PASSWORD_PEPPER')) -@pytest.fixture(scope='function') -def uwsgi_env(): - return { - 'REQUEST_METHOD': 'POST', - 'REQUEST_URI': '/', - 'PATH_INFO': '/', - 'QUERY_STRING': '', - 'SERVER_PROTOCOL': 'HTTP/1.1', - 'SCRIPT_NAME': '', - 'SERVER_NAME': 'mango-habanero', - 'SERVER_PORT': '9091', - 'UWSGI_ROUTER': 'http', - 'REMOTE_ADDR': '127.0.0.1', - 'REMOTE_PORT': '33515', - 'CONTENT_TYPE': 'application/json', - 'HTTP_USER_AGENT': 'PostmanRuntime/7.26.8', - 'HTTP_ACCEPT': '*/*', - 'HTTP_POSTMAN_TOKEN': 'c1f6eb29-8160-497f-a5a1-935d175e2eb7', - 'HTTP_HOST': '127.0.0.1:9091', - 'HTTP_ACCEPT_ENCODING': 'gzip, deflate, br', - 'HTTP_CONNECTION': 'keep-alive', - 'CONTENT_LENGTH': '102', - 'wsgi.version': (1, 0), - 'wsgi.run_once': False, - 'wsgi.multithread': False, - 'wsgi.multiprocess': False, - 'wsgi.url_scheme': 'http', - 'uwsgi.version': b'2.0.19.1', - 'uwsgi.node': b'mango-habanero' - } - - -@pytest.fixture(scope='function') -def setup_metadata_signer(load_config): - temp_dir = tempfile.mkdtemp(dir='/tmp') - logg.debug(f'Created temp dir: {temp_dir}') - Signer.gpg_path = temp_dir - Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE') - Signer.key_file_path = f"{load_config.get('PGP_KEYS_PATH')}{load_config.get('PGP_PRIVATE_KEYS')}" - - -@pytest.fixture(scope='function') -def define_metadata_pointer_url(load_config, create_activated_user): - identifier = blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address) - PersonMetadata.base_url = load_config.get('CIC_META_URL') - person_metadata_client = PersonMetadata(identifier=identifier) - return person_metadata_client.url - - -@pytest.fixture(scope='function') -def setup_chain_spec(load_config): - chain_spec = ChainSpec( - common_name=load_config.get('CIC_COMMON_NAME'), - engine=load_config.get('CIC_ENGINE'), - network_id=load_config.get('CIC_NETWORK_ID') - ) - Chain.spec = chain_spec +@pytest.fixture +def set_locale_files(load_config): + validate_presence(load_config.get('LOCALE_PATH')) + i18n.load_path.append(load_config.get('LOCALE_PATH')) + i18n.set('fallback', load_config.get('LOCALE_FALLBACK')) diff --git a/apps/cic-ussd/tests/fixtures/db.py b/apps/cic-ussd/tests/fixtures/db.py index 3bd84314..db017e18 100644 --- a/apps/cic-ussd/tests/fixtures/db.py +++ b/apps/cic-ussd/tests/fixtures/db.py @@ -1,4 +1,5 @@ # standard imports +import logging import os # third party imports @@ -8,17 +9,30 @@ from alembic.config import Config as AlembicConfig # local imports from cic_ussd.db import dsn_from_config -from cic_ussd.db.models.base import SessionBase -from tests.fixtures.config import root_directory +from cic_ussd.db.models.base import SessionBase, create_engine +from .config import root_directory + +logg = logging.getLogger(__name__) + + +@pytest.fixture(scope='session') +def alembic_engine(load_config): + data_source_name = dsn_from_config(load_config) + return create_engine(data_source_name) @pytest.fixture(scope='session') def database_engine(load_config): - data_source_name = dsn_from_config(load_config) - SessionBase.connect(data_source_name) - yield data_source_name if load_config.get('DATABASE_ENGINE') == 'sqlite': - os.unlink(load_config.get('DATABASE_NAME')) + try: + os.unlink(load_config.get('DATABASE_NAME')) + except FileNotFoundError: + pass + SessionBase.transactional = False + SessionBase.poolable = False + dsn = dsn_from_config(load_config) + SessionBase.connect(dsn, debug=load_config.get('DATABASE_DEBUG') is not None) + return dsn @pytest.fixture(scope='function') @@ -27,8 +41,9 @@ def init_database(load_config, database_engine): migrations_directory = os.path.join(db_directory, 'migrations', load_config.get('DATABASE_ENGINE')) if not os.path.isdir(migrations_directory): migrations_directory = os.path.join(db_directory, 'migrations', 'default') + logg.info(f'using migrations directory {migrations_directory}') - SessionBase.session = SessionBase.create_session() + session = SessionBase.create_session() alembic_config = AlembicConfig(os.path.join(migrations_directory, 'alembic.ini')) alembic_config.set_main_option('sqlalchemy.url', database_engine) @@ -37,8 +52,6 @@ def init_database(load_config, database_engine): alembic.command.downgrade(alembic_config, 'base') alembic.command.upgrade(alembic_config, 'head') - yield SessionBase.session - - SessionBase.session.commit() - SessionBase.session.close() - + yield session + session.commit() + session.close() diff --git a/apps/cic-ussd/tests/fixtures/metadata.py b/apps/cic-ussd/tests/fixtures/metadata.py new file mode 100644 index 00000000..d7925029 --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/metadata.py @@ -0,0 +1,53 @@ +# standard imports +import logging +import os +import tempfile + +# external imports +import pytest +from chainlib.hash import strip_0x +from cic_types.processor import generate_metadata_pointer + +# local imports +from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata +from cic_ussd.metadata.signer import Signer + +logg = logging.getLogger(__name__) + + +@pytest.fixture(scope='function') +def setup_metadata_signer(load_config): + temp_dir = tempfile.mkdtemp(dir='/tmp') + logg.debug(f'Created temp dir: {temp_dir}') + Signer.gpg_path = temp_dir + Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE') + Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS')) + + +@pytest.fixture(scope='function') +def setup_metadata_request_handler(load_config): + Metadata.base_url = load_config.get('CIC_META_URL') + + +@pytest.fixture(scope='function') +def account_phone_pointer(activated_account): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + return generate_metadata_pointer(identifier, ':cic.phone') + + +@pytest.fixture(scope='function') +def person_metadata_url(activated_account, setup_metadata_request_handler): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + return PersonMetadata(identifier).url + + +@pytest.fixture(scope='function') +def phone_pointer_url(activated_account, setup_metadata_request_handler): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + return PhonePointerMetadata(identifier).url + + +@pytest.fixture(scope='function') +def preferences_metadata_url(activated_account, setup_metadata_request_handler): + identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address)) + return PreferencesMetadata(identifier).url diff --git a/apps/cic-ussd/tests/fixtures/mocks.py b/apps/cic-ussd/tests/fixtures/mocks.py deleted file mode 100644 index 33eac01a..00000000 --- a/apps/cic-ussd/tests/fixtures/mocks.py +++ /dev/null @@ -1,95 +0,0 @@ -# standard imports -import json -from io import StringIO - -# third-party imports -import pytest - -# local imports -from cic_ussd.translation import translation_for -from cic_ussd.transactions import truncate - - -@pytest.fixture(scope='function') -def mock_meta_post_response(): - return { - 'name': 'cic', - 'version': '1', - 'ext': { - 'network': { - 'name': 'pgp', - 'version': '2' - }, - 'engine': { - 'name': 'automerge', - 'version': '0.14.1' - } - }, - 'payload': '["~#iL",[["~#iM",["ops",["^0",[["^1",["action","set","obj","00000000-0000-0000-0000-000000000000",' - '"key","id","value","7e2f58335a69ac82f9a965a8fc35403c8585ea601946d858ee97684a285bf857"]],["^1",' - '["action","set","obj","00000000-0000-0000-0000-000000000000","key","timestamp","value",' - '1613487781]], ' - '["^1",["action","set","obj","00000000-0000-0000-0000-000000000000","key","data","value",' - '"{\\"foo\\": ' - '\\"bar\\", \\"xyzzy\\": 42}"]]]],"actor","2b738a75-2aad-4ac8-ae8d-294a5ea4afad","seq",1,"deps",' - '["^1", ' - '[]],"message","Initialization","undoable",false]],["^1",["ops",["^0",[["^1",["action","makeMap",' - '"obj","a921a5ae-0554-497a-ac2e-4e829d8a12b6"]],["^1",["action","set","obj",' - '"a921a5ae-0554-497a-ac2e-4e829d8a12b6","key","digest","value","W10="]],["^1",["action","link",' - '"obj", ' - '"00000000-0000-0000-0000-000000000000","key","signature","value",' - '"a921a5ae-0554-497a-ac2e-4e829d8a12b6"]]]],"actor","2b738a75-2aad-4ac8-ae8d-294a5ea4afad","seq",2,' - '"deps",["^1",[]],"message","sign"]]]]', - 'digest': 'W10=' - } - - -@pytest.fixture(scope='function') -def mock_meta_get_response(): - return { - "foo": "bar", - "xyzzy": 42 - } - - -@pytest.fixture(scope='function') -def mock_notifier_api(mocker): - messages = [] - - def mock_sms_api(self, message: str, recipient: str): - pass - - def mock_send_sms_notification(self, key: str, phone_number: str, preferred_language: str, **kwargs): - message = translation_for(key=key, preferred_language=preferred_language, **kwargs) - messages.append({'message': message, 'recipient': phone_number}) - - mocker.patch('cic_notify.api.Api.sms', mock_sms_api) - mocker.patch('cic_ussd.notifications.Notifier.send_sms_notification', mock_send_sms_notification) - return messages - - -@pytest.fixture(scope='function') -def mock_outgoing_transactions(mocker): - transactions = [] - - def mock_process_outgoing_transfer_transaction(self, amount: int, token_symbol: str = 'SRF'): - transactions.append({ - 'amount': amount, - 'token_symbol': token_symbol - }) - - mocker.patch( - 'cic_ussd.transactions.OutgoingTransactionProcessor.process_outgoing_transfer_transaction', - mock_process_outgoing_transfer_transaction - ) - return transactions - - -@pytest.fixture(scope='function') -def mock_balance(mocker): - mocked_operational_balance = mocker.patch('cic_ussd.accounts.BalanceManager.get_balances') - - def _mock_operational_balance(balance: int): - mocked_operational_balance.return_value = truncate(value=balance, decimals=2) - - return _mock_operational_balance diff --git a/apps/cic-ussd/tests/fixtures/patches/__init__.py b/apps/cic-ussd/tests/fixtures/patches/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-ussd/tests/fixtures/patches/account.py b/apps/cic-ussd/tests/fixtures/patches/account.py new file mode 100644 index 00000000..687e192a --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/patches/account.py @@ -0,0 +1,104 @@ +# standard imports +import os + +# external imports +import pytest + +# local imports +from cic_ussd.translation import translation_for + +# test imports +from tests.helpers.accounts import blockchain_address + + +@pytest.fixture(scope='function') +def mock_account_creation_task_request(mocker, task_uuid): + def mock_request(self): + mocked_task_request = mocker.patch('celery.app.task.Task.request') + mocked_task_request.id = task_uuid + return mocked_task_request + mocker.patch('cic_eth.api.api_task.Api.create_account', mock_request) + + +@pytest.fixture(scope='function') +def mock_account_creation_task_result(mocker, task_uuid): + def task_result(self): + sync_res = mocker.patch('celery.result.AsyncResult') + sync_res.id = task_uuid + sync_res.get.return_value = blockchain_address() + return sync_res + mocker.patch('cic_eth.api.api_task.Api.create_account', task_result) + + +@pytest.fixture(scope='function') +def mock_async_balance_api_query(mocker): + query_args = {} + + def async_api_query(self, address: str, token_symbol: str): + query_args['address'] = address + query_args['token_symbol'] = token_symbol + mocker.patch('cic_eth.api.api_task.Api.balance', async_api_query) + return query_args + + +@pytest.fixture(scope='function') +def mock_notifier_api(mocker): + sms = {} + + def mock_sms_api(self, message: str, recipient: str): + pass + + def send_sms_notification(self, key: str, phone_number: str, preferred_language: str, **kwargs): + message = translation_for(key=key, preferred_language=preferred_language, **kwargs) + sms['message'] = message + sms['recipient'] = phone_number + + mocker.patch('cic_notify.api.Api.sms', mock_sms_api) + mocker.patch('cic_ussd.notifications.Notifier.send_sms_notification', send_sms_notification) + return sms + + +@pytest.fixture(scope='function') +def mock_sync_balance_api_query(balances, mocker, task_uuid): + def sync_api_query(self, address: str, token_symbol: str): + sync_res = mocker.patch('celery.result.AsyncResult') + sync_res.id = task_uuid + sync_res.get.return_value = balances + return sync_res + mocker.patch('cic_eth.api.api_task.Api.balance', sync_api_query) + + +@pytest.fixture(scope='function') +def mock_sync_default_token_api_query(default_token_data, mocker, task_uuid): + def mock_query(self): + sync_res = mocker.patch('celery.result.AsyncResult') + sync_res.id = task_uuid + sync_res.get.return_value = default_token_data + return sync_res + mocker.patch('cic_eth.api.api_task.Api.default_token', mock_query) + + +@pytest.fixture(scope='function') +def mock_transaction_list_query(mocker): + query_args = {} + + def mock_query(self, address: str, limit: int): + query_args['address'] = address + query_args['limit'] = limit + + mocker.patch('cic_eth.api.api_task.Api.list', mock_query) + return query_args + + +@pytest.fixture(scope='function') +def mock_transfer_api(mocker): + transfer_args = {} + + def mock_transfer(self, from_address: str, to_address: str, value: int, token_symbol: str): + transfer_args['from_address'] = from_address + transfer_args['to_address'] = to_address + transfer_args['value'] = value + transfer_args['token_symbol'] = token_symbol + + mocker.patch('cic_eth.api.api_task.Api.transfer', mock_transfer) + return transfer_args diff --git a/apps/cic-ussd/tests/fixtures/redis.py b/apps/cic-ussd/tests/fixtures/redis.py deleted file mode 100644 index b080da48..00000000 --- a/apps/cic-ussd/tests/fixtures/redis.py +++ /dev/null @@ -1,11 +0,0 @@ -# third party imports -import pytest - -# local imports -from cic_ussd.redis import InMemoryStore - - -@pytest.fixture(scope='function') -def init_redis_cache(redisdb): - InMemoryStore.cache = redisdb - return redisdb diff --git a/apps/cic-ussd/tests/fixtures/requests.py b/apps/cic-ussd/tests/fixtures/requests.py deleted file mode 100644 index a47a1057..00000000 --- a/apps/cic-ussd/tests/fixtures/requests.py +++ /dev/null @@ -1,20 +0,0 @@ -# third party imports -import pytest - - -@pytest.fixture(scope='function') -def valid_locked_accounts_env(uwsgi_env): - env = uwsgi_env - env['REQUEST_METHOD'] = 'GET' - env['PATH_INFO'] = '/accounts/locked/10/10' - - return env - - -@pytest.fixture(scope='function') -def get_request_with_params_env(uwsgi_env): - env = uwsgi_env - env['REQUEST_METHOD'] = 'GET' - env['REQUEST_URI'] = '/?phone=0700000000' - - return env diff --git a/apps/cic-ussd/tests/fixtures/celery.py b/apps/cic-ussd/tests/fixtures/tasker.py similarity index 76% rename from apps/cic-ussd/tests/fixtures/celery.py rename to apps/cic-ussd/tests/fixtures/tasker.py index 3ae99190..1a2341d7 100644 --- a/apps/cic-ussd/tests/fixtures/celery.py +++ b/apps/cic-ussd/tests/fixtures/tasker.py @@ -1,19 +1,27 @@ # standard imports import logging +import os import pytest import shutil import tempfile +# external imports +from celery import uuid + + logg = logging.getLogger() @pytest.fixture(scope='session') def celery_includes(): return [ - 'cic_ussd.tasks.ussd', 'cic_ussd.tasks.callback_handler', + 'cic_ussd.tasks.metadata', + 'cic_ussd.tasks.notifications', + 'cic_ussd.tasks.processor', + 'cic_ussd.tasks.ussd_session', + 'cic_eth.queue.balance', 'cic_notify.tasks.sms', - 'cic_ussd.tasks.metadata' ] @@ -42,3 +50,8 @@ def celery_config(): @pytest.fixture(scope='session') def celery_enable_logging(): return True + + +@pytest.fixture(scope='function') +def task_uuid(): + return uuid() diff --git a/apps/cic-ussd/tests/fixtures/transaction.py b/apps/cic-ussd/tests/fixtures/transaction.py new file mode 100644 index 00000000..9dacfaa5 --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/transaction.py @@ -0,0 +1,148 @@ +# standard import +import random + +# external import +import pytest + +# local import + +# tests imports + + +@pytest.fixture(scope='function') +def notification_data(activated_account, cache_person_metadata, cache_preferences, preferences): + return { + 'blockchain_address': activated_account.blockchain_address, + 'token_symbol': 'GFT', + 'token_value': 25000000, + 'role': 'sender', + 'action_tag': 'Sent', + 'direction_tag': 'To', + 'metadata_id': activated_account.standard_metadata_id(), + 'phone_number': activated_account.phone_number, + 'available_balance': 50.0, + 'preferred_language': preferences.get('preferred_language') + } + + +@pytest.fixture(scope='function') +def statement(activated_account): + return [ + { + 'blockchain_address': activated_account.blockchain_address, + 'token_symbol': 'GFT', + 'token_value': 25000000, + 'role': 'sender', + 'action_tag': 'Sent', + 'direction_tag': 'To', + 'metadata_id': activated_account.standard_metadata_id(), + 'phone_number': activated_account.phone_number, + } + ] + + +@pytest.fixture(scope='function') +def transaction_result(activated_account, load_config, valid_recipient): + return { + 'hash': '0xb469fb2ebacc9574afb7b51d44e174fba7129fde71bf757fd39784363270832b', + 'sender': activated_account.blockchain_address, + 'recipient': valid_recipient.blockchain_address, + 'source_token_value': 25000000, + 'destination_token_value': 25000000, + 'source_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', + 'destination_token': '0xa75B519dc9b0A50D267E03D8B6808f85A66932dd', + 'source_token_symbol': load_config.get('TEST_TOKEN_SYMBOL'), + 'destination_token_symbol': load_config.get('TEST_TOKEN_SYMBOL'), + 'source_token_decimals': 6, + 'destination_token_decimals': 6, + 'chain': 'evm:bloxberg:8996' + } + + +@pytest.fixture(scope='function') +def transactions_list(activated_account, valid_recipient): + return [ + { + 'tx_hash': '0x7cdca277861665fa56c4c32930101ff41316c61af3683be12b4879e3d9990125', + 'signed_tx': '0xf8a70201837a120094b708175e3f6cd850643aaf7b32212afad50e254980b844a9059cbb000000000000000000000000367cb0f65137b0a845c1db4b7ca47d3def32dde800000000000000000000000000000000000000000000000000000000017d784082466ba030a75acff9081e57e0a9daa6858d7473fc10348bf95a6da4dd1dc6a602883c8da005358742612001ad44fc142c30bcc23b452af48c90f9c6c80433ae2a93b2e96e', + 'nonce': 2, + 'status': 'SUCCESS', + 'status_code': 4104, + 'source_token': '0xb708175e3f6Cd850643aAF7B32212AFad50e2549', + 'destination_token': '0xb708175e3f6Cd850643aAF7B32212AFad50e2549', + 'block_number': 94, + 'tx_index': 0, + 'sender': activated_account.blockchain_address, + 'recipient': valid_recipient.blockchain_address, + 'from_value': 25000000, + 'to_value': 25000000, + 'date_created': '2021-07-14T14:14:58.117017', + 'date_updated': '2021-07-14T14:14:58.117017', + 'date_checked': '2021-07-14T14:14:58.603124', + 'timestamp': 1626272098, + 'hash': '0x7cdca277861665fa56c4c32930101ff41316c61af3683be12b4879e3d9990125', + 'source_token_symbol': 'GFT', + 'source_token_decimals': 6, + 'destination_token_symbol': 'GFT', + 'destination_token_decimals': 6 + }, + { + 'tx_hash': '0x5bd3b72f07ceb55199e759e8e82006b1c70bd5b87a3d37e3327515ea27872290', + 'signed_tx': '0xf88601018323186094103d1ed6e370dba6267045c70d4999384c18a04a80a463e4bff4000000000000000000000000367cb0f65137b0a845c1db4b7ca47d3def32dde882466ca00beb6913cdd0b9b63469fbca53e2fb48dceeedf73d31d54c23c85392f01419a8a02352fff9187ba3dd6409ef6e473369dc4c3459a8baaa9bc1d68a541ca8a8f923', + 'nonce': 1, + 'status': 'REVERTED', + 'status_code': 5128, + 'source_token': '0x0000000000000000000000000000000000000000', + 'destination_token': '0x0000000000000000000000000000000000000000', + 'block_number': 80, + 'tx_index': 0, + 'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A', + 'from_value': 0, + 'to_value': 0, + 'date_created': '2021-07-14T14:13:46.036198', + 'date_updated': '2021-07-14T14:13:46.036198', + 'date_checked': '2021-07-14T14:13:46.450050', + 'timestamp': 1626272026, + 'hash': '0x5bd3b72f07ceb55199e759e8e82006b1c70bd5b87a3d37e3327515ea27872290'}, + { + 'tx_hash': '0x9d586562e1e40ae80fd506161e59825bc316293b5c522b8f243cf6c804c7843b', + 'signed_tx': '0xf868800182520894367cb0f65137b0a845c1db4b7ca47d3def32dde887066517289880008082466ca0c75083ea13d4fa9dfd408073cd0a8234199b78e79afe441fb71d7c79aa282ca6a00a7dd29e3ec1102817236d85af365fce7593b337ee609d02efdb86d298cf11ab', + 'nonce': 0, + 'status': 'SUCCESS', + 'status_code': 4104, + 'source_token': '0x0000000000000000000000000000000000000000', + 'destination_token': '0x0000000000000000000000000000000000000000', + 'block_number': 78, + 'tx_index': 0, + 'sender': '0xb41BfEE260693A473254D62b81aE1ADCC9E51AFb', + 'recipient': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'from_value': 1800000000000000, + 'to_value': 1800000000000000, + 'date_created': '2021-07-14T14:13:35.839638', + 'date_updated': '2021-07-14T14:13:35.839638', + 'date_checked': '2021-07-14T14:13:36.333426', + 'timestamp': 1626272015, + 'hash': '0x9d586562e1e40ae80fd506161e59825bc316293b5c522b8f243cf6c804c7843b' + }, + { + 'tx_hash': '0x32ca3dd3bef06463b452f4d32f5f563d083cb4759219eed90f3d2a9c1791c5fc', + 'signed_tx': '0xf88680018323186094103d1ed6e370dba6267045c70d4999384c18a04a80a463e4bff4000000000000000000000000367cb0f65137b0a845c1db4b7ca47d3def32dde882466ca0ab9ec1c6affb80f54bb6c2a25e64f38b3da840404180fb189bd6e191266f3c63a03cc53e59f8528da04aeec36ab8ae099553fca366bd067feffed6362ccb28d8f0', + 'nonce': 0, + 'status': 'SUCCESS', + 'status_code': 4104, + 'source_token': '0x0000000000000000000000000000000000000000', + 'destination_token': '0x0000000000000000000000000000000000000000', + 'block_number': 79, + 'tx_index': 0, + 'sender': '0x367cB0F65137b0A845c1DB4B7Ca47D3DEF32dDe8', + 'recipient': '0x103d1ed6e370dBa6267045c70d4999384c18a04A', + 'from_value': 0, + 'to_value': 0, + 'date_created': '2021-07-14T14:13:35.638355', + 'date_updated': '2021-07-14T14:13:35.638355', + 'date_checked': '2021-07-14T14:13:40.927113', + 'timestamp': 1626272015, + 'hash': '0x32ca3dd3bef06463b452f4d32f5f563d083cb4759219eed90f3d2a9c1791c5fc'} + ] + diff --git a/apps/cic-ussd/tests/fixtures/user.py b/apps/cic-ussd/tests/fixtures/user.py deleted file mode 100644 index 0add085e..00000000 --- a/apps/cic-ussd/tests/fixtures/user.py +++ /dev/null @@ -1,120 +0,0 @@ -# standard imports -import json -import uuid -from random import randint - -# third party imports -import pytest -from cic_types.models.person import generate_metadata_pointer -from faker import Faker - -# local imports -from cic_ussd.db.models.account import AccountStatus, Account -from cic_ussd.redis import cache_data -from cic_ussd.metadata import blockchain_address_to_metadata_pointer - - -fake = Faker() - - -@pytest.fixture(scope='function') -def create_activated_user(init_database, set_fernet_key): - user = Account( - blockchain_address='0xFD9c5aD15C72C6F60f1a119A608931226674243f', - phone_number='+25498765432' - ) - user.preferred_language = 'en' - user.create_password('0000') - user.activate_account() - init_database.add(user) - init_database.commit() - return user - - -@pytest.fixture(scope='function') -def create_valid_tx_recipient(init_database, set_fernet_key): - user = Account( - blockchain_address='0xd6204101012270Bf2558EDcFEd595938d1847bf0', - phone_number='+25498765432' - ) - user.preferred_language = 'en' - user.create_password('0000') - user.activate_account() - init_database.add(user) - init_database.commit() - return user - - -@pytest.fixture(scope='function') -def create_valid_tx_sender(init_database, set_fernet_key): - user = Account( - blockchain_address='0xd6204101012270Bf2558EDcFEd595938d1847bf1', - phone_number='+25498765433' - ) - user.preferred_language = 'en' - user.create_password('0000') - user.activate_account() - init_database.add(user) - init_database.commit() - return user - - -@pytest.fixture(scope='function') -def create_pending_user(init_database, set_fernet_key): - user = Account( - blockchain_address='0x0ebdea8612c1b05d952c036859266c7f2cfcd6a29842d9c6cce3b9f1ba427588', - phone_number='+25498765432' - ) - init_database.add(user) - init_database.commit() - return user - - -@pytest.fixture(scope='function') -def create_pin_blocked_user(init_database, set_fernet_key): - user = Account( - blockchain_address='0x0ebdea8612c1b05d952c036859266c7f2cfcd6a29842d9c6cce3b9f1ba427588', - phone_number='+25498765432' - ) - user.create_password('0000') - user.failed_pin_attempts = 3 - user.account_status = 3 - init_database.add(user) - init_database.commit() - return user - - -@pytest.fixture(scope='function') -def create_locked_accounts(init_database, set_fernet_key): - for i in range(20): - blockchain_address = str(uuid.uuid4()) - phone_number = fake.phone_number() - pin = f'{randint(1000, 9999)}' - user = Account(phone_number=phone_number, blockchain_address=blockchain_address) - user.create_password(password=pin) - user.failed_pin_attempts = 3 - user.account_status = AccountStatus.LOCKED.value - user.session.add(user) - user.session.commit() - - -@pytest.fixture(scope='function') -def complete_user_metadata(create_activated_user): - return { - "date_registered": create_activated_user.created, - "family_name": "Snow", - "given_name": "Name", - "gender": 'Male', - "location": "Kangemi", - "products": "Mandazi" - } - - -@pytest.fixture(scope='function') -def cached_user_metadata(create_activated_user, init_redis_cache, person_metadata): - user_metadata = json.dumps(person_metadata) - key = generate_metadata_pointer( - identifier=blockchain_address_to_metadata_pointer(blockchain_address=create_activated_user.blockchain_address), - cic_type=':cic.person' - ) - cache_data(key=key, data=user_metadata) diff --git a/apps/cic-ussd/tests/fixtures/ussd_session.py b/apps/cic-ussd/tests/fixtures/ussd_session.py index 244de286..b04244c4 100644 --- a/apps/cic-ussd/tests/fixtures/ussd_session.py +++ b/apps/cic-ussd/tests/fixtures/ussd_session.py @@ -1,52 +1,69 @@ # standard imports -import json +import os +import random -# third-party imports +# external imports import pytest # local imports from cic_ussd.db.models.ussd_session import UssdSession -from cic_ussd.redis import InMemoryStore -from cic_ussd.session.ussd_session import UssdSession as InMemoryUssdSession +from cic_ussd.session.ussd_session import create_ussd_session + +# test imports +from tests.helpers.accounts import phone_number @pytest.fixture(scope='function') -def ussd_session_data(): +def activated_account_ussd_session(load_config, activated_account): + valid_service_codes = load_config.get('USSD_SERVICE_CODE').split(",") return { - 'external_session_id': 'AT974186', - 'service_code': '*483*46#', - 'msisdn': '+25498765432', - 'user_input': '1', + 'data': {}, + 'external_session_id': os.urandom(20).hex(), + 'msisdn': activated_account.phone_number, + 'service_code': valid_service_codes[0], 'state': 'initial_language_selection', - 'session_data': {}, + 'user_input': '1', + } + + +@pytest.fixture(scope='function') +def generic_ussd_session(load_config, activated_account): + valid_service_codes = load_config.get('USSD_SERVICE_CODE').split(",") + return { + 'data': {}, + 'service_code': valid_service_codes[0], + 'state': 'initial_language_selection', + 'user_input': '1', 'version': 2 } @pytest.fixture(scope='function') -def create_in_redis_ussd_session(ussd_session_data, init_redis_cache): - external_session_id = ussd_session_data.get('external_session_id') - InMemoryUssdSession.redis_cache = InMemoryStore.cache - InMemoryUssdSession.redis_cache.set(external_session_id, json.dumps(ussd_session_data)) - return InMemoryUssdSession.redis_cache +def ussd_session_traffic(generic_ussd_session, init_database, persisted_ussd_session): + for _ in range((random.randint(5, 15))): + generic_ussd_session['external_session_id'] = os.urandom(20).hex() + generic_ussd_session['msisdn'] = phone_number() + ussd = UssdSession(**{key: value for key, value in generic_ussd_session.items()}) + init_database.add(ussd) + init_database.commit() @pytest.fixture(scope='function') -def get_in_redis_ussd_session(ussd_session_data, create_in_redis_ussd_session): - external_session_id = ussd_session_data.get('external_session_id') - ussd_session_data = create_in_redis_ussd_session.get(external_session_id) - ussd_session_data = json.loads(ussd_session_data) - # remove version from ussd_session data because the ussd_session object does not expect a version at initialization - del ussd_session_data['version'] - ussd_session = InMemoryUssdSession(**{key: value for key, value in ussd_session_data.items()}) - ussd_session.version = ussd_session_data.get('version') - return ussd_session +def ussd_session_data(load_config): + return { + 'recipient': phone_number() + } @pytest.fixture(scope='function') -def create_in_db_ussd_session(init_database, ussd_session_data): - ussd_session_data['session_data'] = {} - ussd_session = UssdSession(**{key: value for key, value in ussd_session_data.items()}) - init_database.add(ussd_session) +def cached_ussd_session(init_cache, activated_account_ussd_session): + return create_ussd_session(**{key: value for key, value in activated_account_ussd_session.items()}) + + +@pytest.fixture(scope='function') +def persisted_ussd_session(init_cache, init_database, activated_account_ussd_session): + activated_account_ussd_session['version'] = 2 + ussd = UssdSession(**{key: value for key, value in activated_account_ussd_session.items()}) + init_database.add(ussd) init_database.commit() - return ussd_session + return ussd diff --git a/apps/cic-ussd/tests/fixtures/util.py b/apps/cic-ussd/tests/fixtures/util.py new file mode 100644 index 00000000..3eee3cf9 --- /dev/null +++ b/apps/cic-ussd/tests/fixtures/util.py @@ -0,0 +1,63 @@ +# standard imports + +# external imports +import pytest + +# local imports + + +@pytest.fixture(scope='function') +def uwsgi_env(): + return { + 'REQUEST_METHOD': 'POST', + 'REQUEST_URI': '/', + 'PATH_INFO': '/', + 'QUERY_STRING': '', + 'SERVER_PROTOCOL': 'HTTP/1.1', + 'SCRIPT_NAME': '', + 'SERVER_NAME': 'mango-habanero', + 'SERVER_PORT': '9091', + 'UWSGI_ROUTER': 'http', + 'REMOTE_ADDR': '127.0.0.1', + 'REMOTE_PORT': '33515', + 'CONTENT_TYPE': 'application/json', + 'HTTP_USER_AGENT': 'PostmanRuntime/7.26.8', + 'HTTP_ACCEPT': '*/*', + 'HTTP_POSTMAN_TOKEN': 'c1f6eb29-8160-497f-a5a1-935d175e2eb7', + 'HTTP_HOST': '127.0.0.1:9091', + 'HTTP_ACCEPT_ENCODING': 'gzip, deflate, br', + 'HTTP_CONNECTION': 'keep-alive', + 'CONTENT_LENGTH': '102', + 'wsgi.version': (1, 0), + 'wsgi.run_once': False, + 'wsgi.multithread': False, + 'wsgi.multiprocess': False, + 'wsgi.url_scheme': 'http', + 'uwsgi.version': b'2.0.19.1', + 'uwsgi.node': b'mango-habanero' + } + + +@pytest.fixture(scope='function') +def locked_accounts_env(with_params_env): + with_params_env['PATH_INFO'] = '/accounts/locked/10/10' + return with_params_env + + +@pytest.fixture(scope='function') +def with_params_env(uwsgi_env): + uwsgi_env['REQUEST_METHOD'] = 'GET' + uwsgi_env['REQUEST_URI'] = '/?phone=0700000000' + return uwsgi_env + + +@pytest.fixture(scope='function') +def mock_url(): + return 'https://testing.io' + + +@pytest.fixture(scope='function') +def mock_response(): + return { + 'Looking': 'Good' + } diff --git a/apps/cic-ussd/tests/helpers/accounts.py b/apps/cic-ussd/tests/helpers/accounts.py index 9ba9bd78..93f175a7 100644 --- a/apps/cic-ussd/tests/helpers/accounts.py +++ b/apps/cic-ussd/tests/helpers/accounts.py @@ -1,8 +1,10 @@ # standard imports +import os import random import uuid # external imports +from chainlib.eth.address import to_checksum_address from faker import Faker from faker_e164.providers import E164Provider @@ -18,6 +20,10 @@ def phone_number() -> str: return fake.e164('KE') +def blockchain_address() -> str: + return to_checksum_address('0x' + os.urandom(20).hex()) + + def session_id() -> str: return uuid.uuid4().hex diff --git a/apps/cic-ussd/tests/integration/README.md b/apps/cic-ussd/tests/integration/README.md index dd567c53..89343d61 100644 --- a/apps/cic-ussd/tests/integration/README.md +++ b/apps/cic-ussd/tests/integration/README.md @@ -14,7 +14,7 @@ There are four files defining the integration tests. ## REQUIREMENTS In order to run the transaction tests, please ensure that the faucet amount is set to a non-zero value, ideally `50000000` -which is the value set in the config file `.config/test/integration.ini`. +which is the value set in the config file `config/test/integration.ini`. This implies setting the `DEV_FAUCET_AMOUNT` to a non-zero value before bringing up the contract-migration image: diff --git a/apps/cic-ussd/tests/integration/run.sh b/apps/cic-ussd/tests/integration/run.sh index a6339c09..9e2e2112 100644 --- a/apps/cic-ussd/tests/integration/run.sh +++ b/apps/cic-ussd/tests/integration/run.sh @@ -1,2 +1,2 @@ #!/bin/bash -PYTHONPATH=. py.test --debug -vv --log-level debug -s --log-cli-level debug +PYTHONPATH=. pytest --log-level debug -x . diff --git a/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml b/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml index d602583c..a8c6fb0f 100644 --- a/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_account_creation.tavern.yaml @@ -37,12 +37,13 @@ stages: status_code: - 200 headers: - Content-Length: '175' + Content-Length: '91' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n" + expected_response: "END Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari." + delay_after: 5 - name: Initiate account creation process [second account]. request: @@ -59,12 +60,12 @@ stages: status_code: - 200 headers: - Content-Length: '175' + Content-Length: '91' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Your account is being created. You will receive an SMS when your account is ready.\nAkaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari.\n" + expected_response: "END Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari." delay_after: 5 - name: Initaite account metadata entry [first account] @@ -82,12 +83,12 @@ stages: status_code: - 200 headers: - Content-Length: '61' + Content-Length: '57' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Welcome to Sarafu Network\n1. English\n2. Kiswahili\n3. Help" + expected_response: "CON Karibu Sarafu Network\n1. English\n2. Kiswahili\n3. Help" - name: Initaite account metadata entry [second account] request: @@ -104,12 +105,12 @@ stages: status_code: - 200 headers: - Content-Length: '61' + Content-Length: '57' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Welcome to Sarafu Network\n1. English\n2. Kiswahili\n3. Help" + expected_response: "CON Karibu Sarafu Network\n1. English\n2. Kiswahili\n3. Help" - name: Select preferred language [English] request: @@ -126,12 +127,13 @@ stages: status_code: - 200 headers: - Content-Length: '64' + Content-Length: '71' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Please enter a new four number PIN for your account.\n0. Back" + expected_response: "CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako\n0. Nyuma" + delay_after: 5 - name: Select preferred language [Kiswahili] request: @@ -154,6 +156,7 @@ stages: function: ext.validator:validate_response extra_kwargs: expected_response: "CON Tafadhali weka pin mpya yenye nambari nne kwa akaunti yako\n0. Nyuma" + delay_after: 5 - name: Enter pin number [{first_account_pin_number} - first account] request: @@ -176,6 +179,7 @@ stages: function: ext.validator:validate_response extra_kwargs: expected_response: "CON Enter your four number PIN again\n0. Back" + delay_after: 5 - name: Enter pin number [second_account_pin_number - second account] request: @@ -198,6 +202,7 @@ stages: function: ext.validator:validate_response extra_kwargs: expected_response: "CON Weka PIN yako tena\n0. Nyuma" + delay_after: 5 - name: Pin number confirmation [first_account_pin_number - first account] request: diff --git a/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml b/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml index 1d9ea975..993a2046 100644 --- a/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml +++ b/apps/cic-ussd/tests/integration/test_account_management.tavern.yaml @@ -162,12 +162,13 @@ stages: status_code: - 200 headers: - Content-Length: '30' + Content-Length: '36' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Asante kwa kutumia huduma." + expected_response: "END Thank you for using the service." + delay_after: 5 - name: Select language [second account] request: @@ -184,12 +185,13 @@ stages: status_code: - 200 headers: - Content-Length: '36' + Content-Length: '30' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "END Thank you for using the service." + expected_response: "END Asante kwa kutumia huduma." + delay_after: 5 - name: Second account management start menu [first account] request: @@ -512,12 +514,12 @@ stages: status_code: - 200 headers: - Content-Length: '31' + Content-Length: '43' Content-Type: "text/plain" verify_response_with: function: ext.validator:validate_response extra_kwargs: - expected_response: "CON Weka PIN yako tena\n0. Nyuma" + expected_response: "CON Weka nambari yako ya siri tena\n0. Nyuma" - name: Enter new pin [second account] request: diff --git a/apps/cic-ussd/transitions/age_setting_transitions.json b/apps/cic-ussd/transitions/age_setting_transitions.json index 957c6ae8..36e7e1a1 100644 --- a/apps/cic-ussd/transitions/age_setting_transitions.json +++ b/apps/cic-ussd/transitions/age_setting_transitions.json @@ -3,9 +3,9 @@ "trigger": "scan_data", "source": "enter_date_of_birth", "dest": "dob_edit_pin_authorization", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", "conditions": [ - "cic_ussd.state_machine.logic.validator.has_cached_user_metadata", + "cic_ussd.state_machine.logic.validator.has_cached_person_metadata", "cic_ussd.state_machine.logic.validator.is_valid_date" ] }, @@ -14,8 +14,8 @@ "source": "enter_date_of_birth", "dest": "enter_location", "conditions": "cic_ussd.state_machine.logic.validator.is_valid_date", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", @@ -28,7 +28,7 @@ "source": "dob_edit_pin_authorization", "dest": "exit", "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", - "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + "after": "cic_ussd.state_machine.logic.account.edit_user_metadata_attribute" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/exit_transitions.json b/apps/cic-ussd/transitions/exit_transitions.json index d814040a..54661a03 100644 --- a/apps/cic-ussd/transitions/exit_transitions.json +++ b/apps/cic-ussd/transitions/exit_transitions.json @@ -45,8 +45,7 @@ "trigger": "scan_data", "source": "exit_invalid_recipient", "dest": "exit", - "conditions": "cic_ussd.state_machine.logic.menu.menu_ninety_nine_selected", - "after": "cic_ussd.state_machine.logic.sms.upsell_unregistered_recipient" + "conditions": "cic_ussd.state_machine.logic.menu.menu_ninety_nine_selected" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/gender_setting_transitions.json b/apps/cic-ussd/transitions/gender_setting_transitions.json index e0d2b346..60d5ec7a 100644 --- a/apps/cic-ussd/transitions/gender_setting_transitions.json +++ b/apps/cic-ussd/transitions/gender_setting_transitions.json @@ -3,17 +3,17 @@ "trigger": "scan_data", "source": "enter_gender", "dest": "enter_date_of_birth", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", "conditions": "cic_ussd.state_machine.logic.validator.is_valid_gender_selection", - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "enter_gender", "dest": "gender_edit_pin_authorization", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", "conditions": [ - "cic_ussd.state_machine.logic.validator.has_cached_user_metadata", + "cic_ussd.state_machine.logic.validator.has_cached_person_metadata", "cic_ussd.state_machine.logic.validator.is_valid_gender_selection" ] }, @@ -22,7 +22,7 @@ "source": "gender_edit_pin_authorization", "dest": "exit", "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", - "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + "after": "cic_ussd.state_machine.logic.account.edit_user_metadata_attribute" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/language_setting_transitions.json b/apps/cic-ussd/transitions/language_setting_transitions.json index 15a3b347..4ba0fa3a 100644 --- a/apps/cic-ussd/transitions/language_setting_transitions.json +++ b/apps/cic-ussd/transitions/language_setting_transitions.json @@ -3,14 +3,14 @@ "trigger": "scan_data", "source": "select_preferred_language", "dest": "exit", - "after": "cic_ussd.state_machine.logic.user.change_preferred_language_to_en", + "after": "cic_ussd.state_machine.logic.account.change_preferred_language", "conditions": "cic_ussd.state_machine.logic.menu.menu_one_selected" }, { "trigger": "scan_data", "source": "select_preferred_language", "dest": "exit", - "after": "cic_ussd.state_machine.logic.user.change_preferred_language_to_sw", + "after": "cic_ussd.state_machine.logic.account.change_preferred_language", "conditions": "cic_ussd.state_machine.logic.menu.menu_two_selected" }, { diff --git a/apps/cic-ussd/transitions/location_setting_transitions.json b/apps/cic-ussd/transitions/location_setting_transitions.json index fa6ea52d..edd3c622 100644 --- a/apps/cic-ussd/transitions/location_setting_transitions.json +++ b/apps/cic-ussd/transitions/location_setting_transitions.json @@ -3,22 +3,22 @@ "trigger": "scan_data", "source": "enter_location", "dest": "enter_products", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "enter_location", "dest": "location_edit_pin_authorization", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "conditions": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "conditions": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "location_edit_pin_authorization", "dest": "exit", "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", - "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + "after": "cic_ussd.state_machine.logic.account.edit_user_metadata_attribute" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/name_setting_transitions.json b/apps/cic-ussd/transitions/name_setting_transitions.json index 38b5e58c..b4eb96b7 100644 --- a/apps/cic-ussd/transitions/name_setting_transitions.json +++ b/apps/cic-ussd/transitions/name_setting_transitions.json @@ -3,28 +3,28 @@ "trigger": "scan_data", "source": "enter_given_name", "dest": "enter_family_name", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data" }, { "trigger": "scan_data", "source": "enter_family_name", "dest": "name_edit_pin_authorization", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "conditions": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "conditions": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "enter_family_name", "dest": "enter_gender", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "name_edit_pin_authorization", "dest": "exit", "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", - "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + "after": "cic_ussd.state_machine.logic.account.edit_user_metadata_attribute" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/products_setting_transitions.json b/apps/cic-ussd/transitions/products_setting_transitions.json index ac105842..28977ec9 100644 --- a/apps/cic-ussd/transitions/products_setting_transitions.json +++ b/apps/cic-ussd/transitions/products_setting_transitions.json @@ -3,25 +3,25 @@ "trigger": "scan_data", "source": "enter_products", "dest": "products_edit_pin_authorization", - "conditions": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata", - "after": "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data" + "conditions": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata", + "after": "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data" }, { "trigger": "scan_data", "source": "enter_products", "dest": "start", "after": [ - "cic_ussd.state_machine.logic.user.save_metadata_attribute_to_session_data", - "cic_ussd.state_machine.logic.user.save_complete_user_metadata" + "cic_ussd.state_machine.logic.account.save_metadata_attribute_to_session_data", + "cic_ussd.state_machine.logic.account.save_complete_person_metadata" ], - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata" + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata" }, { "trigger": "scan_data", "source": "products_edit_pin_authorization", "dest": "exit", "conditions": "cic_ussd.state_machine.logic.pin.is_authorized_pin", - "after": "cic_ussd.state_machine.logic.user.edit_user_metadata_attribute" + "after": "cic_ussd.state_machine.logic.account.edit_user_metadata_attribute" }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/transitions/signup_transitions.json b/apps/cic-ussd/transitions/signup_transitions.json index 731c8131..a0b14049 100644 --- a/apps/cic-ussd/transitions/signup_transitions.json +++ b/apps/cic-ussd/transitions/signup_transitions.json @@ -3,14 +3,14 @@ "trigger": "scan_data", "source": "initial_language_selection", "dest": "initial_pin_entry", - "after": "cic_ussd.state_machine.logic.user.change_preferred_language_to_en", + "after": "cic_ussd.state_machine.logic.account.change_preferred_language", "conditions": "cic_ussd.state_machine.logic.menu.menu_one_selected" }, { "trigger": "scan_data", "source": "initial_language_selection", "dest": "initial_pin_entry", - "after": "cic_ussd.state_machine.logic.user.change_preferred_language_to_sw", + "after": "cic_ussd.state_machine.logic.account.change_preferred_language", "conditions": "cic_ussd.state_machine.logic.menu.menu_two_selected" }, { @@ -39,12 +39,12 @@ { "trigger": "scan_data", "source": "initial_pin_confirmation", - "unless": "cic_ussd.state_machine.logic.validator.has_cached_user_metadata", + "unless": "cic_ussd.state_machine.logic.validator.has_cached_person_metadata", "conditions": "cic_ussd.state_machine.logic.pin.pins_match", "dest": "start", "after": [ "cic_ussd.state_machine.logic.pin.complete_pin_change", - "cic_ussd.state_machine.logic.user.update_account_status_to_active" + "cic_ussd.state_machine.logic.account.update_account_status_to_active" ] } ] diff --git a/apps/cic-ussd/transitions/transaction_transitions.json b/apps/cic-ussd/transitions/transaction_transitions.json index e12e9221..b674c8f5 100644 --- a/apps/cic-ussd/transitions/transaction_transitions.json +++ b/apps/cic-ussd/transitions/transaction_transitions.json @@ -12,7 +12,10 @@ { "trigger": "scan_data", "source": "enter_transaction_recipient", - "dest": "exit_invalid_recipient" + "dest": "exit_invalid_recipient", + "after": [ + "cic_ussd.state_machine.logic.transaction.save_recipient_phone_to_session_data", + "cic_ussd.state_machine.logic.sms.upsell_unregistered_recipient"] }, { "trigger": "scan_data", diff --git a/apps/cic-ussd/var/lib/locale/helpers.en.yml b/apps/cic-ussd/var/lib/locale/helpers.en.yml new file mode 100644 index 00000000..2ac7333e --- /dev/null +++ b/apps/cic-ussd/var/lib/locale/helpers.en.yml @@ -0,0 +1,19 @@ +en: + female: |- + Female + from: |- + From + male: |- + Male + not_provided: |- + Not provided + no_transaction_history: |- + No transaction history + other: |- + Other + received: |- + Received + sent: |- + Sent + to: |- + To \ No newline at end of file diff --git a/apps/cic-ussd/var/lib/locale/helpers.sw.yml b/apps/cic-ussd/var/lib/locale/helpers.sw.yml new file mode 100644 index 00000000..f6781fc3 --- /dev/null +++ b/apps/cic-ussd/var/lib/locale/helpers.sw.yml @@ -0,0 +1,19 @@ +sw: + female: |- + Mwanamke + from: |- + Kutoka kwa + male: |- + Mwanaume + not_provided: |- + Haijawekwa + no_transaction_history: |- + Hamna ripoti ya matumizi + other: |- + Nyingine + received: |- + Ulipokea + sent: |- + Ulituma + to: |- + Kwa \ No newline at end of file diff --git a/apps/cic-ussd/var/lib/locale/sms.en.yml b/apps/cic-ussd/var/lib/locale/sms.en.yml index 6c199336..a999c799 100644 --- a/apps/cic-ussd/var/lib/locale/sms.en.yml +++ b/apps/cic-ussd/var/lib/locale/sms.en.yml @@ -7,3 +7,5 @@ en: Successfully sent %{amount} %{token_symbol} to %{tx_recipient_information} %{timestamp}. New balance is %{balance} %{token_symbol}. terms: |- By using the service, you agree to the terms and conditions at http://grassecon.org/tos + upsell_unregistered_recipient: |- + %{tx_sender_information} tried to send you %{token_symbol} but you are not registered. To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}. \ No newline at end of file diff --git a/apps/cic-ussd/var/lib/locale/sms.sw.yml b/apps/cic-ussd/var/lib/locale/sms.sw.yml index 6e9b9b62..df07d132 100644 --- a/apps/cic-ussd/var/lib/locale/sms.sw.yml +++ b/apps/cic-ussd/var/lib/locale/sms.sw.yml @@ -6,4 +6,6 @@ sw: sent_tokens: |- Umetuma %{amount} %{token_symbol} kwa %{tx_recipient_information} %{timestamp}. Salio lako ni %{balance} %{token_symbol}. terms: |- - Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos \ No newline at end of file + Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos + upsell_unregistered_recipient: |- + %{tx_sender_information} amejaribu kukutumia %{token_symbol} lakini hujasajili. Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}. \ No newline at end of file diff --git a/apps/cic-ussd/var/lib/locale/ussd.en.yml b/apps/cic-ussd/var/lib/locale/ussd.en.yml index bfb6545a..ac7d4e5c 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.en.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.en.yml @@ -73,13 +73,15 @@ en: 1. English 2. Kiswahili 0. Back + retry_pin_entry: |- + CON Incorrect PIN entered, please try again. You have %{remaining_attempts} attempts remaining. + 0. Back enter_current_pin: first: |- CON Enter current PIN. 0. Back retry: |- - CON Enter current PIN. You have %{remaining_attempts} attempts remaining. - 0. Back + %{retry_pin_entry} enter_new_pin: |- CON Enter your new four number PIN 0. Back @@ -92,67 +94,58 @@ en: Please enter your PIN to confirm. 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} display_metadata_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} account_balances_pin_authorization: first: |- CON Please enter your PIN to view balances 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} account_statement_pin_authorization: first: |- CON Please enter your PIN to view statement 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} name_edit_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} dob_edit_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} gender_edit_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} location_edit_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} products_edit_pin_authorization: first: |- CON Please enter your PIN 0. Back retry: |- - CON Please enter your PIN. You have %{remaining_attempts} attempts remaining - 0. Back + %{retry_pin_entry} account_balances: |- CON Your balances are as follows: - balance: %{operational_balance} %{token_symbol} + balance: %{available_balance} %{token_symbol} fees: %{tax} %{token_symbol} rewards: %{bonus} %{token_symbol} 0. Back @@ -213,4 +206,4 @@ en: 00. Back 99. Exit account_creation_prompt: |- - Your account is being created. You will receive an SMS when your account is ready. + END Your account is being created. You will receive an SMS when your account is ready. diff --git a/apps/cic-ussd/var/lib/locale/ussd.sw.yml b/apps/cic-ussd/var/lib/locale/ussd.sw.yml index 81690287..4c16c1cf 100644 --- a/apps/cic-ussd/var/lib/locale/ussd.sw.yml +++ b/apps/cic-ussd/var/lib/locale/ussd.sw.yml @@ -73,18 +73,20 @@ sw: 1. Kingereza 2. Kiswahili 0. Nyuma + retry_pin_entry: |- + CON Nambari uliyoweka si sahihi, jaribu tena. Una majaribio %{remaining_attempts} yaliyobaki. + 0. Back enter_current_pin: first: |- CON Weka nambari ya siri. 0. Nyuma retry: |- - CON Weka nambari ya siri. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Back + %{retry_pin_entry} enter_new_pin: |- CON Weka nambari ya siri mpya 0. Nyuma new_pin_confirmation: |- - CON Weka PIN yako tena + CON Weka nambari yako ya siri tena 0. Nyuma transaction_pin_authorization: first: |- @@ -92,67 +94,58 @@ sw: Tafadhali weka nambari yako ya siri kudhibitisha. 0. Nyuma retry: |- - CON Weka nambari ya siri. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} display_metadata_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} account_balances_pin_authorization: first: |- CON Tafadhali weka PIN yako kuona salio. 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} account_statement_pin_authorization: first: |- CON Tafadhali weka PIN yako kuona taarifa ya matumizi. 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} name_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} dob_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} gender_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} location_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} products_edit_pin_authorization: first: |- CON Tafadhali weka PIN yako 0. Nyuma retry: |- - CON Tafadhali weka PIN yako. Una majaribio %{remaining_attempts} yaliyobaki. - 0. Nyuma + %{retry_pin_entry} account_balances: |- CON Salio zako ni zifuatazo: - salio: %{operational_balance} %{token_symbol} + salio: %{available_balance} %{token_symbol} ushuru: %{tax} %{token_symbol} tuzo: %{bonus} %{token_symbol} 0. Nyuma @@ -213,4 +206,4 @@ sw: 00. Nyuma 99. Ondoka account_creation_prompt: |- - Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari. + END Akaunti yako ya Sarafu inatayarishwa. Utapokea ujumbe wa SMS akaunti yako ikiwa tayari. diff --git a/apps/data-seeding/cic_ussd/import_task.py b/apps/data-seeding/cic_ussd/import_task.py index 2feb9cbb..d04f6dea 100644 --- a/apps/data-seeding/cic_ussd/import_task.py +++ b/apps/data-seeding/cic_ussd/import_task.py @@ -295,7 +295,7 @@ def set_ussd_data(config: dict, ussd_data: dict): preferred_language = ussd_data['preferred_language'] phone_number = ussd_data['phone'] - sql = 'UPDATE account SET account_status = %s, preferred_language = %s WHERE phone_number = %s' + sql = 'UPDATE account SET status = %s, preferred_language = %s WHERE phone_number = %s' db_cursor.execute(sql, (account_status, preferred_language, phone_number)) # commit changes From 8de5dc15408d56411c7a985a20d44ca8d6a0687f Mon Sep 17 00:00:00 2001 From: PhilipWafula Date: Fri, 6 Aug 2021 19:53:36 +0300 Subject: [PATCH 2/6] Adds password pepper value defaults. --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 187fa8b4..3c9a4540 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -477,6 +477,7 @@ services: context: apps/cic-ussd dockerfile: docker/Dockerfile environment: + APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} DATABASE_USER: grassroots DATABASE_HOST: postgres DATABASE_PORT: 5432 @@ -508,6 +509,7 @@ services: context: apps/cic-ussd dockerfile: docker/Dockerfile environment: + APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} DATABASE_USER: grassroots DATABASE_HOST: postgres DATABASE_PORT: 5432 @@ -532,6 +534,7 @@ services: context: apps/cic-ussd/ dockerfile: docker/Dockerfile environment: + APP_PASSWORD_PEPPER: ${APP_PASSWORD_PEPPER:-"QYbzKff6NhiQzY3ygl2BkiKOpER8RE/Upqs/5aZWW+I="} DATABASE_USER: grassroots DATABASE_HOST: postgres DATABASE_PORT: 5432 From d6c763f2d7e1827bb7bd2e79569a5c6b6402479a Mon Sep 17 00:00:00 2001 From: Spencer Ofwiti Date: Mon, 9 Aug 2021 19:13:05 +0000 Subject: [PATCH 3/6] Meta returns 404 if resource is not found. --- apps/cic-meta/scripts/server/handlers.ts | 2 +- apps/cic-meta/scripts/server/server.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cic-meta/scripts/server/handlers.ts b/apps/cic-meta/scripts/server/handlers.ts index 08bf2d9b..949ef61e 100644 --- a/apps/cic-meta/scripts/server/handlers.ts +++ b/apps/cic-meta/scripts/server/handlers.ts @@ -147,7 +147,7 @@ function handleClientMergeGet(db, digest, keystore) { doh(e); }); }).catch((e) => { - console.error('mesage', e); + console.error('message', e); doh(e); }); }); diff --git a/apps/cic-meta/scripts/server/server.ts b/apps/cic-meta/scripts/server/server.ts index 6107bdf5..ebe3cafc 100755 --- a/apps/cic-meta/scripts/server/server.ts +++ b/apps/cic-meta/scripts/server/server.ts @@ -205,7 +205,7 @@ async function processRequest(req, res) { if (content === undefined) { console.error('empty content', data); - res.writeHead(400, {"Content-Type": "text/plain"}); + res.writeHead(404, {"Content-Type": "text/plain"}); res.end(); return; } From bb3d38a1f9ee0895f364d7d9689de48bea83b67f Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Tue, 17 Aug 2021 06:46:51 +0000 Subject: [PATCH 4/6] Replace cic-base with chainlib cli utils --- apps/cic-cache/.coveragerc | 1 + apps/cic-cache/MANIFEST.in | 1 + apps/cic-cache/cic_cache/cli/__init__.py | 15 ++ apps/cic-cache/cic_cache/cli/arg.py | 20 +++ apps/cic-cache/cic_cache/cli/base.py | 31 ++++ apps/cic-cache/cic_cache/cli/celery.py | 24 +++ apps/cic-cache/cic_cache/cli/chain.py | 21 +++ apps/cic-cache/cic_cache/cli/config.py | 63 +++++++ apps/cic-cache/cic_cache/cli/registry.py | 33 ++++ apps/cic-cache/cic_cache/cli/rpc.py | 43 +++++ .../cic_cache/data/config/celery.ini | 5 + apps/cic-cache/cic_cache/data/config/cic.ini | 4 + .../cic_cache/data/config/database.ini | 10 ++ .../cic_cache/data/config/signer.ini | 2 + .../cic_cache/data/config/syncer.ini | 4 + .../versions/6604de4203e2_add_chain_syncer.py | 2 +- .../cic_cache/runnable/daemons/tracker.py | 61 +++---- apps/cic-cache/config/bancor.ini | 2 - apps/cic-cache/config/cic.ini | 1 - apps/cic-cache/config/docker/bancor.ini | 3 - apps/cic-cache/config/docker/cic.ini | 1 - apps/cic-cache/config/docker/eth.ini | 2 - apps/cic-cache/config/docker/syncer.ini | 3 +- apps/cic-cache/config/eth.ini | 2 - apps/cic-cache/config/syncer.ini | 3 - apps/cic-cache/requirements.txt | 13 +- apps/cic-cache/setup.cfg | 2 + apps/cic-cache/test_requirements.txt | 4 +- .../erc20_demurrage_token/__init__.py | 2 +- .../erc20-demurrage-token/requirements.txt | 8 +- .../erc20-demurrage-token/setup.cfg | 2 +- .../test_requirements.txt | 7 +- apps/cic-eth/.coveragerc | 1 + apps/cic-eth/MANIFEST.in | 2 +- apps/cic-eth/cic_eth/check/gas.py | 2 +- apps/cic-eth/cic_eth/check/signer.py | 2 +- apps/cic-eth/cic_eth/cli/__init__.py | 10 ++ apps/cic-eth/cic_eth/cli/arg.py | 31 ++++ apps/cic-eth/cic_eth/cli/base.py | 31 ++++ apps/cic-eth/cic_eth/cli/celery.py | 24 +++ apps/cic-eth/cic_eth/cli/chain.py | 21 +++ apps/cic-eth/cic_eth/cli/config.py | 63 +++++++ apps/cic-eth/cic_eth/cli/rpc.py | 86 +++++++++ apps/cic-eth/cic_eth/data/config/celery.ini | 5 + .../cic-eth/{ => cic_eth/data}/config/cic.ini | 4 +- apps/cic-eth/cic_eth/data/config/database.ini | 10 ++ apps/cic-eth/cic_eth/data/config/eth.ini | 2 + .../docker => cic_eth/data/config}/redis.ini | 3 +- apps/cic-eth/cic_eth/data/config/signer.ini | 2 + apps/cic-eth/cic_eth/data/config/syncer.ini | 4 + .../docker => cic_eth/data/config}/tasks.ini | 0 apps/cic-eth/cic_eth/runnable/create.py | 58 ++----- apps/cic-eth/cic_eth/runnable/ctrl.py | 50 ++---- .../cic_eth/runnable/daemons/dispatcher.py | 54 ++---- .../cic-eth/cic_eth/runnable/daemons/retry.py | 67 +++---- .../cic_eth/runnable/daemons/tasker.py | 164 +++++------------- .../cic_eth/runnable/daemons/tracker.py | 95 +++++----- apps/cic-eth/cic_eth/runnable/info.py | 41 +---- apps/cic-eth/cic_eth/runnable/resend.py | 59 ++----- apps/cic-eth/cic_eth/runnable/tag.py | 46 ++--- apps/cic-eth/cic_eth/runnable/transfer.py | 78 +++------ apps/cic-eth/cic_eth/sync/retry.py | 6 +- apps/cic-eth/cic_eth/version.py | 2 +- apps/cic-eth/config/bancor.ini | 2 - apps/cic-eth/config/celery.ini | 4 - apps/cic-eth/config/custody.ini | 2 - apps/cic-eth/config/database.ini | 6 +- apps/cic-eth/config/dispatcer.ini | 2 - apps/cic-eth/config/docker/bancor.ini | 2 - apps/cic-eth/config/docker/celery.ini | 4 - apps/cic-eth/config/docker/cic.ini | 8 - apps/cic-eth/config/docker/custody.ini | 2 - apps/cic-eth/config/docker/dispatcer.ini | 2 - apps/cic-eth/config/docker/eth.ini | 3 - apps/cic-eth/config/docker/signer.ini | 5 - apps/cic-eth/config/docker/ssl.ini | 6 - apps/cic-eth/config/docker/syncer.ini | 3 - apps/cic-eth/config/eth.ini | 3 - apps/cic-eth/config/lash/bancor.ini | 3 - apps/cic-eth/config/lash/database.ini | 8 - apps/cic-eth/config/lash/eth.ini | 3 - apps/cic-eth/config/lash/redis.ini | 3 - apps/cic-eth/config/lash/signer.ini | 2 - apps/cic-eth/config/lash/ssl.ini | 6 - apps/cic-eth/config/lash/syncer.ini | 2 - apps/cic-eth/config/redis.ini | 4 - apps/cic-eth/config/signer.ini | 5 - apps/cic-eth/config/ssl.ini | 6 - apps/cic-eth/config/syncer.ini | 3 - apps/cic-eth/config/tasks.ini | 3 - apps/cic-eth/config/test/chain.ini | 2 + apps/cic-eth/docker/Dockerfile | 6 + apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/services_requirements.txt | 14 +- apps/cic-eth/setup.cfg | 2 + apps/cic-eth/test_requirements.txt | 2 +- apps/cic-eth/tests/check/test_check_gas.py | 2 +- apps/cic-eth/tests/check/test_check_signer.py | 2 +- apps/cic-eth/tools_requirements.txt | 4 +- apps/cic-ussd/requirements.txt | 8 +- .../config_template/celery.ini | 3 - .../config_template/chain.ini | 2 - .../config_template/cic.ini | 7 - .../config_template/database.ini | 8 - .../config_template/dev.ini | 33 ---- .../config_template/empty/config.ini | 0 .../config_template/eth.ini | 3 - .../config_template/legacy.ini | 7 + .../config_template/meta.ini | 2 - .../config_template/pgp.ini | 4 - .../config_template/redis.ini | 4 - .../config_template/rpc.ini | 5 - .../config_template/signer.ini | 3 - .../config_template/syncer.ini | 2 - .../config_template/tasks.ini | 5 - .../config_template/token.ini | 8 - .../config_template/wallet.ini | 3 - apps/contract-migration/requirements.txt | 20 +-- apps/contract-migration/reset.sh | 20 ++- apps/contract-migration/seed_cic_eth.sh | 41 +++-- apps/data-seeding/requirements.txt | 22 +-- docker-compose.yml | 50 ++++-- 122 files changed, 984 insertions(+), 832 deletions(-) create mode 100644 apps/cic-cache/MANIFEST.in create mode 100644 apps/cic-cache/cic_cache/cli/__init__.py create mode 100644 apps/cic-cache/cic_cache/cli/arg.py create mode 100644 apps/cic-cache/cic_cache/cli/base.py create mode 100644 apps/cic-cache/cic_cache/cli/celery.py create mode 100644 apps/cic-cache/cic_cache/cli/chain.py create mode 100644 apps/cic-cache/cic_cache/cli/config.py create mode 100644 apps/cic-cache/cic_cache/cli/registry.py create mode 100644 apps/cic-cache/cic_cache/cli/rpc.py create mode 100644 apps/cic-cache/cic_cache/data/config/celery.ini create mode 100644 apps/cic-cache/cic_cache/data/config/cic.ini create mode 100644 apps/cic-cache/cic_cache/data/config/database.ini create mode 100644 apps/cic-cache/cic_cache/data/config/signer.ini create mode 100644 apps/cic-cache/cic_cache/data/config/syncer.ini delete mode 100644 apps/cic-cache/config/bancor.ini delete mode 100644 apps/cic-cache/config/docker/bancor.ini delete mode 100644 apps/cic-cache/config/docker/eth.ini delete mode 100644 apps/cic-cache/config/eth.ini delete mode 100644 apps/cic-cache/config/syncer.ini create mode 100644 apps/cic-eth/cic_eth/cli/__init__.py create mode 100644 apps/cic-eth/cic_eth/cli/arg.py create mode 100644 apps/cic-eth/cic_eth/cli/base.py create mode 100644 apps/cic-eth/cic_eth/cli/celery.py create mode 100644 apps/cic-eth/cic_eth/cli/chain.py create mode 100644 apps/cic-eth/cic_eth/cli/config.py create mode 100644 apps/cic-eth/cic_eth/cli/rpc.py create mode 100644 apps/cic-eth/cic_eth/data/config/celery.ini rename apps/cic-eth/{ => cic_eth/data}/config/cic.ini (66%) create mode 100644 apps/cic-eth/cic_eth/data/config/database.ini create mode 100644 apps/cic-eth/cic_eth/data/config/eth.ini rename apps/cic-eth/{config/docker => cic_eth/data/config}/redis.ini (54%) create mode 100644 apps/cic-eth/cic_eth/data/config/signer.ini create mode 100644 apps/cic-eth/cic_eth/data/config/syncer.ini rename apps/cic-eth/{config/docker => cic_eth/data/config}/tasks.ini (100%) delete mode 100644 apps/cic-eth/config/bancor.ini delete mode 100644 apps/cic-eth/config/celery.ini delete mode 100644 apps/cic-eth/config/custody.ini delete mode 100644 apps/cic-eth/config/dispatcer.ini delete mode 100644 apps/cic-eth/config/docker/bancor.ini delete mode 100644 apps/cic-eth/config/docker/celery.ini delete mode 100644 apps/cic-eth/config/docker/cic.ini delete mode 100644 apps/cic-eth/config/docker/custody.ini delete mode 100644 apps/cic-eth/config/docker/dispatcer.ini delete mode 100644 apps/cic-eth/config/docker/eth.ini delete mode 100644 apps/cic-eth/config/docker/signer.ini delete mode 100644 apps/cic-eth/config/docker/ssl.ini delete mode 100644 apps/cic-eth/config/docker/syncer.ini delete mode 100644 apps/cic-eth/config/eth.ini delete mode 100644 apps/cic-eth/config/lash/bancor.ini delete mode 100644 apps/cic-eth/config/lash/database.ini delete mode 100644 apps/cic-eth/config/lash/eth.ini delete mode 100644 apps/cic-eth/config/lash/redis.ini delete mode 100644 apps/cic-eth/config/lash/signer.ini delete mode 100644 apps/cic-eth/config/lash/ssl.ini delete mode 100644 apps/cic-eth/config/lash/syncer.ini delete mode 100644 apps/cic-eth/config/redis.ini delete mode 100644 apps/cic-eth/config/signer.ini delete mode 100644 apps/cic-eth/config/ssl.ini delete mode 100644 apps/cic-eth/config/syncer.ini delete mode 100644 apps/cic-eth/config/tasks.ini create mode 100644 apps/cic-eth/config/test/chain.ini delete mode 100644 apps/contract-migration/config_template/chain.ini delete mode 100644 apps/contract-migration/config_template/cic.ini delete mode 100644 apps/contract-migration/config_template/database.ini delete mode 100644 apps/contract-migration/config_template/dev.ini create mode 100644 apps/contract-migration/config_template/empty/config.ini delete mode 100644 apps/contract-migration/config_template/eth.ini create mode 100644 apps/contract-migration/config_template/legacy.ini delete mode 100644 apps/contract-migration/config_template/meta.ini delete mode 100644 apps/contract-migration/config_template/pgp.ini delete mode 100644 apps/contract-migration/config_template/redis.ini delete mode 100644 apps/contract-migration/config_template/rpc.ini delete mode 100644 apps/contract-migration/config_template/signer.ini delete mode 100644 apps/contract-migration/config_template/syncer.ini delete mode 100644 apps/contract-migration/config_template/tasks.ini delete mode 100644 apps/contract-migration/config_template/token.ini delete mode 100644 apps/contract-migration/config_template/wallet.ini diff --git a/apps/cic-cache/.coveragerc b/apps/cic-cache/.coveragerc index a8162630..1fd1312c 100644 --- a/apps/cic-cache/.coveragerc +++ b/apps/cic-cache/.coveragerc @@ -4,3 +4,4 @@ omit = scripts/* cic_cache/db/migrations/* cic_cache/version.py + cic_cache/cli diff --git a/apps/cic-cache/MANIFEST.in b/apps/cic-cache/MANIFEST.in new file mode 100644 index 00000000..2630c39c --- /dev/null +++ b/apps/cic-cache/MANIFEST.in @@ -0,0 +1 @@ +include *requirements.txt cic_cache/data/config/* diff --git a/apps/cic-cache/cic_cache/cli/__init__.py b/apps/cic-cache/cic_cache/cli/__init__.py new file mode 100644 index 00000000..4952837d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/__init__.py @@ -0,0 +1,15 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp +from .registry import ( + connect_registry, + connect_token_registry, + connect_declarator, + ) diff --git a/apps/cic-cache/cic_cache/cli/arg.py b/apps/cic-cache/cic_cache/cli/arg.py new file mode 100644 index 00000000..43428cb1 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/arg.py @@ -0,0 +1,20 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') diff --git a/apps/cic-cache/cic_cache/cli/base.py b/apps/cic-cache/cic_cache/cli/base.py new file mode 100644 index 00000000..84e4ab8d --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 +# REDIS = 16 +# REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-cache/cic_cache/cli/celery.py b/apps/cic-cache/cic_cache/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-cache/cic_cache/cli/chain.py b/apps/cic-cache/cic_cache/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-cache/cic_cache/cli/config.py b/apps/cic-cache/cic_cache/cli/config.py new file mode 100644 index 00000000..c21d46dc --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} +# if local_arg_flags & CICFlag.REDIS: +# local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') +# local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') +# local_args_override['REDIS_DB'] = getattr(args, 'redis_db') +# local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + +# if local_arg_flags & CICFlag.REDIS_CALLBACK: +# config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') +# config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-cache/cic_cache/cli/registry.py b/apps/cic-cache/cic_cache/cli/registry.py new file mode 100644 index 00000000..1d3e0d31 --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/registry.py @@ -0,0 +1,33 @@ +# standard imports +import logging + +# external imports +from cic_eth_registry import CICRegistry +from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup +from cic_eth_registry.lookup.tokenindex import TokenIndexLookup +from chainlib.eth.constant import ZERO_ADDRESS + +logg = logging.getLogger() + + +def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address) + logg.debug('using token registry address {}'.format(token_registry_address)) + lookup = TokenIndexLookup(chain_spec, token_registry_address) + CICRegistry.add_lookup(lookup) + + +def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS): + registry = CICRegistry(chain_spec, conn) + declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address) + logg.debug('using declarator address {}'.format(declarator_address)) + lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses) + CICRegistry.add_lookup(lookup) + + +def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS): + CICRegistry.address = registry_address + registry = CICRegistry(chain_spec, conn) + registry_address = registry.by_name('ContractRegistry', sender_address=sender_address) + return registry diff --git a/apps/cic-cache/cic_cache/cli/rpc.py b/apps/cic-cache/cic_cache/cli/rpc.py new file mode 100644 index 00000000..7f6500ae --- /dev/null +++ b/apps/cic-cache/cic_cache/cli/rpc.py @@ -0,0 +1,43 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import EthUnixSignerConnection +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return RPCConnection.connect(self.chain_spec, 'default') + + + @staticmethod + def from_config(config): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + if config.get('SIGNER_PROVIDER'): + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer') + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + + diff --git a/apps/cic-cache/cic_cache/data/config/celery.ini b/apps/cic-cache/cic_cache/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-cache/cic_cache/data/config/cic.ini b/apps/cic-cache/cic_cache/data/config/cic.ini new file mode 100644 index 00000000..b4275352 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/cic.ini @@ -0,0 +1,4 @@ +[cic] +registry_address = +trust_address = +health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas diff --git a/apps/cic-cache/cic_cache/data/config/database.ini b/apps/cic-cache/cic_cache/data/config/database.ini new file mode 100644 index 00000000..e32e62f3 --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = cic-cache +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-cache/cic_cache/data/config/signer.ini b/apps/cic-cache/cic_cache/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-cache/cic_cache/data/config/syncer.ini b/apps/cic-cache/cic_cache/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-cache/cic_cache/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py index 55a5ecca..1b845ff6 100644 --- a/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py +++ b/apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py @@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243 """ from alembic import op import sqlalchemy as sa -from chainsyncer.db.migrations.sqlalchemy import ( +from chainsyncer.db.migrations.default.export import ( chainsyncer_upgrade, chainsyncer_downgrade, ) diff --git a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py index ec03f6b7..50876e5f 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/tracker.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/tracker.py @@ -8,15 +8,7 @@ import sys import re # external imports -import confini -import celery import sqlalchemy -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry import CICRegistry from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec @@ -34,6 +26,7 @@ from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase # local imports +import cic_cache.cli from cic_cache.db import ( dsn_from_config, add_tag, @@ -43,32 +36,36 @@ from cic_cache.runnable.daemons.filters import ( FaucetFilter, ) -script_dir = os.path.realpath(os.path.dirname(__file__)) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process args +arg_flags = cic_cache.cli.argflag_std_read +local_arg_flags = cic_cache.cli.argflag_local_sync +argparser = cic_cache.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +# process config +config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# connect to database dsn = dsn_from_config(config) - SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_cache.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None +try: + registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) +except UnknownContractError as e: + logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) + sys.exit(1) +logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) def register_filter_tags(filters, session): @@ -95,14 +92,12 @@ def main(): syncers = [] - #if SQLBackend.first(chain_spec): - # backend = SQLBackend.initial(chain_spec, block_offset) syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.get('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -112,10 +107,10 @@ def main(): logg.info('resuming sync session {}'.format(syncer_backend)) for syncer_backend in syncer_backends: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface)) syncer_backend = SQLBackend.live(chain_spec, block_offset+1) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface)) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: diff --git a/apps/cic-cache/config/bancor.ini b/apps/cic-cache/config/bancor.ini deleted file mode 100644 index c1591774..00000000 --- a/apps/cic-cache/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = diff --git a/apps/cic-cache/config/cic.ini b/apps/cic-cache/config/cic.ini index cbb3fb2e..48c32fad 100644 --- a/apps/cic-cache/config/cic.ini +++ b/apps/cic-cache/config/cic.ini @@ -1,4 +1,3 @@ [cic] registry_address = -chain_spec = trust_address = diff --git a/apps/cic-cache/config/docker/bancor.ini b/apps/cic-cache/config/docker/bancor.ini deleted file mode 100644 index 7e53c791..00000000 --- a/apps/cic-cache/config/docker/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = -dir = /usr/local/share/bancor diff --git a/apps/cic-cache/config/docker/cic.ini b/apps/cic-cache/config/docker/cic.ini index 1fd98612..af112724 100644 --- a/apps/cic-cache/config/docker/cic.ini +++ b/apps/cic-cache/config/docker/cic.ini @@ -1,4 +1,3 @@ [cic] -chain_spec = registry_address = trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C diff --git a/apps/cic-cache/config/docker/eth.ini b/apps/cic-cache/config/docker/eth.ini deleted file mode 100644 index 321384f8..00000000 --- a/apps/cic-cache/config/docker/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = http://localhost:63545 diff --git a/apps/cic-cache/config/docker/syncer.ini b/apps/cic-cache/config/docker/syncer.ini index c811bccc..255dc00f 100644 --- a/apps/cic-cache/config/docker/syncer.ini +++ b/apps/cic-cache/config/docker/syncer.ini @@ -1,3 +1,4 @@ [syncer] loop_interval = 1 -history_start = 0 +offset = 0 +no_history = 0 diff --git a/apps/cic-cache/config/eth.ini b/apps/cic-cache/config/eth.ini deleted file mode 100644 index 98b831e5..00000000 --- a/apps/cic-cache/config/eth.ini +++ /dev/null @@ -1,2 +0,0 @@ -[eth] -provider = ws://localhost:8545 diff --git a/apps/cic-cache/config/syncer.ini b/apps/cic-cache/config/syncer.ini deleted file mode 100644 index f26739c7..00000000 --- a/apps/cic-cache/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[syncer] -loop_interval = 5 -history_start = 0 diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 54b7a895..e01c9bbe 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -1,13 +1,14 @@ -cic-base~=0.2.0a4 alembic==1.4.2 -confini>=0.3.6rc3,<0.5.0 +confini>=0.3.6rc4,<0.5.0 uwsgi==2.0.19.1 -moolb~=0.1.0 -cic-eth-registry~=0.5.6a2 +moolb~=0.1.1b2 +cic-eth-registry~=0.5.8a1 SQLAlchemy==1.3.20 semver==2.13.0 psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 -chainsyncer[sql]~=0.0.3a5 -erc20-faucet~=0.2.2a2 +chainsyncer[sql]>=0.0.6a1,<0.1.0 +erc20-faucet~=0.2.4a1 +chainlib>=0.0.7a1,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 diff --git a/apps/cic-cache/setup.cfg b/apps/cic-cache/setup.cfg index 4616a475..b6de6a9c 100644 --- a/apps/cic-cache/setup.cfg +++ b/apps/cic-cache/setup.cfg @@ -23,11 +23,13 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_cache cic_cache.tasks cic_cache.db cic_cache.db.models + cic_cache.cli cic_cache.runnable cic_cache.runnable.daemons cic_cache.runnable.daemons.filters diff --git a/apps/cic-cache/test_requirements.txt b/apps/cic-cache/test_requirements.txt index f602fe01..9d46b791 100644 --- a/apps/cic-cache/test_requirements.txt +++ b/apps/cic-cache/test_requirements.txt @@ -6,5 +6,5 @@ sqlparse==0.4.1 pytest-celery==0.0.0a1 eth_tester==0.5.0b3 py-evm==0.3.0a20 -cic_base[full]==0.1.3a3+build.984b5cff -sarafu-faucet~=0.0.4a1 +sarafu-faucet~=0.0.5a2 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 diff --git a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py index ff63b895..eef7ecda 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py +++ b/apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py @@ -46,7 +46,7 @@ def get_adjusted_balance(self, token_symbol, amount, timestamp): def aux_setup(rpc, config, sender_address=ZERO_ADDRESS): - chain_spec_str = config.get('CIC_CHAIN_SPEC') + chain_spec_str = config.get('CHAIN_SPEC') chain_spec = ChainSpec.from_chain_str(chain_spec_str) token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') diff --git a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt index 9d7f6f9a..764056bb 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/requirements.txt @@ -1,5 +1,5 @@ celery==4.4.7 -erc20-demurrage-token~=0.0.2a3 -cic-eth-registry~=0.5.6a1 -chainlib~=0.0.5a1 -cic_eth~=0.12.0a2 +erc20-demurrage-token~=0.0.3a1 +cic-eth-registry~=0.5.8a1 +chainlib~=0.0.7a1 +cic_eth~=0.12.2a4 diff --git a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg index 69a41b7c..0e32d37f 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg +++ b/apps/cic-eth-aux/erc20-demurrage-token/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = cic-eth-aux-erc20-demurrage-token -version = 0.0.2a4 +version = 0.0.2a6 description = cic-eth tasks supporting erc20 demurrage token author = Louis Holbrook author_email = dev@holbrook.no diff --git a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt index 0faedff4..a273b82a 100644 --- a/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt +++ b/apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt @@ -5,8 +5,7 @@ pytest-cov==2.10.1 eth-tester==0.5.0b3 py-evm==0.3.0a20 SQLAlchemy==1.3.20 -cic-eth~=0.12.0a1 liveness~=0.0.1a7 -eth-accounts-index==0.0.12a1 -eth-contract-registry==0.5.6a1 -eth-address-index==0.1.2a1 +eth-accounts-index==0.1.1a1 +eth-contract-registry==0.5.8a1 +eth-address-index==0.2.1a1 diff --git a/apps/cic-eth/.coveragerc b/apps/cic-eth/.coveragerc index 47acdac6..8c586e19 100644 --- a/apps/cic-eth/.coveragerc +++ b/apps/cic-eth/.coveragerc @@ -6,4 +6,5 @@ omit = cic_eth/sync/head.py cic_eth/sync/mempool.py cic_eth/queue/state.py + cic_eth/cli *redis*.py diff --git a/apps/cic-eth/MANIFEST.in b/apps/cic-eth/MANIFEST.in index 5a9554ac..52d3a6f4 100644 --- a/apps/cic-eth/MANIFEST.in +++ b/apps/cic-eth/MANIFEST.in @@ -1,2 +1,2 @@ -include *requirements.txt config/test/* +include *requirements.txt config/test/* cic_eth/data/config/* diff --git a/apps/cic-eth/cic_eth/check/gas.py b/apps/cic-eth/cic_eth/check/gas.py index b1646fd6..4add981b 100644 --- a/apps/cic-eth/cic_eth/check/gas.py +++ b/apps/cic-eth/cic_eth/check/gas.py @@ -21,7 +21,7 @@ def health(*args, **kwargs): session = SessionBase.create_session() config = kwargs['config'] - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec)) try: diff --git a/apps/cic-eth/cic_eth/check/signer.py b/apps/cic-eth/cic_eth/check/signer.py index 87b7785f..0395f0a1 100644 --- a/apps/cic-eth/cic_eth/check/signer.py +++ b/apps/cic-eth/cic_eth/check/signer.py @@ -15,7 +15,7 @@ logg = logging.getLogger().getChild(__name__) def health(*args, **kwargs): blocked = True max_attempts = 5 - conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer') + conn = RPCConnection.connect(kwargs['config'].get('CHAIN_SPEC'), tag='signer') for i in range(max_attempts): idx = i + 1 logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts)) diff --git a/apps/cic-eth/cic_eth/cli/__init__.py b/apps/cic-eth/cic_eth/cli/__init__.py new file mode 100644 index 00000000..60eb0fb1 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/__init__.py @@ -0,0 +1,10 @@ +# local imports +from .base import * +from .chain import ( + EthChainInterface, + chain_interface, + ) +from .rpc import RPC +from .arg import ArgumentParser +from .config import Config +from .celery import CeleryApp diff --git a/apps/cic-eth/cic_eth/cli/arg.py b/apps/cic-eth/cic_eth/cli/arg.py new file mode 100644 index 00000000..91fcc071 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/arg.py @@ -0,0 +1,31 @@ +# external imports +from chainlib.eth.cli import ArgumentParser as BaseArgumentParser + +# local imports +from .base import ( + CICFlag, + Flag, + ) + + +class ArgumentParser(BaseArgumentParser): + + def process_local_flags(self, local_arg_flags): + if local_arg_flags & CICFlag.REDIS: + self.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') + self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') + self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use') + if local_arg_flags & CICFlag.REDIS_CALLBACK: + self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') + self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') + self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout') + if local_arg_flags & CICFlag.CELERY: + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + if local_arg_flags & CICFlag.SYNCER: + self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') + self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') + if local_arg_flags & CICFlag.CHAIN: + self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address') + + + diff --git a/apps/cic-eth/cic_eth/cli/base.py b/apps/cic-eth/cic_eth/cli/base.py new file mode 100644 index 00000000..b8b3ef2f --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/base.py @@ -0,0 +1,31 @@ +# standard imports +import enum + +# external imports +from chainlib.eth.cli import ( + argflag_std_read, + argflag_std_write, + argflag_std_base, + Flag, + ) + +class CICFlag(enum.IntEnum): + + # celery - nibble 1 + CELERY = 1 + + # redis - nibble 2 + REDIS = 16 + REDIS_CALLBACK = 32 + + # chain - nibble 3 + CHAIN = 256 + + # sync - nibble 4 + SYNCER = 4096 + + +argflag_local_task = CICFlag.CELERY +argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK +argflag_local_chain = CICFlag.CHAIN +argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN diff --git a/apps/cic-eth/cic_eth/cli/celery.py b/apps/cic-eth/cic_eth/cli/celery.py new file mode 100644 index 00000000..16180d4a --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/celery.py @@ -0,0 +1,24 @@ +# standard imports +import logging + +# external imports +import celery + +logg = logging.getLogger(__name__) + + +class CeleryApp: + + @classmethod + def from_config(cls, config): + backend_url = config.get('CELERY_RESULT_URL') + broker_url = config.get('CELERY_BROKER_URL') + celery_app = None + if backend_url != None: + celery_app = celery.Celery(broker=broker_url, backend=backend_url) + logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url)) + else: + celery_app = celery.Celery(broker=broker_url) + logg.info('creating celery app without results backend on {}'.format(broker_url)) + + return celery_app diff --git a/apps/cic-eth/cic_eth/cli/chain.py b/apps/cic-eth/cic_eth/cli/chain.py new file mode 100644 index 00000000..100139e7 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/chain.py @@ -0,0 +1,21 @@ +# external imports +from chainlib.eth.block import ( + block_by_number, + Block, + ) +from chainlib.eth.tx import ( + receipt, + Tx, + ) +from chainlib.interface import ChainInterface + + +class EthChainInterface(ChainInterface): + + def __init__(self): + self._tx_receipt = receipt + self._block_by_number = block_by_number + self._block_from_src = Block.from_src + self._src_normalize = Tx.src_normalize + +chain_interface = EthChainInterface() diff --git a/apps/cic-eth/cic_eth/cli/config.py b/apps/cic-eth/cic_eth/cli/config.py new file mode 100644 index 00000000..8bfadc67 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/config.py @@ -0,0 +1,63 @@ +# standard imports +import os +import logging + +# external imports +from chainlib.eth.cli import ( + Config as BaseConfig, + Flag, + ) + +# local imports +from .base import CICFlag + +script_dir = os.path.dirname(os.path.realpath(__file__)) + +logg = logging.getLogger(__name__) + + +class Config(BaseConfig): + + local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config') + + @classmethod + def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None): + expanded_base_config_dir = [cls.local_base_config_dir] + if base_config_dir != None: + if isinstance(base_config_dir, str): + base_config_dir = [base_config_dir] + for d in base_config_dir: + expanded_base_config_dir.append(d) + config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None) + + local_args_override = {} + if local_arg_flags & CICFlag.REDIS: + local_args_override['REDIS_HOST'] = getattr(args, 'redis_host') + local_args_override['REDIS_PORT'] = getattr(args, 'redis_port') + local_args_override['REDIS_DB'] = getattr(args, 'redis_db') + local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout') + + if local_arg_flags & CICFlag.CHAIN: + local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address') + + if local_arg_flags & CICFlag.CELERY: + local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue') + + if local_arg_flags & CICFlag.SYNCER: + local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset') + local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history') + + config.dict_override(local_args_override, 'local cli args') + + if local_arg_flags & CICFlag.REDIS_CALLBACK: + config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK') + config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK') + + if local_arg_flags & CICFlag.CELERY: + config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True) + + logg.debug('config loaded:\n{}'.format(config)) + + return config + + diff --git a/apps/cic-eth/cic_eth/cli/rpc.py b/apps/cic-eth/cic_eth/cli/rpc.py new file mode 100644 index 00000000..cc073767 --- /dev/null +++ b/apps/cic-eth/cic_eth/cli/rpc.py @@ -0,0 +1,86 @@ +# standard imports +import logging + +# external imports +from chainlib.connection import ( + RPCConnection, + ConnType, + ) +from chainlib.eth.connection import ( + EthUnixSignerConnection, + EthHTTPSignerConnection, + ) +from chainlib.chain import ChainSpec + +logg = logging.getLogger(__name__) + + +class RPC: + + def __init__(self, chain_spec, rpc_provider, signer_provider=None): + self.chain_spec = chain_spec + self.rpc_provider = rpc_provider + self.signer_provider = signer_provider + + + def get_default(self): + return RPCConnection.connect(self.chain_spec, 'default') + + + @staticmethod + def from_config(config, use_signer=False): + chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + if use_signer: + + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') + RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') + RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) + logg.info('set up rpc: {}'.format(rpc)) + return rpc + + + def __str__(self): + return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider) + + +# TOOD: re-implement file backend option from omittec code: +#broker = config.get('CELERY_BROKER_URL') +#if broker[:4] == 'file': +# bq = tempfile.mkdtemp() +# bp = tempfile.mkdtemp() +# conf_update = { +# 'broker_url': broker, +# 'broker_transport_options': { +# 'data_folder_in': bq, +# 'data_folder_out': bq, +# 'data_folder_processed': bp, +# }, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) +#else: +# conf_update = { +# 'broker_url': broker, +# } +# if config.true('CELERY_DEBUG'): +# conf_update['result_extended'] = True +# current_app.conf.update(conf_update) +# +#result = config.get('CELERY_RESULT_URL') +#if result[:4] == 'file': +# rq = tempfile.mkdtemp() +# current_app.conf.update({ +# 'result_backend': 'file://{}'.format(rq), +# }) +# logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) +#else: +# current_app.conf.update({ +# 'result_backend': result, +# }) +# diff --git a/apps/cic-eth/cic_eth/data/config/celery.ini b/apps/cic-eth/cic_eth/data/config/celery.ini new file mode 100644 index 00000000..f2ad10ab --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/celery.ini @@ -0,0 +1,5 @@ +[celery] +broker_url = redis://localhost:6379 +result_url = +queue = cic-eth +debug = 0 diff --git a/apps/cic-eth/config/cic.ini b/apps/cic-eth/cic_eth/data/config/cic.ini similarity index 66% rename from apps/cic-eth/config/cic.ini rename to apps/cic-eth/cic_eth/data/config/cic.ini index d3840fac..103566ff 100644 --- a/apps/cic-eth/config/cic.ini +++ b/apps/cic-eth/cic_eth/data/config/cic.ini @@ -1,8 +1,6 @@ [cic] registry_address = -chain_spec = evm:bloxberg:8996 -tx_retry_delay = trust_address = -default_token_symbol = GFT +default_token_symbol = health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas run_dir = /run diff --git a/apps/cic-eth/cic_eth/data/config/database.ini b/apps/cic-eth/cic_eth/data/config/database.ini new file mode 100644 index 00000000..7a295a56 --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/database.ini @@ -0,0 +1,10 @@ +[database] +engine = +driver = +host = +port = +name = +user = +password = +debug = 0 +pool_size = 0 diff --git a/apps/cic-eth/cic_eth/data/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini new file mode 100644 index 00000000..3bba24bf --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -0,0 +1,2 @@ +[eth] +gas_gifter_minimum_balance = 10000000000000000000 diff --git a/apps/cic-eth/config/docker/redis.ini b/apps/cic-eth/cic_eth/data/config/redis.ini similarity index 54% rename from apps/cic-eth/config/docker/redis.ini rename to apps/cic-eth/cic_eth/data/config/redis.ini index ea321378..f836ebfd 100644 --- a/apps/cic-eth/config/docker/redis.ini +++ b/apps/cic-eth/cic_eth/data/config/redis.ini @@ -1,4 +1,5 @@ [redis] host = localhost -port = 63379 +port = 6379 db = 0 +timeout = 20.0 diff --git a/apps/cic-eth/cic_eth/data/config/signer.ini b/apps/cic-eth/cic_eth/data/config/signer.ini new file mode 100644 index 00000000..21327bac --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/signer.ini @@ -0,0 +1,2 @@ +[signer] +provider = diff --git a/apps/cic-eth/cic_eth/data/config/syncer.ini b/apps/cic-eth/cic_eth/data/config/syncer.ini new file mode 100644 index 00000000..255dc00f --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/syncer.ini @@ -0,0 +1,4 @@ +[syncer] +loop_interval = 1 +offset = 0 +no_history = 0 diff --git a/apps/cic-eth/config/docker/tasks.ini b/apps/cic-eth/cic_eth/data/config/tasks.ini similarity index 100% rename from apps/cic-eth/config/docker/tasks.ini rename to apps/cic-eth/cic_eth/data/config/tasks.ini diff --git a/apps/cic-eth/cic_eth/runnable/create.py b/apps/cic-eth/cic_eth/runnable/create.py index 973ac2f5..31dd1991 100644 --- a/apps/cic-eth/cic_eth/runnable/create.py +++ b/apps/cic-eth/cic_eth/runnable/create.py @@ -7,54 +7,30 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home +from chainlib.chain import ChainSpec # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) -logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) +logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'no_register': None, } -config.dict_override(args_override, 'cli') - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): redis_host = config.get('REDIS_HOST') @@ -68,20 +44,20 @@ def main(): ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE'), ) - register = not args.no_register + register = not config.get('_NO_REGISTER') logg.debug('register {}'.format(register)) t = api.create_account(register=register) ps.get_message() try: - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) except TimeoutError as e: sys.stderr.write('got no new address from cic-eth before timeout: {}\n'.format(e)) sys.exit(1) diff --git a/apps/cic-eth/cic_eth/runnable/ctrl.py b/apps/cic-eth/cic_eth/runnable/ctrl.py index fc35548e..5f260d16 100644 --- a/apps/cic-eth/cic_eth/runnable/ctrl.py +++ b/apps/cic-eth/cic_eth/runnable/ctrl.py @@ -12,64 +12,38 @@ from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.address import is_checksum_address # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi from cic_eth.db.enum import LockEnum logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task | cic_eth.cli.argflag_local_chain +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') +argparser.process_local_flags(local_arg_flags) def process_lock_args(argparser): argparser.add_argument('flags', type=str, help='Flags to manipulate') argparser.add_argument('address', default=ZERO_ADDRESS, nargs='?', type=str, help='Ethereum address to unlock,') -sub = argparser.add_subparsers() +sub = argparser.add_subparsers(help='') sub.dest = "command" sub_lock = sub.add_parser('lock', help='Set or reset locks') sub_unlock = sub.add_parser('unlock', help='Set or reset locks') process_lock_args(sub_lock) process_lock_args(sub_unlock) - args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -# override args -config.dict_override(args_override, 'cli') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -queue = args.q - -chain_spec = None -if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::': - chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) admin_api = AdminApi(None) @@ -100,7 +74,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('unlock {} on {} task {}'.format(flags, args.address, t)) @@ -119,7 +93,7 @@ def main(): args.address, flags, ], - queue=queue, + queue=config.get('CELERY_QUEUE'), ) t = s.apply_async() logg.debug('lock {} on {} task {}'.format(flags, args.address, t)) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py index 7e22e911..64b51937 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/dispatcher.py @@ -8,8 +8,7 @@ import sys import re import datetime -# third-party imports -import confini +# external imports import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -24,7 +23,7 @@ from chainqueue.error import NotLocalTxError from chainqueue.sql.state import set_reserved # local imports -import cic_eth +import cic_eth.cli from cic_eth.db import SessionBase from cic_eth.db.enum import LockEnum from cic_eth.db import dsn_from_config @@ -39,51 +38,30 @@ from cic_eth.error import ( logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -config_dir = os.path.join('/usr/local/etc/cic-eth') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', default='http://localhost:8545', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'ETH_PROVIDER': getattr(args, 'p'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() run = True + class DispatchSyncer: yield_delay = 0.0005 diff --git a/apps/cic-eth/cic_eth/runnable/daemons/retry.py b/apps/cic-eth/cic_eth/runnable/daemons/retry.py index 8974c33d..354fd5bc 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/retry.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/retry.py @@ -6,7 +6,6 @@ import argparse import re # external imports -import confini import celery from cic_eth_registry import CICRegistry from chainlib.chain import ChainSpec @@ -14,6 +13,7 @@ from chainlib.connection import RPCConnection from chainsyncer.filter import SyncFilter # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.db import SessionBase from cic_eth.admin.ctrl import lock_send @@ -25,66 +25,41 @@ from cic_eth.stat import init_chain_stat logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync | cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--batch-size', dest='batch_size', type=int, default=50, help='max amount of txs to resend per iteration') -argparser.add_argument('--retry-delay', dest='retry_delay', type=int, help='seconds to wait for retrying a transaction that is marked as sent') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -args = argparser.parse_args(sys.argv[1:]) +argparser.add_argument('--retry-delay', dest='retry_delay', type=int, default=20, help='seconds to wait for retrying a transaction that is marked as sent') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() - -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) - -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -# override args -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_TX_RETRY_DELAY': getattr(args, 'retry_delay'), +extra_args = { + 'retry_delay': 'RETRY_DELAY', + 'batch_size': 'RETRY_BATCH_SIZE', } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.batch_size, '_BATCH_SIZE', True) - -app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL')) - -queue = args.q - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) - -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, tag='default') +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) +# connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG')) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) + +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() + def main(): - conn = RPCConnection.connect(chain_spec, 'default') - - straggler_delay = int(config.get('CIC_TX_RETRY_DELAY')) + straggler_delay = int(config.get('RETRY_DELAY')) loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: stat = init_chain_stat(conn) loop_interval = stat.block_average() - syncer = RetrySyncer(conn, chain_spec, straggler_delay, batch_size=config.get('_BATCH_SIZE')) + syncer = RetrySyncer(conn, chain_spec, cic_eth.cli.chain_interface, straggler_delay, batch_size=config.get('RETRY_BATCH_SIZE')) syncer.backend.set(0, 0) - fltr = StragglerFilter(chain_spec, queue=queue) + fltr = StragglerFilter(chain_spec, queue=config.get('CELERY_QUEUE')) syncer.add_filter(fltr) syncer.loop(int(loop_interval), conn) diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py index 837da5cb..fab470f3 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tasker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tasker.py @@ -21,14 +21,17 @@ from chainlib.eth.connection import ( EthUnixSignerConnection, EthHTTPSignerConnection, ) +from chainlib.eth.address import to_checksum_address from chainlib.chain import ChainSpec from chainqueue.db.models.otx import Otx from cic_eth_registry.error import UnknownContractError from cic_eth_registry.erc20 import ERC20Token +from hexathon import add_0x import liveness.linux # local imports +import cic_eth.cli from cic_eth.eth import ( erc20, tx, @@ -70,114 +73,53 @@ from cic_eth.task import BaseTask logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -script_dir = os.path.dirname(os.path.realpath(__file__)) - -config_dir = os.path.join('/usr/local/etc/cic-eth') - -argparser = argparse.ArgumentParser() -argparser.add_argument('-p', '--provider', dest='p', type=str, help='rpc provider') -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('-q', type=str, default='cic-eth', help='queue name for worker tasks') -argparser.add_argument('-r', type=str, help='CIC registry address') +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_task +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use') argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path') argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() -# override args -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'CIC_REGISTRY_ADDRESS': getattr(args, 'r'), - 'CIC_DEFAULT_TOKEN_SYMBOL': getattr(args, 'default_token_symbol'), - 'ETH_PROVIDER': getattr(args, 'p'), - 'TASKS_TRACE_QUEUE_STATUS': getattr(args, 'trace_queue_status'), +# process config +extra_args = { + 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL', + 'aux_all': None, + 'aux': None, + 'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS', } -config.add(args.q, '_CELERY_QUEUE', True) -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(args.c, config)) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -health_modules = config.get('CIC_HEALTH_MODULES', []) -if len(health_modules) != 0: - health_modules = health_modules.split(',') -logg.debug('health mods {}'.format(health_modules)) +# connect to celery +celery_app = cic_eth.cli.CeleryApp.from_config(config) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config, use_signer=True) +conn = rpc.get_default() # connect to database dsn = dsn_from_config(config) SessionBase.connect(dsn, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG')) - - -# set up celery -current_app = celery.Celery(__name__) - -broker = config.get('CELERY_BROKER_URL') -if broker[:4] == 'file': - bq = tempfile.mkdtemp() - bp = tempfile.mkdtemp() - conf_update = { - 'broker_url': broker, - 'broker_transport_options': { - 'data_folder_in': bq, - 'data_folder_out': bq, - 'data_folder_processed': bp, - }, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) -else: - conf_update = { - 'broker_url': broker, - } - if config.true('CELERY_DEBUG'): - conf_update['result_extended'] = True - current_app.conf.update(conf_update) - -result = config.get('CELERY_RESULT_URL') -if result[:4] == 'file': - rq = tempfile.mkdtemp() - current_app.conf.update({ - 'result_backend': 'file://{}'.format(rq), - }) - logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) -else: - current_app.conf.update({ - 'result_backend': result, - }) - -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) -RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') -RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') -RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default') -RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer') - Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') -#import cic_eth.checks.gas -#if not cic_eth.checks.gas.health(config=config): -# raise RuntimeError() + +# execute health checks +# TODO: health should be separate service with endpoint that can be queried +health_modules = config.get('CIC_HEALTH_MODULES', []) +if len(health_modules) != 0: + health_modules = health_modules.split(',') +logg.debug('health mods {}'.format(health_modules)) liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config, unit='cic-eth-tasker') -rpc = RPCConnection.connect(chain_spec, 'default') + +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) +registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) @@ -188,15 +130,15 @@ if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') -for address in trusted_addresses: +for i, address in enumerate(trusted_addresses): + if config.get('_UNSAFE'): + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) +connect_declarator(conn, chain_spec, trusted_addresses) +connect_token_registry(conn, chain_spec) -connect_declarator(rpc, chain_spec, trusted_addresses) -connect_token_registry(rpc, chain_spec) - -# detect aux +# detect auxiliary task modules (plugins) # TODO: move to separate file -#aux_dir = os.path.join(script_dir, '..', '..', 'aux') aux = [] if args.aux_all: if len(args.aux) > 0: @@ -249,36 +191,24 @@ elif len(args.aux) > 0: for v in aux: mname = 'cic_eth_aux.' + v mod = importlib.import_module(mname) - mod.aux_setup(rpc, config) + mod.aux_setup(conn, config) logg.info('loaded aux module {}'.format(mname)) def main(): argv = ['worker'] - if args.vv: - argv.append('--loglevel=DEBUG') - elif args.v: - argv.append('--loglevel=INFO') + log_level = logg.getEffectiveLevel() + log_level_name = logging.getLevelName(log_level) + argv.append('--loglevel=' + log_level_name) argv.append('-Q') - argv.append(args.q) + argv.append(config.get('CELERY_QUEUE')) argv.append('-n') - argv.append(args.q) - -# if config.true('SSL_ENABLE_CLIENT'): -# Callback.ssl = True -# Callback.ssl_cert_file = config.get('SSL_CERT_FILE') -# Callback.ssl_key_file = config.get('SSL_KEY_FILE') -# Callback.ssl_password = config.get('SSL_PASSWORD') -# -# if config.get('SSL_CA_FILE') != '': -# Callback.ssl_ca_file = config.get('SSL_CA_FILE') - - rpc = RPCConnection.connect(chain_spec, 'default') + argv.append(config.get('CELERY_QUEUE')) BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) - default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address) - default_token.load(rpc) + default_token = ERC20Token(chain_spec, conn, BaseTask.default_token_address) + default_token.load(conn) BaseTask.default_token_decimals = default_token.decimals BaseTask.default_token_name = default_token.name @@ -286,13 +216,13 @@ def main(): logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) liveness.linux.set(rundir=config.get('CIC_RUN_DIR')) - current_app.worker_main(argv) + celery_app.worker_main(argv) liveness.linux.reset(rundir=config.get('CIC_RUN_DIR')) @celery.signals.eventlet_pool_postshutdown.connect def shutdown(sender=None, headers=None, body=None, **kwargs): - logg.warning('in shudown event hook') + logg.warning('in shutdown event hook') if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py index 94b29883..5117537c 100644 --- a/apps/cic-eth/cic_eth/runnable/daemons/tracker.py +++ b/apps/cic-eth/cic_eth/runnable/daemons/tracker.py @@ -8,14 +8,6 @@ import sys import re # external imports -import confini -import celery -import rlp -import cic_base.config -import cic_base.log -import cic_base.argparse -import cic_base.rpc -from cic_base.eth.syncer import chain_interface from cic_eth_registry.error import UnknownContractError from chainlib.chain import ChainSpec from chainlib.eth.constant import ZERO_ADDRESS @@ -30,8 +22,13 @@ from chainsyncer.backend.sql import SQLBackend from chainsyncer.driver.head import HeadSyncer from chainsyncer.driver.history import HistorySyncer from chainsyncer.db.models.base import SessionBase +from chainlib.eth.address import ( + is_checksum_address, + to_checksum_address, + ) # local imports +import cic_eth.cli from cic_eth.db import dsn_from_config from cic_eth.runnable.daemons.filters import ( CallbackFilter, @@ -47,61 +44,50 @@ from cic_eth.registry import ( connect_token_registry, ) +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() -script_dir = os.path.realpath(os.path.dirname(__file__)) +arg_flags = cic_eth.cli.argflag_std_read +local_arg_flags = cic_eth.cli.argflag_local_sync +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() -def add_block_args(argparser): - argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync') - argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') - return argparser +# process config +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) +# connect to celery +cic_eth.cli.CeleryApp.from_config(config) -logg = cic_base.log.create() -argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template) -argparser = cic_base.argparse.add(argparser, add_block_args, 'block') -args = cic_base.argparse.parse(argparser, logg) - -config = cic_base.config.create(args.c, args, args.env_prefix) - -config.add(args.y, '_KEYSTORE_FILE', True) -config.add(args.q, '_CELERY_QUEUE', True) -config.add(args.history_start, 'SYNCER_HISTORY_START', True) -config.add(args.no_history, '_NO_HISTORY', True) - -cic_base.config.log(config) - +# set up database dsn = dsn_from_config(config) - SessionBase.connect(dsn, pool_size=16, debug=config.true('DATABASE_DEBUG')) -chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) +# set up rpc +rpc = cic_eth.cli.RPC.from_config(config) +conn = rpc.get_default() -cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER')) - -rpc = RPCConnection.connect(chain_spec, 'default') +# set up chain provisions +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) registry = None try: - registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + registry = connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) except UnknownContractError as e: logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e)) sys.exit(1) logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS'))) - def main(): - # connect to celery - celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - # Connect to blockchain with chainlib o = block_latest() - r = rpc.do(o) + r = conn.do(o) block_current = int(r, 16) block_offset = block_current + 1 loop_interval = config.get('SYNCER_LOOP_INTERVAL') if loop_interval == None: - stat = init_chain_stat(rpc, block_start=block_current) + stat = init_chain_stat(conn, block_start=block_current) loop_interval = stat.block_average() logg.debug('current block height {}'.format(block_offset)) @@ -113,9 +99,9 @@ def main(): syncer_backends = SQLBackend.resume(chain_spec, block_offset) if len(syncer_backends) == 0: - initial_block_start = config.get('SYNCER_HISTORY_START') + initial_block_start = config.get('SYNCER_OFFSET') initial_block_offset = block_offset - if config.get('_NO_HISTORY'): + if config.true('SYNCER_NO_HISTORY'): initial_block_start = block_offset initial_block_offset += 1 syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start)) @@ -128,40 +114,45 @@ def main(): for syncer_backend in syncer_backends: try: - syncers.append(HistorySyncer(syncer_backend, chain_interface)) + syncers.append(HistorySyncer(syncer_backend, cic_eth.cli.chain_interface)) logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend)) except AttributeError: logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend)) - syncers.append(HeadSyncer(syncer_backend, chain_interface)) + syncers.append(HeadSyncer(syncer_backend, cic_eth.cli.chain_interface)) - connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) + connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') if trusted_addresses_src == None: logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS') sys.exit(1) trusted_addresses = trusted_addresses_src.split(',') - for address in trusted_addresses: + for i, address in enumerate(trusted_addresses): + if not config.get('_UNSAFE'): + if not is_checksum_address(address): + raise ValueError('address {} is not a valid checksum address'.format(address)) + else: + trusted_addresses[i] = to_checksum_address(address) logg.info('using trusted address {}'.format(address)) - connect_declarator(rpc, chain_spec, trusted_addresses) - connect_token_registry(rpc, chain_spec) + connect_declarator(conn, chain_spec, trusted_addresses) + connect_token_registry(conn, chain_spec) CallbackFilter.trusted_addresses = trusted_addresses callback_filters = [] for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','): task_split = cb.split(':') - task_queue = config.get('_CELERY_QUEUE') + task_queue = config.get('CELERY_QUEUE') if len(task_split) > 1: task_queue = task_split[0] callback_filter = CallbackFilter(chain_spec, task_split[1], task_queue) callback_filters.append(callback_filter) - tx_filter = TxFilter(chain_spec, config.get('_CELERY_QUEUE')) + tx_filter = TxFilter(chain_spec, config.get('CELERY_QUEUE')) account_registry_address = registry.by_name('AccountRegistry') - registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('_CELERY_QUEUE')) + registration_filter = RegistrationFilter(chain_spec, account_registry_address, queue=config.get('CELERY_QUEUE')) - gas_filter = GasFilter(chain_spec, config.get('_CELERY_QUEUE')) + gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE')) #transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE')) @@ -176,7 +167,7 @@ def main(): for cf in callback_filters: syncer.add_filter(cf) - r = syncer.loop(int(loop_interval), rpc) + r = syncer.loop(int(loop_interval), conn) sys.stderr.write("sync {} done at block {}\n".format(syncer, r)) i += 1 diff --git a/apps/cic-eth/cic_eth/runnable/info.py b/apps/cic-eth/cic_eth/runnable/info.py index f8e7b50f..40e85103 100644 --- a/apps/cic-eth/cic_eth/runnable/info.py +++ b/apps/cic-eth/cic_eth/runnable/info.py @@ -12,50 +12,27 @@ import confini import celery # local imports +import cic_eth.cli from cic_eth.api import Api from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_format = 'terminal' -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', help='be more verbose', action='store_true') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli args') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) +celery_app = cic_eth.cli.CeleryApp.from_config(config) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) - -queue = args.q - -api = Api(config.get('CIC_CHAIN_SPEC'), queue=queue) +api = Api(config.get('CHAIN_SPEC'), queue=config.get('CELERY_QUEUE')) admin_api = AdminApi(None) + def main(): t = admin_api.registry() registry_address = t.get() diff --git a/apps/cic-eth/cic_eth/runnable/resend.py b/apps/cic-eth/cic_eth/runnable/resend.py index 6901ef34..9861e293 100644 --- a/apps/cic-eth/cic_eth/runnable/resend.py +++ b/apps/cic-eth/cic_eth/runnable/resend.py @@ -5,65 +5,38 @@ import re import os # third-party imports -import celery -import confini from chainlib.chain import ChainSpec from chainlib.eth.connection import EthHTTPConnection # local imports +import cic_eth.cli from cic_eth.api.admin import AdminApi logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -logging.getLogger('web3').setLevel(logging.WARNING) -logging.getLogger('urllib3').setLevel(logging.WARNING) - -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - - -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string') -argparser.add_argument('--unlock', action='store_true', help='Append task to unlock account') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('tx_hash', type=str, help='Transaction hash') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--unlock', action='store_true', help='Unlock account after resend') +argparser.add_positional('tx_hash', type=str, help='Transaction hash') +argparser.process_local_flags(local_arg_flags) +extra_args = { + 'unlock': None, + 'tx_hash': None, + } args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) -if args.vv: - logg.setLevel(logging.DEBUG) -elif args.v: - logg.setLevel(logging.INFO) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -config_dir = os.path.join(args.c) -os.makedirs(config_dir, 0o777, True) -config = confini.Config(config_dir, args.env_prefix) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } +celery_app = cic_eth.cli.CeleryApp.from_config(config) -# override args -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) -config.add(args.tx_hash, '_TX_HASH', True) -config.add(args.unlock, '_UNLOCK', True) - -chain_spec = ChainSpec.from_chain_str(args.i) - -rpc = EthHTTPConnection(config.get('ETH_PROVIDER')) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) def main(): - api = AdminApi(rpc) - tx_details = api.tx(chain_spec, args.tx_hash) + api = AdminApi(None) + tx_details = api.tx(chain_spec, config.get('_TX_HASH')) t = api.resend(args.tx_hash, chain_spec, unlock=config.get('_UNLOCK')) print(t.get_leaf()) diff --git a/apps/cic-eth/cic_eth/runnable/tag.py b/apps/cic-eth/cic_eth/runnable/tag.py index f7d6af11..6aeffc90 100644 --- a/apps/cic-eth/cic_eth/runnable/tag.py +++ b/apps/cic-eth/cic_eth/runnable/tag.py @@ -6,8 +6,7 @@ import argparse import re # external imports -import celery -import confini +import cic_eth.cli from chainlib.chain import ChainSpec from xdg.BaseDirectory import xdg_config_home @@ -19,43 +18,28 @@ from cic_eth.db.models.base import SessionBase logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_positional('tag', type=str, help='address tag') +argparser.add_positional('address', type=str, help='address') +argparser.process_local_flags(local_arg_flags) +args = argparser.parse_args() +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags) -argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') -argparser.add_argument('-p', '--provider', dest='p', type=str, help='Web3 provider url (http only)') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use') -argparser.add_argument('-v', help='be verbose', action='store_true') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('-vv', help='be more verbose', action='store_true') -argparser.add_argument('tag', type=str, help='address tag') -argparser.add_argument('address', type=str, help='address') -args = argparser.parse_args(sys.argv[1:]) +celery_app = cic_eth.cli.CeleryApp.from_config(config) -if args.v == True: - logging.getLogger().setLevel(logging.INFO) -elif args.vv == True: - logging.getLogger().setLevel(logging.DEBUG) +admin_api = AdminApi(None) -config = confini.Config(args.c) -config.process() -args_override = { - 'ETH_PROVIDER': getattr(args, 'p'), - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - } -config.dict_override(args_override, 'cli flag') -config.censor('PASSWORD', 'DATABASE') -config.censor('PASSWORD', 'SSL') -logg.debug('config loaded from {}\n{}'.format(args.c, config)) +chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) -chain_spec = ChainSpec.from_chain_str(args.i) - -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +celery_app = cic_eth.cli.CeleryApp.from_config(config) +api = AdminApi(None) def main(): - api = AdminApi(None) - api.tag_account(args.tag, args.address, chain_spec) + admin_api.tag_account(args.tag, args.address, chain_spec) if __name__ == '__main__': diff --git a/apps/cic-eth/cic_eth/runnable/transfer.py b/apps/cic-eth/cic_eth/runnable/transfer.py index c63c72a7..4bbaa306 100644 --- a/apps/cic-eth/cic_eth/runnable/transfer.py +++ b/apps/cic-eth/cic_eth/runnable/transfer.py @@ -7,88 +7,58 @@ import json import argparse # external imports -import celery -import confini import redis from xdg.BaseDirectory import xdg_config_home from chainlib.eth.address import to_checksum_address # local imports +import cic_eth.cli from cic_eth.api import Api logging.basicConfig(level=logging.WARNING) logg = logging.getLogger('create_account_script') -logging.getLogger('confini').setLevel(logging.WARNING) -logging.getLogger('gnupg').setLevel(logging.WARNING) -default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic') - -argparser = argparse.ArgumentParser() -argparser.add_argument('--no-register', dest='no_register', action='store_true', help='Do not register new account in on-chain accounts index') -argparser.add_argument('-c', type=str, default=default_config_dir, help='config file') -argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec') -argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Symbol of token to transfer') -argparser.add_argument('--redis-host', dest='redis_host', type=str, help='redis host to use for task submission') -argparser.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission') -argparser.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use for task submission and callback') -argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback') -argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback') -argparser.add_argument('--timeout', default=20.0, type=float, help='Callback timeout') -argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue') -argparser.add_argument('-v', action='store_true', help='Be verbose') -argparser.add_argument('-vv', action='store_true', help='Be more verbose') -argparser.add_argument('sender', type=str, help='Transaction sender') -argparser.add_argument('recipient', type=str, help='Transaction recipient') -argparser.add_argument('value', type=int, help='Transaction value with decimals') +arg_flags = cic_eth.cli.argflag_std_base +local_arg_flags = cic_eth.cli.argflag_local_taskcallback +argparser = cic_eth.cli.ArgumentParser(arg_flags) +argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol') +argparser.add_positional('sender', type=str, help='Token transaction sender') +argparser.add_positional('recipient', type=str, help='Token transaction recipient') +argparser.add_positional('value', type=int, help='Token transaction value') +argparser.process_local_flags(local_arg_flags) args = argparser.parse_args() -if args.vv: - logg.setLevel(logging.DEBUG) -if args.v: - logg.setLevel(logging.INFO) - -config_dir = args.c -config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX')) -config.process() -args_override = { - 'CIC_CHAIN_SPEC': getattr(args, 'i'), - 'REDIS_HOST': getattr(args, 'redis_host'), - 'REDIS_PORT': getattr(args, 'redis_port'), - 'REDIS_DB': getattr(args, 'redis_db'), +extra_args = { + 'token_symbol': None, + 'sender': None, + 'recipient': None, + 'value': None, } -config.dict_override(args_override, 'cli') -config.add(to_checksum_address(args.sender), '_SENDER', True) -config.add(to_checksum_address(args.recipient), '_RECIPIENT', True) -config.add(args.value, '_VALUE', True) -config.add(args.token_symbol, '_SYMBOL', True) -if config.get('_SYMBOL') == None: - raise ValueError('gas transfers not yet supported; token symbol required') -celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL')) +config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args) + +celery_app = cic_eth.cli.CeleryApp.from_config(config) def main(): - redis_host = config.get('REDIS_HOST') - redis_port = config.get('REDIS_PORT') - redis_db = config.get('REDIS_DB') redis_channel = str(uuid.uuid4()) - r = redis.Redis(redis_host, redis_port, redis_db) + r = redis.Redis(config.get('REDIS_HOST'), config.get('REDIS_PORT'), config.get('REDIS_DB')) ps = r.pubsub() ps.subscribe(redis_channel) ps.get_message() api = Api( - config.get('CIC_CHAIN_SPEC'), - queue=args.q, - callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel), + config.get('CHAIN_SPEC'), + queue=config.get('CELERY_QUEUE'), + callback_param='{}:{}:{}:{}'.format(config.get('_REDIS_HOST_CALLBACK'), config.get('_REDIS_PORT_CALLBACK'), config.get('REDIS_DB'), redis_channel), callback_task='cic_eth.callbacks.redis.redis', - callback_queue=args.q, + callback_queue=config.get('CELERY_QUEUE') ) - t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_SYMBOL')) + t = api.transfer(config.get('_SENDER'), config.get('_RECIPIENT'), config.get('_VALUE'), config.get('_TOKEN_SYMBOL')) ps.get_message() - o = ps.get_message(timeout=args.timeout) + o = ps.get_message(timeout=config.get('REDIS_TIMEOUT')) m = json.loads(o['data']) print(m['result']) diff --git a/apps/cic-eth/cic_eth/sync/retry.py b/apps/cic-eth/cic_eth/sync/retry.py index b311964d..9118ebf4 100644 --- a/apps/cic-eth/cic_eth/sync/retry.py +++ b/apps/cic-eth/cic_eth/sync/retry.py @@ -3,7 +3,7 @@ import logging import datetime # external imports -from chainsyncer.driver import HeadSyncer +from chainsyncer.driver.head import HeadSyncer from chainsyncer.backend.memory import MemBackend from chainsyncer.error import NoBlockForYou from chainlib.eth.block import ( @@ -39,9 +39,9 @@ class DbSessionMemBackend(MemBackend): class RetrySyncer(HeadSyncer): - def __init__(self, conn, chain_spec, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): + def __init__(self, conn, chain_spec, chain_interface, stalled_grace_seconds, batch_size=50, failed_grace_seconds=None): backend = DbSessionMemBackend(chain_spec, None) - super(RetrySyncer, self).__init__(backend) + super(RetrySyncer, self).__init__(backend, chain_interface) self.chain_spec = chain_spec if failed_grace_seconds == None: failed_grace_seconds = stalled_grace_seconds diff --git a/apps/cic-eth/cic_eth/version.py b/apps/cic-eth/cic_eth/version.py index 039b9baf..f9ac618b 100644 --- a/apps/cic-eth/cic_eth/version.py +++ b/apps/cic-eth/cic_eth/version.py @@ -10,7 +10,7 @@ version = ( 0, 12, 2, - 'alpha.3', + 'alpha.4', ) version_object = semver.VersionInfo( diff --git a/apps/cic-eth/config/bancor.ini b/apps/cic-eth/config/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/celery.ini b/apps/cic-eth/config/celery.ini deleted file mode 100644 index 038f83ad..00000000 --- a/apps/cic-eth/config/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis:// -result_url = redis:// -debug = 0 diff --git a/apps/cic-eth/config/custody.ini b/apps/cic-eth/config/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/database.ini b/apps/cic-eth/config/database.ini index 4517445c..6448be0e 100644 --- a/apps/cic-eth/config/database.ini +++ b/apps/cic-eth/config/database.ini @@ -1,9 +1,9 @@ [database] -NAME=cic-eth +NAME=cic_eth USER=postgres -PASSWORD= +PASSWORD=tralala HOST=localhost -PORT=5432 +PORT=63432 ENGINE=postgresql DRIVER=psycopg2 POOL_SIZE=50 diff --git a/apps/cic-eth/config/dispatcer.ini b/apps/cic-eth/config/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/bancor.ini b/apps/cic-eth/config/docker/bancor.ini deleted file mode 100644 index 443924f5..00000000 --- a/apps/cic-eth/config/docker/bancor.ini +++ /dev/null @@ -1,2 +0,0 @@ -[bancor] -dir = /usr/local/share/cic/bancor diff --git a/apps/cic-eth/config/docker/celery.ini b/apps/cic-eth/config/docker/celery.ini deleted file mode 100644 index 646d6603..00000000 --- a/apps/cic-eth/config/docker/celery.ini +++ /dev/null @@ -1,4 +0,0 @@ -[celery] -broker_url = redis://localhost:63379 -result_url = redis://localhost:63379 -debug = 0 diff --git a/apps/cic-eth/config/docker/cic.ini b/apps/cic-eth/config/docker/cic.ini deleted file mode 100644 index 4fdefca0..00000000 --- a/apps/cic-eth/config/docker/cic.ini +++ /dev/null @@ -1,8 +0,0 @@ -[cic] -registry_address = -chain_spec = evm:bloxberg:8996 -trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -tx_retry_delay = 20 -default_token_symbol = GFT -health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas -run_dir = /run diff --git a/apps/cic-eth/config/docker/custody.ini b/apps/cic-eth/config/docker/custody.ini deleted file mode 100644 index 8523f03e..00000000 --- a/apps/cic-eth/config/docker/custody.ini +++ /dev/null @@ -1,2 +0,0 @@ -[custody] -account_index_address = diff --git a/apps/cic-eth/config/docker/dispatcer.ini b/apps/cic-eth/config/docker/dispatcer.ini deleted file mode 100644 index 85dea83e..00000000 --- a/apps/cic-eth/config/docker/dispatcer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[dispatcher] -loop_interval = 0.9 diff --git a/apps/cic-eth/config/docker/eth.ini b/apps/cic-eth/config/docker/eth.ini deleted file mode 100644 index 534a2eea..00000000 --- a/apps/cic-eth/config/docker/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://localhost:63545 -gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/docker/signer.ini b/apps/cic-eth/config/docker/signer.ini deleted file mode 100644 index fe72206f..00000000 --- a/apps/cic-eth/config/docker/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/docker/ssl.ini b/apps/cic-eth/config/docker/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/docker/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/docker/syncer.ini b/apps/cic-eth/config/docker/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/docker/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/eth.ini b/apps/cic-eth/config/eth.ini deleted file mode 100644 index 3c589cb2..00000000 --- a/apps/cic-eth/config/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://localhost:8545 -gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/config/lash/bancor.ini b/apps/cic-eth/config/lash/bancor.ini deleted file mode 100644 index 0bff1ce0..00000000 --- a/apps/cic-eth/config/lash/bancor.ini +++ /dev/null @@ -1,3 +0,0 @@ -[bancor] -registry_address = 0xb708175e3f6Cd850643aAF7B32212AFad50e2549 -dir = /home/lash/src/ext/cic/grassrootseconomics/cic-platform/contrib/bancor_0.6 diff --git a/apps/cic-eth/config/lash/database.ini b/apps/cic-eth/config/lash/database.ini deleted file mode 100644 index 77ac57d4..00000000 --- a/apps/cic-eth/config/lash/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -NAME=cic-eth -USER=postgres -PASSWORD= -HOST=localhost -PORT=5432 -ENGINE=sqlite -DRIVER=pysqlite diff --git a/apps/cic-eth/config/lash/eth.ini b/apps/cic-eth/config/lash/eth.ini deleted file mode 100644 index 324dba08..00000000 --- a/apps/cic-eth/config/lash/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -gas_provider_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -provider = http://localhost:8545 diff --git a/apps/cic-eth/config/lash/redis.ini b/apps/cic-eth/config/lash/redis.ini deleted file mode 100644 index d21df9f8..00000000 --- a/apps/cic-eth/config/lash/redis.ini +++ /dev/null @@ -1,3 +0,0 @@ -[celery] -broker_url = redis:// -result_url = file:// diff --git a/apps/cic-eth/config/lash/signer.ini b/apps/cic-eth/config/lash/signer.ini deleted file mode 100644 index 9222aef8..00000000 --- a/apps/cic-eth/config/lash/signer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[signer] -socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc diff --git a/apps/cic-eth/config/lash/ssl.ini b/apps/cic-eth/config/lash/ssl.ini deleted file mode 100644 index 746ff856..00000000 --- a/apps/cic-eth/config/lash/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = true -cert_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.crt -key_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/client.key -password = test -ca_file = /home/lash/src/ext/cic/grassrootseconomics/cic-auth/examples/ca.crt diff --git a/apps/cic-eth/config/lash/syncer.ini b/apps/cic-eth/config/lash/syncer.ini deleted file mode 100644 index 97236743..00000000 --- a/apps/cic-eth/config/lash/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[SYNCER] -loop_interval = 1 diff --git a/apps/cic-eth/config/redis.ini b/apps/cic-eth/config/redis.ini deleted file mode 100644 index 1a7acb52..00000000 --- a/apps/cic-eth/config/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = -port = -db = diff --git a/apps/cic-eth/config/signer.ini b/apps/cic-eth/config/signer.ini deleted file mode 100644 index 0fa1f015..00000000 --- a/apps/cic-eth/config/signer.ini +++ /dev/null @@ -1,5 +0,0 @@ -[signer] -socket_path = /run/crypto-dev-signer/jsonrpc.ipc -secret = deedbeef -database_name = signer_test -dev_keys_path = diff --git a/apps/cic-eth/config/ssl.ini b/apps/cic-eth/config/ssl.ini deleted file mode 100644 index bd494729..00000000 --- a/apps/cic-eth/config/ssl.ini +++ /dev/null @@ -1,6 +0,0 @@ -[SSL] -enable_client = false -cert_file = -key_file = -password = -ca_file = diff --git a/apps/cic-eth/config/syncer.ini b/apps/cic-eth/config/syncer.ini deleted file mode 100644 index fc88c7f4..00000000 --- a/apps/cic-eth/config/syncer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[SYNCER] -loop_interval = -history_start = 0 diff --git a/apps/cic-eth/config/tasks.ini b/apps/cic-eth/config/tasks.ini deleted file mode 100644 index 732d797e..00000000 --- a/apps/cic-eth/config/tasks.ini +++ /dev/null @@ -1,3 +0,0 @@ -[tasks] -transfer_callbacks = taskcall:cic_eth.callbacks.noop.noop -trace_queue_status = 1 diff --git a/apps/cic-eth/config/test/chain.ini b/apps/cic-eth/config/test/chain.ini new file mode 100644 index 00000000..9fda0988 --- /dev/null +++ b/apps/cic-eth/config/test/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 64ce7d1b..e627361c 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -27,6 +27,12 @@ RUN python setup.py install ENV PYTHONPATH . +RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ + pip install --index-url https://pypi.org/simple \ + --extra-index-url $GITLAB_PYTHON_REGISTRY \ + --extra-index-url $EXTRA_INDEX_URL \ + cic-eth-aux-erc20-demurrage-token~=0.0.2a6 + COPY docker/entrypoints/* ./ RUN chmod 755 *.sh diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 09811a0f..196a1284 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.6a1,<0.1.0 +chainlib-eth>=0.0.7a1,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index c70193a6..5308d2f4 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -1,15 +1,15 @@ chainqueue>=0.0.3a2,<0.1.0 -chainsyncer[sql]>=0.0.5a1,<0.1.0 +chainsyncer[sql]>=0.0.6a1,<0.1.0 alembic==1.4.2 confini>=0.3.6rc4,<0.5.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 liveness~=0.0.1a7 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a5,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a1,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a2,<0.1.0 moolb~=0.1.1b2 diff --git a/apps/cic-eth/setup.cfg b/apps/cic-eth/setup.cfg index a0354019..2b798095 100644 --- a/apps/cic-eth/setup.cfg +++ b/apps/cic-eth/setup.cfg @@ -24,8 +24,10 @@ licence_files = [options] python_requires = >= 3.6 +include_package_data = True packages = cic_eth + cic_eth.cli cic_eth.admin cic_eth.eth cic_eth.api diff --git a/apps/cic-eth/test_requirements.txt b/apps/cic-eth/test_requirements.txt index cf7c693d..bf5ecdcc 100644 --- a/apps/cic-eth/test_requirements.txt +++ b/apps/cic-eth/test_requirements.txt @@ -6,4 +6,4 @@ pytest-redis==2.0.0 redis==3.5.3 eth-tester==0.5.0b3 py-evm==0.3.0a20 -eth-erc20~=0.0.11a1 +eth-erc20~=0.0.12a1 diff --git a/apps/cic-eth/tests/check/test_check_gas.py b/apps/cic-eth/tests/check/test_check_gas.py index ea0ffb88..056120c8 100644 --- a/apps/cic-eth/tests/check/test_check_gas.py +++ b/apps/cic-eth/tests/check/test_check_gas.py @@ -11,7 +11,7 @@ def test_check_gas( whoever, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) config.add(100, 'ETH_GAS_GIFTER_MINIMUM_BALANCE', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tests/check/test_check_signer.py b/apps/cic-eth/tests/check/test_check_signer.py index 5572e195..63959de9 100644 --- a/apps/cic-eth/tests/check/test_check_signer.py +++ b/apps/cic-eth/tests/check/test_check_signer.py @@ -9,5 +9,5 @@ def test_check_signer( eth_rpc, ): - config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True) + config.add(str(default_chain_spec), 'CHAIN_SPEC', exists_ok=True) assert health(config=config) diff --git a/apps/cic-eth/tools_requirements.txt b/apps/cic-eth/tools_requirements.txt index 03a00145..9dcf8d51 100644 --- a/apps/cic-eth/tools_requirements.txt +++ b/apps/cic-eth/tools_requirements.txt @@ -1,7 +1,7 @@ crypto-dev-signer>=0.4.14b7,<=0.4.14 -chainqueue~=0.0.2b6 +chainqueue>=0.0.3a1,<0.1.0 confini>=0.3.6rc4,<0.5.0 -cic-eth-registry>=0.5.7a1,<0.6.0 +cic-eth-registry>=0.5.8a1,<0.6.0 redis==3.5.3 hexathon~=0.0.1a7 pycryptodome==3.10.1 diff --git a/apps/cic-ussd/requirements.txt b/apps/cic-ussd/requirements.txt index b0b81f95..9d71a990 100644 --- a/apps/cic-ussd/requirements.txt +++ b/apps/cic-ussd/requirements.txt @@ -13,5 +13,11 @@ redis==3.5.3 semver==2.13.0 SQLAlchemy==1.3.20 tinydb==4.2.0 +phonenumbers==8.12.12 +redis==3.5.3 +celery==4.4.7 +python-i18n[YAML]==0.3.9 +pyxdg==0.27 +bcrypt==3.2.0 +uWSGI==2.0.19.1 transitions==0.8.4 -uWSGI==2.0.19.1 \ No newline at end of file diff --git a/apps/contract-migration/config_template/celery.ini b/apps/contract-migration/config_template/celery.ini index dac9a5f4..e69de29b 100644 --- a/apps/contract-migration/config_template/celery.ini +++ b/apps/contract-migration/config_template/celery.ini @@ -1,3 +0,0 @@ -[celery] -broker_url = redis://redis:6379 -result_url = redis://redis:6379 diff --git a/apps/contract-migration/config_template/chain.ini b/apps/contract-migration/config_template/chain.ini deleted file mode 100644 index 6a2a8cba..00000000 --- a/apps/contract-migration/config_template/chain.ini +++ /dev/null @@ -1,2 +0,0 @@ -[chain] -spec = evm:ethereum:1 diff --git a/apps/contract-migration/config_template/cic.ini b/apps/contract-migration/config_template/cic.ini deleted file mode 100644 index bf598b98..00000000 --- a/apps/contract-migration/config_template/cic.ini +++ /dev/null @@ -1,7 +0,0 @@ -[cic] -registry_address = -token_index_address = -accounts_index_address = -declarator_address = -approval_escrow_address = -chain_spec = Bloxberg:8996 diff --git a/apps/contract-migration/config_template/database.ini b/apps/contract-migration/config_template/database.ini deleted file mode 100644 index db8227bc..00000000 --- a/apps/contract-migration/config_template/database.ini +++ /dev/null @@ -1,8 +0,0 @@ -[database] -user = postgres -password = tralala -name = -host = postgres -port = 5432 -engine = postgres -driver = psycopg2 diff --git a/apps/contract-migration/config_template/dev.ini b/apps/contract-migration/config_template/dev.ini deleted file mode 100644 index a2c663ea..00000000 --- a/apps/contract-migration/config_template/dev.ini +++ /dev/null @@ -1,33 +0,0 @@ -[dev] -mnemonic = "history stumble mystery avoid embark arrive mom foil pledge keep grain dice" -eth_reserve_address = -eth_accounts_index_address = -# 10m * 10^18 (10^7 * 10^18) -eth_reserve_amount = 1000000000000000000000000 - -eth_account_contract_deployer=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C -eth_account_gas_provider=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_gas_gifter=0xFfe6E610d1Ea184AEE71c6B39828ecE921f04a7F -eth_account_reserve_owner=0xc14958CD9A605AB0d9A36850362AaD2b9D42DF97 -eth_account_reserve_minter=0xe3C4db5947409Aff0FF8D643047EA41515cA4B8e -eth_account_accounts_index_owner=0x1AB26f3AAaaa884B651E213508592F3Ec6BfCe74 -eth_account_accounts_index_writer=0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA -eth_account_sarafu_owner=0x5567139c7a1C2977A391f51D8cA45B1D6700f5F6 -eth_account_sarafu_gifter=0x1149D55B3495CdC8FcfaF2816384AafaeFfaEa24 -eth_account_approval_escrow_owner=0x8754122F5718dC02Db5062c62fD4a63A5448623E -eth_account_single_shot_faucet_owner=0x34C85E47f45b0bea09F37c83e2fb02ECBC3a395d - -eth_sarafu_token_address = -eth_token_index_address = -eth_sarafu_faucet_address = -eth_erc20_approval_escrow_address = - -eth_sarafu_token_name=Sarafu -eth_sarafu_token_symbol=SRF -eth_sarafu_token_decimals=18 - -pgp_publickeys_active_file = publickeys.asc -pgp_publickeys_trusted_file = -pgp_publickeys_encrypt_file = - -faucet_amount = 1000000 diff --git a/apps/contract-migration/config_template/empty/config.ini b/apps/contract-migration/config_template/empty/config.ini new file mode 100644 index 00000000..e69de29b diff --git a/apps/contract-migration/config_template/eth.ini b/apps/contract-migration/config_template/eth.ini deleted file mode 100644 index 5506851b..00000000 --- a/apps/contract-migration/config_template/eth.ini +++ /dev/null @@ -1,3 +0,0 @@ -[eth] -provider = http://eth:8545 -abi_dir = /usr/local/share/cic/solidity/abi diff --git a/apps/contract-migration/config_template/legacy.ini b/apps/contract-migration/config_template/legacy.ini new file mode 100644 index 00000000..02eb709c --- /dev/null +++ b/apps/contract-migration/config_template/legacy.ini @@ -0,0 +1,7 @@ +[celery] +broker_url = +result_url = +debug = + +[cic] +chain_spec = diff --git a/apps/contract-migration/config_template/meta.ini b/apps/contract-migration/config_template/meta.ini deleted file mode 100644 index cbcc16ac..00000000 --- a/apps/contract-migration/config_template/meta.ini +++ /dev/null @@ -1,2 +0,0 @@ -[meta] -provider = http://cic-meta-server:80 diff --git a/apps/contract-migration/config_template/pgp.ini b/apps/contract-migration/config_template/pgp.ini deleted file mode 100644 index fade0d5a..00000000 --- a/apps/contract-migration/config_template/pgp.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pgp] -exports_dir = /tmp/cic/pgp -privatekey_file = ge.priv.asc -passphrase = ge diff --git a/apps/contract-migration/config_template/redis.ini b/apps/contract-migration/config_template/redis.ini deleted file mode 100644 index ea321378..00000000 --- a/apps/contract-migration/config_template/redis.ini +++ /dev/null @@ -1,4 +0,0 @@ -[redis] -host = localhost -port = 63379 -db = 0 diff --git a/apps/contract-migration/config_template/rpc.ini b/apps/contract-migration/config_template/rpc.ini deleted file mode 100644 index 31dc0614..00000000 --- a/apps/contract-migration/config_template/rpc.ini +++ /dev/null @@ -1,5 +0,0 @@ -[rpc] -http_provider = http://localhost:8545 -http_authentication = -http_username = -http_password = diff --git a/apps/contract-migration/config_template/signer.ini b/apps/contract-migration/config_template/signer.ini deleted file mode 100644 index 58ad0c0a..00000000 --- a/apps/contract-migration/config_template/signer.ini +++ /dev/null @@ -1,3 +0,0 @@ -[signer] -socket_path = /tmp/cic/signer/jsonrpc.ipc -secret = deadbeef diff --git a/apps/contract-migration/config_template/syncer.ini b/apps/contract-migration/config_template/syncer.ini deleted file mode 100644 index 60c0c007..00000000 --- a/apps/contract-migration/config_template/syncer.ini +++ /dev/null @@ -1,2 +0,0 @@ -[syncer] -loop_interval = diff --git a/apps/contract-migration/config_template/tasks.ini b/apps/contract-migration/config_template/tasks.ini deleted file mode 100644 index b49ecad5..00000000 --- a/apps/contract-migration/config_template/tasks.ini +++ /dev/null @@ -1,5 +0,0 @@ -[tasks] -africastalking = cic_notify.tasks.sms.africastalking -sms_db = cic_notify.tasks.sms.db -log = cic_notify.tasks.sms.log -transfer_callbacks=cic-ussd:cic_ussd.tasks.callback_handler.process_incoming_transfer_callback diff --git a/apps/contract-migration/config_template/token.ini b/apps/contract-migration/config_template/token.ini deleted file mode 100644 index 865e7f84..00000000 --- a/apps/contract-migration/config_template/token.ini +++ /dev/null @@ -1,8 +0,0 @@ -[token] -name = -symbol = -decimals = -demurrage_level = -redistribution_period = -sink_address = -supply_limit = 0 diff --git a/apps/contract-migration/config_template/wallet.ini b/apps/contract-migration/config_template/wallet.ini deleted file mode 100644 index 696cee43..00000000 --- a/apps/contract-migration/config_template/wallet.ini +++ /dev/null @@ -1,3 +0,0 @@ -[wallet] -key_file = -passphrase = diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index 9d5cb9b2..d374e8d5 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,10 +1,10 @@ -cic-eth[tools]==0.12.2a3 -eth-erc20>=0.0.11a1,<0.1.0 -erc20-demurrage-token>=0.0.2a5,<0.1.0 -eth-address-index>=0.1.3a1,<0.2.0 -eth-accounts-index>=0.0.13a1,<0.1.0 -cic-eth-registry>=0.5.7a1,<=0.6.0 -erc20-faucet>=0.2.3a1,<0.3.0 -erc20-transfer-authorization>=0.3.3a1,<0.4.0 -sarafu-faucet>=0.0.4a4,<0.1.0 -chainlib-eth>=0.0.6a1,<0.1.0 +cic-eth[tools]==0.12.2a4 +eth-erc20>=0.0.12a1,<0.1.0 +erc20-demurrage-token>=0.0.3a1,<0.1.0 +eth-address-index>=0.1.4a1,<0.2.0 +eth-accounts-index>=0.0.14a1,<0.1.0 +cic-eth-registry>=0.5.8a1,<0.6.0 +erc20-faucet>=0.2.4a1,<0.3.0 +erc20-transfer-authorization>=0.3.4a1,<0.4.0 +sarafu-faucet>=0.0.5a5,<0.1.0 +chainlib-eth>=0.0.7a1,<0.1.0 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index ed6912c6..3a9d1b7c 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -17,7 +17,9 @@ token_redistribution_period: $TOKEN_REDISTRIBUTION_PERIOD token_supply_limit: $TOKEN_SUPPLY_LIMIT EOF -CIC_CHAIN_SPEC=${CIC_CHAIN_SPEC:-evm:bloxberg:8995} +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + DEV_ETH_ACCOUNT_RESERVE_MINTER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=${DEV_ETH_ACCOUNT_RESERVE_MINTER:-$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER} DEV_RESERVE_AMOUNT=${DEV_ETH_RESERVE_AMOUNT:-""10000000000000000000000000000000000} @@ -30,6 +32,7 @@ DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=`eth-checksum $(cat $DEV_ETH_KEYSTORE_FILE | j if [ ! -z $DEV_ETH_GAS_PRICE ]; then gas_price_arg="--gas-price $DEV_ETH_GAS_PRICE" + fee_price_arg="--fee-price $DEV_ETH_GAS_PRICE" >&2 echo using static gas price $DEV_ETH_GAS_PRICE fi @@ -133,13 +136,22 @@ if [[ -n "${ETH_PROVIDER}" ]]; then # Sarafu faucet contract >&2 echo "deploy token faucet contract" - DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS` + export _CONFINI_DIR=$CONFINI_DIR + unset CONFINI_DIR + DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s` + + >&2 echo "set token faucet amount" + sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT + + export CONFINI_DIR=$_CONFINI_DIR + + >&2 echo "register faucet in registry" eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS + >&2 echo "set faucet as token minter" giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS - >&2 echo "set token faucet amount" - sarafu-faucet-set $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_FAUCET_ADDRESS -vv $DEV_FAUCET_AMOUNT + else diff --git a/apps/contract-migration/seed_cic_eth.sh b/apps/contract-migration/seed_cic_eth.sh index aba7ea6c..29a80220 100755 --- a/apps/contract-migration/seed_cic_eth.sh +++ b/apps/contract-migration/seed_cic_eth.sh @@ -16,15 +16,18 @@ ETH_PASSPHRASE='' CIC_DEFAULT_TOKEN_SYMBOL=${CIC_DEFAULT_TOKEN_SYMBOL:-GFT} TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL +CHAIN_SPEC=${CHAIN_SPEC:-$CIC_CHAIN_SPEC} +RPC_HTTP_PROVIDER=${RPC_HTTP_PROVIDER:-$ETH_PROVIDER} + # Debug flag DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C keystore_file=./keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c debug='-vv' -abi_dir=${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} gas_amount=100000000000000000000000 token_amount=${gas_amount} env_out_file=${CIC_DATA_DIR}/.env_seed init_level_file=${CIC_DATA_DIR}/.init +empty_config_dir=$CONFINI_DIR/empty truncate $env_out_file -s 0 set -e @@ -33,9 +36,9 @@ set -a #pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL eth-address-index==0.1.1a7 # get required addresses from registries -DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` -DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` -DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` +DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry` +DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry` +DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL` cat <&2 echo "create account for gas gifter" old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER -DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create --timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_GAS_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create --redis-timeout 120 $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER +cic-eth-tag -i $CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER >&2 echo "create account for sarafu gifter" -DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_SARAFU_GIFTER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER +cic-eth-tag -i $CHAIN_SPEC SARAFU_GIFTER $DEV_ETH_ACCOUNT_SARAFU_GIFTER >&2 echo "create account for approval escrow owner" -DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER +cic-eth-tag -i $CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER #>&2 echo "create account for faucet owner" #DEV_ETH_ACCOUNT_FAUCET_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` @@ -65,32 +68,32 @@ cic-eth-tag -i $CIC_CHAIN_SPEC TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRA #cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER >&2 echo "create account for accounts index writer" -DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` +DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-create $debug --redis-host $REDIS_HOST --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file -cic-eth-tag -i $CIC_CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >&2 echo "add acccounts index writer account as writer on contract" -eth-accounts-index-writer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER +eth-accounts-index-writer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER # Transfer gas to custodial gas provider adddress _CONFINI_DIR=$CONFINI_DIR unset CONFINI_DIR >&2 echo gift gas to gas gifter ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount >&2 echo gift gas to sarafu token owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount >&2 echo gift gas to account index owner ->&2 eth-gas --send -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount +>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount # Send token to token creator >&2 echo "gift tokens to sarafu owner" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount # Send token to token gifter >&2 echo "gift tokens to keystore address" ->&2 giftable-token-gift -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount +>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)" echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file @@ -98,7 +101,7 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS # Transfer tokens to gifter address >&2 echo "transfer tokens to token gifter address" ->&2 erc20-transfer -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} +>&2 erc20-transfer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1} #echo -n 0 > $init_level_file diff --git a/apps/data-seeding/requirements.txt b/apps/data-seeding/requirements.txt index 10a752fd..7317ddf8 100644 --- a/apps/data-seeding/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,13 +1,13 @@ -sarafu-faucet==0.0.4a3 -cic-eth[tools]==0.12.1a1 -cic-types==0.1.0a13 -crypto-dev-signer==0.4.14b7 +sarafu-faucet~=0.0.5a2 +cic-eth[tools]~=0.12.2a3 +cic-types~=0.1.0a14 +crypto-dev-signer~=0.4.14rc1 faker==4.17.1 -chainsyncer~=0.0.3a3 -chainlib-eth~=0.0.5a1 -eth-address-index~=0.1.2a1 -eth-contract-registry~=0.5.6a1 -eth-accounts-index~=0.0.12a1 -eth-erc20~=0.0.10a3 -erc20-faucet~=0.2.2a1 +chainsyncer~=0.0.6a1 +chainlib-eth~=0.0.7a1 +eth-address-index~=0.1.4a1 +eth-contract-registry~=0.5.8a1 +eth-accounts-index~=0.0.14a1 +eth-erc20~=0.0.12a1 +erc20-faucet~=0.2.4a1 psycopg2==2.8.6 diff --git a/docker-compose.yml b/docker-compose.yml index 3c9a4540..57b6be2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,16 +79,18 @@ services: args: pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} pip_extra_args: $PIP_EXTRA_ARGS + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS - # ETH_PROVIDER should be broken out into host/port but cic-eth expects this - ETH_PROVIDER: http://eth:8545 + ETH_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${CIC_HTTP_PROVIDER:-http://eth:8545} # And these two are for wait-for-it (could parse this) DEV_USE_DOCKER_WAIT_SCRIPT: 1 ETH_PROVIDER_HOST: eth ETH_PROVIDER_PORT: 8545 - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_DATA_DIR: ${CIC_DATA_DIR:-/tmp/cic/config} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PORT: ${DATABASE_PORT:-5432} @@ -131,9 +133,12 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -142,9 +147,9 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 CELERY_RESULT_URL: redis://redis:6379 deploy: @@ -169,6 +174,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} @@ -181,7 +188,6 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: 1 DATABASE_POOL_SIZE: 0 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: redis://redis:6379 @@ -208,6 +214,8 @@ services: build: context: apps/cic-cache dockerfile: docker/Dockerfile + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} @@ -240,11 +248,13 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS ETH_GAS_PROVIDER_ADDRESS: $DEV_ETH_ACCOUNT_GAS_PROVIDER - ETH_PROVIDER: http://eth:8545 - ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + RPC_HTTP_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -258,11 +268,12 @@ services: REDIS_HOST: redis PGPASSWORD: ${DATABASE_PASSWORD:-tralala} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} - BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis} CELERY_DEBUG: ${CELERY_DEBUG:-1} SIGNER_SOCKET_PATH: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} + SIGNER_PROVIDER: ${SIGNER_SOCKET_PATH:-ipc:///run/crypto-dev-signer/jsonrpc.ipc} SIGNER_SECRET: ${SIGNER_SECRET:-deadbeef} ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER: ${DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER:-0xACB0BC74E1686D62dE7DC6414C999EA60C09F0eA} TASKS_TRACE_QUEUE_STATUS: ${TASKS_TRACE_QUEUE_STATUS:-1} @@ -290,8 +301,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: - ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} + ETH_PROVIDER: ${RPC_HTTP_PROVIDER:-http://eth:8545} DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -300,7 +314,8 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} - CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CIC_CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -320,7 +335,8 @@ services: - -c - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - ./start_tracker.sh -vv -c /usr/local/etc/cic-eth + #./start_tracker.sh -vv -c /usr/local/etc/cic-eth + ./start_tracker.sh -vv # command: "/root/start_manager.sh head -vv" @@ -329,8 +345,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -339,6 +358,7 @@ services: DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} @@ -369,8 +389,11 @@ services: context: apps/cic-eth dockerfile: docker/Dockerfile target: dev + args: + EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} environment: ETH_PROVIDER: http://eth:8545 + RPC_HTTP_PROVIDER: http://eth:8545 DATABASE_USER: ${DATABASE_USER:-grassroots} DATABASE_HOST: ${DATABASE_HOST:-postgres} DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} @@ -380,6 +403,7 @@ services: DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} DATABASE_DEBUG: ${DATABASE_DEBUG:-0} CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} + CHAIN_SPEC: ${CIC_CHAIN_SPEC:-evm:bloxberg:8996} CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS #BANCOR_DIR: $BANCOR_DIR CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} From 0ec9813e5fadc51c8071a3446c82bb4ce323f0d3 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Tue, 17 Aug 2021 08:03:14 +0000 Subject: [PATCH 5/6] Finish cic-cache integration for transaction listings --- apps/cic-cache/cic_cache/api.py | 30 ++- apps/cic-cache/cic_cache/cache.py | 50 ++++- apps/cic-cache/cic_cache/cli/arg.py | 2 +- .../cic_cache/data/config/celery.ini | 2 +- apps/cic-cache/cic_cache/db/list.py | 152 ++++++++++++- .../cic_cache/runnable/daemons/query.py | 7 +- apps/cic-cache/cic_cache/tasks/tx.py | 24 +- apps/cic-cache/cic_cache/version.py | 4 +- apps/cic-cache/requirements.txt | 5 +- apps/cic-cache/setup.cfg | 1 + apps/cic-cache/tests/cli/test_cli_args.py | 40 ++++ apps/cic-cache/tests/cli/test_cli_celery.py | 17 ++ apps/cic-cache/tests/cli/test_cli_chain.py | 68 ++++++ apps/cic-cache/tests/conftest.py | 39 +++- apps/cic-cache/tests/test_cache.py | 212 +++++++++++++++++- apps/cic-cache/tests/testdata/config/test.ini | 2 + apps/cic-eth/cic_eth/api/api_task.py | 2 +- apps/cic-eth/cic_eth/cli/rpc.py | 18 +- .../cic_eth/data/config/dispatcher.ini | 2 + apps/cic-eth/cic_eth/data/config/eth.ini | 2 +- apps/cic-eth/cic_eth/data/config/retry.ini | 3 + apps/cic-eth/cic_eth/ext/tx.py | 58 ++++- apps/cic-eth/config/database.ini | 10 - apps/cic-eth/config/docker/chain.ini | 2 + apps/cic-eth/config/docker/database.ini | 10 - apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/services_requirements.txt | 2 +- .../tests/{unit/ext => task}/test_ext_tx.py | 17 +- apps/cic-eth/tests/testdata/config/test.ini | 2 + apps/cic-eth/tests/unit/cli/test_cli_args.py | 56 +++++ .../cic-eth/tests/unit/cli/test_cli_celery.py | 17 ++ apps/cic-eth/tests/unit/cli/test_cli_chain.py | 64 ++++++ docker-compose.yml | 2 +- 33 files changed, 833 insertions(+), 91 deletions(-) create mode 100644 apps/cic-cache/tests/cli/test_cli_args.py create mode 100644 apps/cic-cache/tests/cli/test_cli_celery.py create mode 100644 apps/cic-cache/tests/cli/test_cli_chain.py create mode 100644 apps/cic-cache/tests/testdata/config/test.ini create mode 100644 apps/cic-eth/cic_eth/data/config/dispatcher.ini create mode 100644 apps/cic-eth/cic_eth/data/config/retry.ini delete mode 100644 apps/cic-eth/config/database.ini create mode 100644 apps/cic-eth/config/docker/chain.ini delete mode 100644 apps/cic-eth/config/docker/database.ini rename apps/cic-eth/tests/{unit/ext => task}/test_ext_tx.py (84%) create mode 100644 apps/cic-eth/tests/testdata/config/test.ini create mode 100644 apps/cic-eth/tests/unit/cli/test_cli_args.py create mode 100644 apps/cic-eth/tests/unit/cli/test_cli_celery.py create mode 100644 apps/cic-eth/tests/unit/cli/test_cli_chain.py diff --git a/apps/cic-cache/cic_cache/api.py b/apps/cic-cache/cic_cache/api.py index 0340e65c..74083ba5 100644 --- a/apps/cic-cache/cic_cache/api.py +++ b/apps/cic-cache/cic_cache/api.py @@ -55,15 +55,37 @@ class Api: queue=callback_queue, ) - def list(self, offset, limit, address=None): + def list(self, offset=0, limit=100, address=None, oldest=False): s = celery.signature( 'cic_cache.tasks.tx.tx_filter', [ - 0, - 100, + offset, + limit, address, + oldest, ], - queue=None + queue=self.queue, + ) + if self.callback_param != None: + s.link(self.callback_success).on_error(self.callback_error) + + t = s.apply_async() + + return t + + + def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False): + s = celery.signature( + 'cic_cache.tasks.tx.tx_filter_content', + [ + offset, + limit, + address, + block_offset, + block_limit, + oldest, + ], + queue=self.queue, ) if self.callback_param != None: s.link(self.callback_success).on_error(self.callback_error) diff --git a/apps/cic-cache/cic_cache/cache.py b/apps/cic-cache/cic_cache/cache.py index 7c4d1040..10e8c458 100644 --- a/apps/cic-cache/cic_cache/cache.py +++ b/apps/cic-cache/cic_cache/cache.py @@ -10,12 +10,16 @@ from cic_cache.db.list import ( list_transactions_mined, list_transactions_account_mined, list_transactions_mined_with_data, + list_transactions_mined_with_data_index, + list_transactions_account_mined_with_data_index, + list_transactions_account_mined_with_data, ) logg = logging.getLogger() DEFAULT_FILTER_SIZE = 8192 * 8 +DEFAULT_LIMIT = 100 class Cache: @@ -32,7 +36,7 @@ class BloomCache(Cache): return n - def load_transactions(self, offset, limit): + def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False): """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions. Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index. @@ -49,7 +53,7 @@ class BloomCache(Cache): :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx :rtype: tuple """ - rows = list_transactions_mined(self.session, offset, limit) + rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) @@ -58,7 +62,12 @@ class BloomCache(Cache): for r in rows: if highest_block == -1: highest_block = r[0] - lowest_block = r[0] + lowest_block = r[0] + else: + if oldest: + highest_block = r[0] + else: + lowest_block = r[0] block = r[0].to_bytes(4, byteorder='big') tx = r[1].to_bytes(4, byteorder='big') f_block.add(block) @@ -67,7 +76,7 @@ class BloomCache(Cache): return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),) - def load_transactions_account(self, address, offset, limit): + def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False): """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient. :param address: Address to retrieve transactions for. @@ -79,7 +88,7 @@ class BloomCache(Cache): :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx :rtype: tuple """ - rows = list_transactions_account_mined(self.session, address, offset, limit) + rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3) @@ -88,7 +97,12 @@ class BloomCache(Cache): for r in rows: if highest_block == -1: highest_block = r[0] - lowest_block = r[0] + lowest_block = r[0] + else: + if oldest: + highest_block = r[0] + else: + lowest_block = r[0] block = r[0].to_bytes(4, byteorder='big') tx = r[1].to_bytes(4, byteorder='big') f_block.add(block) @@ -99,8 +113,21 @@ class BloomCache(Cache): class DataCache(Cache): - def load_transactions_with_data(self, offset, end): - rows = list_transactions_mined_with_data(self.session, offset, end) + def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False): + if limit == 0: + limit = DEFAULT_LIMIT + rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) + return self.__process_rows(rows, oldest) + + + def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False): + if limit == 0: + limit = DEFAULT_LIMIT + rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) + return self.__process_rows(rows, oldest) + + + def __process_rows(self, rows, oldest): tx_cache = [] highest_block = -1; lowest_block = -1; @@ -108,7 +135,12 @@ class DataCache(Cache): for r in rows: if highest_block == -1: highest_block = r['block_number'] - lowest_block = r['block_number'] + lowest_block = r['block_number'] + else: + if oldest: + highest_block = r['block_number'] + else: + lowest_block = r['block_number'] tx_type = 'unknown' if r['value'] != None: diff --git a/apps/cic-cache/cic_cache/cli/arg.py b/apps/cic-cache/cic_cache/cli/arg.py index 43428cb1..2d4e6e8a 100644 --- a/apps/cic-cache/cic_cache/cli/arg.py +++ b/apps/cic-cache/cic_cache/cli/arg.py @@ -12,7 +12,7 @@ class ArgumentParser(BaseArgumentParser): def process_local_flags(self, local_arg_flags): if local_arg_flags & CICFlag.CELERY: - self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue') + self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue') if local_arg_flags & CICFlag.SYNCER: self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync') self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync') diff --git a/apps/cic-cache/cic_cache/data/config/celery.ini b/apps/cic-cache/cic_cache/data/config/celery.ini index f2ad10ab..a799cb4a 100644 --- a/apps/cic-cache/cic_cache/data/config/celery.ini +++ b/apps/cic-cache/cic_cache/data/config/celery.ini @@ -1,5 +1,5 @@ [celery] broker_url = redis://localhost:6379 result_url = -queue = cic-eth +queue = cic-cache debug = 0 diff --git a/apps/cic-cache/cic_cache/db/list.py b/apps/cic-cache/cic_cache/db/list.py index 62e0a2e0..4052471e 100644 --- a/apps/cic-cache/cic_cache/db/list.py +++ b/apps/cic-cache/cic_cache/db/list.py @@ -13,6 +13,9 @@ def list_transactions_mined( session, offset, limit, + block_offset, + block_limit, + oldest=False, ): """Executes db query to return all confirmed transactions according to the specified offset and limit. @@ -23,15 +26,62 @@ def list_transactions_mined( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset) + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset) r = session.execute(s) return r def list_transactions_mined_with_data( + session, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit. + + :param block_offset: First block to include in search + :type block_offset: int + :param block_limit: Last block to include in search + :type block_limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset) + + + r = session.execute(s) + return r + + +def list_transactions_mined_with_data_index( session, offset, end, + block_offset, + block_limit, + oldest=False, ): """Executes db query to return all confirmed transactions according to the specified offset and limit. @@ -42,7 +92,87 @@ def list_transactions_mined_with_data( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end) + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end) + + r = session.execute(s) + return r + + +def list_transactions_account_mined_with_data_index( + session, + address, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address + + :param offset: Offset in data set to return transactions from + :type offset: int + :param limit: Max number of transactions to retrieve + :type limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) + + r = session.execute(s) + return r + +def list_transactions_account_mined_with_data( + session, + address, + offset, + limit, + block_offset, + block_limit, + oldest=False, + ): + """Executes db query to return all confirmed transactions according to the specified offset and limit. + + :param block_offset: First block to include in search + :type block_offset: int + :param block_limit: Last block to include in search + :type block_limit: int + :result: Result set + :rtype: SQLAlchemy.ResultProxy + """ + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) r = session.execute(s) return r @@ -53,6 +183,9 @@ def list_transactions_account_mined( address, offset, limit, + block_offset, + block_limit, + oldest=False, ): """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient. @@ -65,7 +198,20 @@ def list_transactions_account_mined( :result: Result set :rtype: SQLAlchemy.ResultProxy """ - s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset) + + order_by = 'DESC' + if oldest: + order_by = 'ASC' + + if block_offset: + if block_limit: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset) + else: + s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset) + + else: + s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset) + r = session.execute(s) return r diff --git a/apps/cic-cache/cic_cache/runnable/daemons/query.py b/apps/cic-cache/cic_cache/runnable/daemons/query.py index a698a692..d7edf6ab 100644 --- a/apps/cic-cache/cic_cache/runnable/daemons/query.py +++ b/apps/cic-cache/cic_cache/runnable/daemons/query.py @@ -91,13 +91,14 @@ def process_transactions_all_data(session, env): if env.get('HTTP_X_CIC_CACHE_MODE') != 'all': return None - offset = r[1] - end = r[2] + logg.debug('got data request {}'.format(env)) + block_offset = r[1] + block_end = r[2] if int(r[2]) < int(r[1]): raise ValueError('cart before the horse, dude') c = DataCache(session) - (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end) + (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable for r in tx_cache: r['date_block'] = r['date_block'].timestamp() diff --git a/apps/cic-cache/cic_cache/tasks/tx.py b/apps/cic-cache/cic_cache/tasks/tx.py index 0e2f7435..7524fe6f 100644 --- a/apps/cic-cache/cic_cache/tasks/tx.py +++ b/apps/cic-cache/cic_cache/tasks/tx.py @@ -2,14 +2,17 @@ import celery # local imports -from cic_cache.cache import BloomCache +from cic_cache.cache import ( + BloomCache, + DataCache, + ) from cic_cache.db.models.base import SessionBase celery_app = celery.current_app @celery_app.task(bind=True) -def tx_filter(self, offset, limit, address=None, encoding='hex'): +def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'): queue = self.request.delivery_info.get('routing_key') session = SessionBase.create_session() @@ -17,9 +20,9 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'): c = BloomCache(session) b = None if address == None: - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest) else: - (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit) + (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest) session.close() @@ -35,4 +38,17 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'): return o +@celery_app.task(bind=True) +def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'): + session = SessionBase.create_session() + c = DataCache(session) + b = None + if address == None: + (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) + else: + (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit) + + session.close() + + return (lowest_block, highest_block, tx_cache,) diff --git a/apps/cic-cache/cic_cache/version.py b/apps/cic-cache/cic_cache/version.py index 338bb831..b5293a59 100644 --- a/apps/cic-cache/cic_cache/version.py +++ b/apps/cic-cache/cic_cache/version.py @@ -4,8 +4,8 @@ import semver version = ( 0, 2, - 0, - 'alpha.2', + 1, + 'alpha.1', ) version_object = semver.VersionInfo( diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index e01c9bbe..40e9bdeb 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -9,6 +9,7 @@ psycopg2==2.8.6 celery==4.4.7 redis==3.5.3 chainsyncer[sql]>=0.0.6a1,<0.1.0 -erc20-faucet~=0.2.4a1 -chainlib>=0.0.7a1,<0.1.0 +erc20-faucet>=0.2.4a2, <0.3.0 +chainlib-eth>=0.0.7a3,<0.1.0 +chainlib>=0.0.7a3,<0.1.0 eth-address-index>=0.1.4a1,<0.2.0 diff --git a/apps/cic-cache/setup.cfg b/apps/cic-cache/setup.cfg index b6de6a9c..f8393c92 100644 --- a/apps/cic-cache/setup.cfg +++ b/apps/cic-cache/setup.cfg @@ -41,3 +41,4 @@ console_scripts = cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main cic-cache-serverd = cic_cache.runnable.daemons.server:main cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main + cic-cache-list = cic_cache.runable.list:main diff --git a/apps/cic-cache/tests/cli/test_cli_args.py b/apps/cic-cache/tests/cli/test_cli_args.py new file mode 100644 index 00000000..6e0c8a26 --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_args.py @@ -0,0 +1,40 @@ +# standard imports +import os + +# external imports +import chainlib.cli + +# local imports +import cic_cache.cli + +script_dir = os.path.dirname(os.path.realpath(__file__)) +config_dir = os.path.join(script_dir, '..', 'testdata', 'config') + + +def test_argumentparserto_config(): + + argparser = cic_cache.cli.ArgumentParser() + + local_flags = 0xffff + argparser.process_local_flags(local_flags) + argparser.add_argument('--foo', type=str) + args = argparser.parse_args([ + '-q', 'baz', + '--offset', '13', + '--no-history', + '-r','0xdeadbeef', + '-vv', + '--foo', 'bar', + ]) + + extra_args = { + 'foo': '_BARBARBAR', + } + config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir) + + assert config.get('_BARBARBAR') == 'bar' + assert config.get('CELERY_QUEUE') == 'baz' + assert config.get('SYNCER_NO_HISTORY') == True + assert config.get('SYNCER_OFFSET') == 13 + assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef' + diff --git a/apps/cic-cache/tests/cli/test_cli_celery.py b/apps/cic-cache/tests/cli/test_cli_celery.py new file mode 100644 index 00000000..81d975af --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_celery.py @@ -0,0 +1,17 @@ +# standard imports +import tempfile + +# local imports +import cic_cache.cli + + +def test_cli_celery(): + cf = tempfile.mkdtemp() + + config = { + 'CELERY_RESULT_URL': 'filesystem://' + cf, + } + cic_cache.cli.CeleryApp.from_config(config) + + config['CELERY_BROKER_URL'] = 'filesystem://' + cf + cic_cache.cli.CeleryApp.from_config(config) diff --git a/apps/cic-cache/tests/cli/test_cli_chain.py b/apps/cic-cache/tests/cli/test_cli_chain.py new file mode 100644 index 00000000..0cf9d5da --- /dev/null +++ b/apps/cic-cache/tests/cli/test_cli_chain.py @@ -0,0 +1,68 @@ +# external imports +import pytest +from chainlib.eth.gas import ( + Gas, + RPCGasOracle, + ) +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.block import ( + block_latest, + Block, + ) +from chainlib.eth.pytest.fixtures_chain import default_chain_spec +from chainlib.eth.pytest.fixtures_ethtester import * +from cic_eth_registry.pytest.fixtures_contracts import * +from hexathon import add_0x + +# local imports +import cic_cache.cli + + +@pytest.mark.xfail() +def test_cli_rpc( + eth_rpc, + eth_signer, + default_chain_spec, + ): + config = { + 'CHAIN_SPEC': str(default_chain_spec), + 'RPC_HTTP_PROVIDER': 'http://localhost:8545', + } + rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') + conn = rpc.get_by_label('foo') + #o = block_latest() + #conn.do(o) + + +def test_cli_chain( + default_chain_spec, + eth_rpc, + eth_signer, + contract_roles, + ): + ifc = cic_cache.cli.EthChainInterface() + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc) + gas_oracle = RPCGasOracle(conn=eth_rpc) + c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer) + recipient = add_0x(os.urandom(20).hex()) + (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024) + r = eth_rpc.do(o) + + o = ifc.tx_receipt(r) + r = eth_rpc.do(o) + assert r['status'] == 1 + + o = ifc.block_by_number(1) + block_src = eth_rpc.do(o) + block = ifc.block_from_src(block_src) + assert block.number == 1 + + with pytest.raises(KeyError): + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + + block_src = ifc.src_normalize(block_src) + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + diff --git a/apps/cic-cache/tests/conftest.py b/apps/cic-cache/tests/conftest.py index 61db702f..478c261f 100644 --- a/apps/cic-cache/tests/conftest.py +++ b/apps/cic-cache/tests/conftest.py @@ -64,7 +64,6 @@ def txs( dt.timestamp(), ) - tx_number = 42 tx_hash_second = '0x' + os.urandom(32).hex() tx_signed_second = '0x' + os.urandom(128).hex() @@ -93,6 +92,44 @@ def txs( ] +@pytest.fixture(scope='function') +def more_txs( + init_database, + list_defaults, + list_actors, + list_tokens, + txs, + ): + + session = init_database + + tx_number = 666 + tx_hash = '0x' + os.urandom(32).hex() + tx_signed = '0x' + os.urandom(128).hex() + nonce = 3 + + dt = datetime.datetime.utcnow() + dt += datetime.timedelta(hours=1) + db.add_transaction( + session, + tx_hash, + list_defaults['block']+2, + tx_number, + list_actors['alice'], + list_actors['diane'], + list_tokens['bar'], + list_tokens['bar'], + 2048, + 4096, + False, + dt.timestamp(), + ) + + session.commit() + + return [tx_hash] + txs + + @pytest.fixture(scope='function') def tag_txs( init_database, diff --git a/apps/cic-cache/tests/test_cache.py b/apps/cic-cache/tests/test_cache.py index cdd038c1..a5fdcc0d 100644 --- a/apps/cic-cache/tests/test_cache.py +++ b/apps/cic-cache/tests/test_cache.py @@ -8,6 +8,7 @@ import json import pytest # local imports +from cic_cache import db from cic_cache import BloomCache from cic_cache.cache import DataCache @@ -18,7 +19,6 @@ def test_cache( init_database, list_defaults, list_actors, - list_tokens, txs, ): @@ -37,9 +37,6 @@ def test_cache( def test_cache_data( init_database, - list_defaults, - list_actors, - list_tokens, txs, tag_txs, ): @@ -47,10 +44,209 @@ def test_cache_data( session = init_database c = DataCache(session) - b = c.load_transactions_with_data(410000, 420000) + b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True) assert len(b[2]) == 2 - assert b[2][0]['tx_hash'] == txs[1] - assert b[2][1]['tx_type'] == 'unknown' - assert b[2][0]['tx_type'] == 'test.taag' + assert b[2][0]['tx_hash'] == txs[0] + assert b[2][0]['tx_type'] == 'unknown' + assert b[2][1]['tx_type'] == 'test.taag' + + +def test_cache_ranges( + init_database, + list_defaults, + list_actors, + list_tokens, + more_txs, + ): + + session = init_database + + oldest = list_defaults['block'] - 1 + mid = list_defaults['block'] + newest = list_defaults['block'] + 2 + + c = BloomCache(session) + b = c.load_transactions(0, 100) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions(1, 2) + assert b[0] == oldest + assert b[1] == mid + + b = c.load_transactions(0, 2) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions(0, 1) + assert b[0] == newest + assert b[1] == newest + + b = c.load_transactions(0, 100, oldest=True) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions(0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == mid + + b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True) + assert b[0] == oldest + assert b[1] == mid + + # now check when supplying account + b = c.load_transactions_account(list_actors['alice'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + + b = c.load_transactions_account(list_actors['bob'], 0, 100) + assert b[0] == mid + assert b[1] == mid + + b = c.load_transactions_account(list_actors['diane'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + + # add block filter to the mix + b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + + b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == mid + assert b[1] == mid + + b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == oldest + + +def test_cache_ranges_data( + init_database, + list_defaults, + list_actors, + list_tokens, + more_txs, + ): + + session = init_database + + oldest = list_defaults['block'] - 1 + mid = list_defaults['block'] + newest = list_defaults['block'] + 2 + + c = DataCache(session) + + b = c.load_transactions_with_data(0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][2]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(1, 2) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[1] + assert b[2][1]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(0, 2) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_with_data(0, 1) + assert b[0] == newest + assert b[1] == newest + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[0] + + b = c.load_transactions_with_data(0, 100, oldest=True) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[2] + assert b[2][1]['tx_hash'] == more_txs[1] + assert b[2][2]['tx_hash'] == more_txs[0] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[1] + assert b[2][1]['tx_hash'] == more_txs[2] + + b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True) + assert b[0] == oldest + assert b[1] == mid + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[2] + assert b[2][1]['tx_hash'] == more_txs[1] + + # now check when supplying account + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 3 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + assert b[2][2]['tx_hash'] == more_txs[2] + + b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100) + assert b[0] == mid + assert b[1] == mid + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100) + assert b[0] == oldest + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[2] + + # add block filter to the mix + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block']) + assert b[0] == mid + assert b[1] == newest + assert len(b[2]) == 2 + assert b[2][0]['tx_hash'] == more_txs[0] + assert b[2][1]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == mid + assert b[1] == mid + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[1] + + b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block']) + assert b[0] == oldest + assert b[1] == oldest + assert len(b[2]) == 1 + assert b[2][0]['tx_hash'] == more_txs[2] diff --git a/apps/cic-cache/tests/testdata/config/test.ini b/apps/cic-cache/tests/testdata/config/test.ini new file mode 100644 index 00000000..70903618 --- /dev/null +++ b/apps/cic-cache/tests/testdata/config/test.ini @@ -0,0 +1,2 @@ +[foo] +bar_baz = xyzzy diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index 64448bfb..79501d30 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -520,9 +520,9 @@ class Api(ApiBase): s_external_get = celery.signature( external_task, [ - address, offset, limit, + address, ], queue=external_queue, ) diff --git a/apps/cic-eth/cic_eth/cli/rpc.py b/apps/cic-eth/cic_eth/cli/rpc.py index cc073767..25e28f9b 100644 --- a/apps/cic-eth/cic_eth/cli/rpc.py +++ b/apps/cic-eth/cic_eth/cli/rpc.py @@ -24,20 +24,24 @@ class RPC: def get_default(self): - return RPCConnection.connect(self.chain_spec, 'default') + return self.get_by_label('default') + + + def get_by_label(self, label): + return RPCConnection.connect(self.chain_spec, label) @staticmethod - def from_config(config, use_signer=False): + def from_config(config, use_signer=False, default_label='default', signer_label='signer'): chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC')) - RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default') + RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label) if use_signer: - RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer') - RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer') - RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer') - RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer') + RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label) + RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label) + RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER')) logg.info('set up rpc: {}'.format(rpc)) return rpc diff --git a/apps/cic-eth/cic_eth/data/config/dispatcher.ini b/apps/cic-eth/cic_eth/data/config/dispatcher.ini new file mode 100644 index 00000000..40e1b86e --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/dispatcher.ini @@ -0,0 +1,2 @@ +[dispatcher] +loop_interval = 1 diff --git a/apps/cic-eth/cic_eth/data/config/eth.ini b/apps/cic-eth/cic_eth/data/config/eth.ini index 3bba24bf..fd58e918 100644 --- a/apps/cic-eth/cic_eth/data/config/eth.ini +++ b/apps/cic-eth/cic_eth/data/config/eth.ini @@ -1,2 +1,2 @@ [eth] -gas_gifter_minimum_balance = 10000000000000000000 +gas_gifter_minimum_balance = 10000000000000000000000 diff --git a/apps/cic-eth/cic_eth/data/config/retry.ini b/apps/cic-eth/cic_eth/data/config/retry.ini new file mode 100644 index 00000000..4bec6b6b --- /dev/null +++ b/apps/cic-eth/cic_eth/data/config/retry.ini @@ -0,0 +1,3 @@ +[retry] +delay = +batch_size = diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py index 77083874..3adefa2e 100644 --- a/apps/cic-eth/cic_eth/ext/tx.py +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -12,9 +12,11 @@ from chainlib.eth.tx import ( transaction_by_block, receipt, ) +from chainlib.eth.error import RequestMismatchException from chainlib.eth.block import block_by_number from chainlib.eth.contract import abi_decode_single from chainlib.eth.constant import ZERO_ADDRESS +from chainlib.eth.tx import Tx from hexathon import strip_0x from cic_eth_registry import CICRegistry from cic_eth_registry.erc20 import ERC20Token @@ -23,6 +25,8 @@ from chainqueue.db.models.otx import Otx from chainqueue.db.enum import StatusEnum from chainqueue.sql.query import get_tx_cache from eth_erc20 import ERC20 +from erc20_faucet import Faucet +from potaahto.symbols import snake_and_camel # local imports from cic_eth.queue.time import tx_times @@ -35,6 +39,32 @@ logg = logging.getLogger() MAX_BLOCK_TX = 250 +def parse_transaction(chain_spec, rpc, tx, sender_address=None): + try: + transfer_data = ERC20.parse_transfer_request(tx['input']) + tx_address = transfer_data[0] + tx_token_value = transfer_data[1] + logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value)) + return (tx_address, tx_token_value) + except RequestMismatchException: + pass + + try: + transfer_data = Faucet.parse_give_to_request(tx['input']) + tx_address = transfer_data[0] + c = Faucet(chain_spec) + o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number']) + r = rpc.do(o) + tx_token_value = Faucet.parse_token_amount(r) + logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value)) + return (tx_address, tx_token_value) + + except RequestMismatchException: + pass + + return None + + # TODO: Make this method easier to read @celery_app.task(bind=True, base=BaseTask) def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): @@ -71,36 +101,39 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data) txs = {} + logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high'])) for block_height in range(bloomspec['low'], bloomspec['high']): block_height_bytes = block_height.to_bytes(4, 'big') if block_filter.check(block_height_bytes): logg.debug('filter matched block {}'.format(block_height)) o = block_by_number(block_height) block = rpc.do(o) - logg.debug('block {}'.format(block)) for tx_index in range(0, len(block['transactions'])): - composite = tx_index + block_height - tx_index_bytes = composite.to_bytes(4, 'big') - if tx_filter.check(tx_index_bytes): + tx_index_bytes = tx_index.to_bytes(4, 'big') + composite = block_height_bytes + tx_index_bytes + if tx_filter.check(composite): logg.debug('filter matched block {} tx {}'.format(block_height, tx_index)) + o = transaction_by_block(block['hash'], tx_index) try: - #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index) - o = transaction_by_block(block['hash'], tx_index) tx = rpc.do(o) except Exception as e: logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e)) continue + + tx = Tx(tx).src() + + logg.debug('got tx {}'.format(tx)) tx_address = None tx_token_value = 0 - try: - transfer_data = ERC20.parse_transfer_request(tx['data']) - tx_address = transfer_data[0] - tx_token_value = transfer_data[1] - except ValueError: - logg.debug('not a transfer transaction, skipping {}'.format(tx)) + + transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address) + if transfer_data == None: continue + tx_address = transfer_data[0] + tx_token_value = transfer_data[1] + if address == tx_address: status = StatusEnum.SENT try: @@ -136,6 +169,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict): return txs + # TODO: Surely it must be possible to optimize this # TODO: DRY this with callback filter in cic_eth/runnable/manager # TODO: Remove redundant fields from end representation (timestamp, tx_hash) diff --git a/apps/cic-eth/config/database.ini b/apps/cic-eth/config/database.ini deleted file mode 100644 index 6448be0e..00000000 --- a/apps/cic-eth/config/database.ini +++ /dev/null @@ -1,10 +0,0 @@ -[database] -NAME=cic_eth -USER=postgres -PASSWORD=tralala -HOST=localhost -PORT=63432 -ENGINE=postgresql -DRIVER=psycopg2 -POOL_SIZE=50 -DEBUG=0 diff --git a/apps/cic-eth/config/docker/chain.ini b/apps/cic-eth/config/docker/chain.ini new file mode 100644 index 00000000..3a23481e --- /dev/null +++ b/apps/cic-eth/config/docker/chain.ini @@ -0,0 +1,2 @@ +[chain] +spec = evm:bloxberg:8996 diff --git a/apps/cic-eth/config/docker/database.ini b/apps/cic-eth/config/docker/database.ini deleted file mode 100644 index 6448be0e..00000000 --- a/apps/cic-eth/config/docker/database.ini +++ /dev/null @@ -1,10 +0,0 @@ -[database] -NAME=cic_eth -USER=postgres -PASSWORD=tralala -HOST=localhost -PORT=63432 -ENGINE=postgresql -DRIVER=psycopg2 -POOL_SIZE=50 -DEBUG=0 diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 196a1284..968deb53 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.7a1,<0.1.0 +chainlib-eth>=0.0.7a5,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/services_requirements.txt b/apps/cic-eth/services_requirements.txt index 5308d2f4..80d8009c 100644 --- a/apps/cic-eth/services_requirements.txt +++ b/apps/cic-eth/services_requirements.txt @@ -9,7 +9,7 @@ liveness~=0.0.1a7 eth-address-index>=0.1.4a1,<0.2.0 eth-accounts-index>=0.0.14a1,<0.1.0 cic-eth-registry>=0.5.8a1,<0.6.0 -erc20-faucet>=0.2.4a1,<0.3.0 +erc20-faucet>=0.2.4a2,<0.3.0 erc20-transfer-authorization>=0.3.4a1,<0.4.0 sarafu-faucet>=0.0.5a2,<0.1.0 moolb~=0.1.1b2 diff --git a/apps/cic-eth/tests/unit/ext/test_ext_tx.py b/apps/cic-eth/tests/task/test_ext_tx.py similarity index 84% rename from apps/cic-eth/tests/unit/ext/test_ext_tx.py rename to apps/cic-eth/tests/task/test_ext_tx.py index 90241865..2740bf20 100644 --- a/apps/cic-eth/tests/unit/ext/test_ext_tx.py +++ b/apps/cic-eth/tests/task/test_ext_tx.py @@ -20,7 +20,6 @@ from cic_eth.db.models.nonce import ( logg = logging.getLogger() -# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state def test_filter_process( init_database, default_chain_spec, @@ -48,10 +47,10 @@ def test_filter_process( eth_rpc.do(o) o = receipt(tx_hash_hex) r = eth_rpc.do(o) - a = r['block_number'] - b.add(a.to_bytes(4, 'big')) - a = r['block_number'] + r['transaction_index'] - t.add(a.to_bytes(4, 'big')) + block_bytes = r['block_number'].to_bytes(4, 'big') + b.add(block_bytes) + tx_index_bytes = r['transaction_index'].to_bytes(4, 'big') + t.add(block_bytes + tx_index_bytes) tx_hashes.append(tx_hash_hex) # external tx @@ -61,10 +60,10 @@ def test_filter_process( eth_rpc.do(o) o = receipt(tx_hash_hex) r = eth_rpc.do(o) - a = r['block_number'] - b.add(a.to_bytes(4, 'big')) - a = r['block_number'] + r['transaction_index'] - t.add(a.to_bytes(4, 'big')) + block_bytes = r['block_number'].to_bytes(4, 'big') + b.add(block_bytes) + tx_index_bytes = r['transaction_index'].to_bytes(4, 'big') + t.add(block_bytes + tx_index_bytes) tx_hashes.append(tx_hash_hex) init_eth_tester.mine_blocks(10) diff --git a/apps/cic-eth/tests/testdata/config/test.ini b/apps/cic-eth/tests/testdata/config/test.ini new file mode 100644 index 00000000..70903618 --- /dev/null +++ b/apps/cic-eth/tests/testdata/config/test.ini @@ -0,0 +1,2 @@ +[foo] +bar_baz = xyzzy diff --git a/apps/cic-eth/tests/unit/cli/test_cli_args.py b/apps/cic-eth/tests/unit/cli/test_cli_args.py new file mode 100644 index 00000000..bda5f183 --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_args.py @@ -0,0 +1,56 @@ +# standard imports +import os +import logging + +# external imports +import chainlib.cli + +# local imports +import cic_eth.cli + +logg = logging.getLogger() + +script_dir = os.path.dirname(os.path.realpath(__file__)) +#config_dir = os.path.join(script_dir, '..', '..', 'testdata', 'config') + + +def test_argumentparser_to_config(): + + argparser = cic_eth.cli.ArgumentParser() + + local_flags = 0xffff + argparser.process_local_flags(local_flags) + argparser.add_argument('--foo', type=str) + args = argparser.parse_args([ + '--redis-host', 'foo', + '--redis-port', '123', + '--redis-db', '0', + '--redis-host-callback', 'bar', + '--redis-port-callback', '456', + '--redis-timeout', '10.0', + '-q', 'baz', + '--offset', '13', + '--no-history', + '-r','0xdeadbeef', + '-vv', + '--foo', 'bar', + ]) + + extra_args = { + 'foo': '_BARBARBAR', + } + #config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir) + config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args) + + assert config.get('_BARBARBAR') == 'bar' + assert config.get('REDIS_HOST') == 'foo' + assert config.get('REDIS_PORT') == 123 + assert config.get('REDIS_DB') == 0 + assert config.get('_REDIS_HOST_CALLBACK') == 'bar' + assert config.get('_REDIS_PORT_CALLBACK') == 456 + assert config.get('REDIS_TIMEOUT') == 10.0 + assert config.get('CELERY_QUEUE') == 'baz' + assert config.get('SYNCER_NO_HISTORY') == True + assert config.get('SYNCER_OFFSET') == 13 + assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef' + diff --git a/apps/cic-eth/tests/unit/cli/test_cli_celery.py b/apps/cic-eth/tests/unit/cli/test_cli_celery.py new file mode 100644 index 00000000..1727008b --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_celery.py @@ -0,0 +1,17 @@ +# standard imports +import tempfile + +# local imports +import cic_eth.cli + + +def test_cli_celery(): + cf = tempfile.mkdtemp() + + config = { + 'CELERY_RESULT_URL': 'filesystem://' + cf, + } + cic_eth.cli.CeleryApp.from_config(config) + + config['CELERY_BROKER_URL'] = 'filesystem://' + cf + cic_eth.cli.CeleryApp.from_config(config) diff --git a/apps/cic-eth/tests/unit/cli/test_cli_chain.py b/apps/cic-eth/tests/unit/cli/test_cli_chain.py new file mode 100644 index 00000000..8ebebc15 --- /dev/null +++ b/apps/cic-eth/tests/unit/cli/test_cli_chain.py @@ -0,0 +1,64 @@ +# external imports +import pytest +from chainlib.eth.gas import ( + Gas, + RPCGasOracle, + ) +from chainlib.eth.nonce import RPCNonceOracle +from chainlib.eth.block import ( + block_latest, + Block, + ) + +# local imports +import cic_eth.cli + + +@pytest.mark.xfail() +def test_cli_rpc( + eth_rpc, + eth_signer, + default_chain_spec, + ): + config = { + 'CHAIN_SPEC': str(default_chain_spec), + 'RPC_HTTP_PROVIDER': 'http://localhost:8545', + } + rpc = cic_eth.cli.RPC.from_config(config, default_label='foo') + conn = rpc.get_by_label('foo') + #o = block_latest() + #conn.do(o) + + +def test_cli_chain( + default_chain_spec, + eth_rpc, + eth_signer, + contract_roles, + agent_roles, + ): + ifc = cic_eth.cli.EthChainInterface() + + nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc) + gas_oracle = RPCGasOracle(conn=eth_rpc) + c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer) + (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], 1024) + r = eth_rpc.do(o) + + o = ifc.tx_receipt(r) + r = eth_rpc.do(o) + assert r['status'] == 1 + + o = ifc.block_by_number(1) + block_src = eth_rpc.do(o) + block = ifc.block_from_src(block_src) + assert block.number == 1 + + with pytest.raises(KeyError): + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + + block_src = ifc.src_normalize(block_src) + assert block_src['gasUsed'] == 21000 + assert block_src['gas_used'] == 21000 + diff --git a/docker-compose.yml b/docker-compose.yml index 57b6be2d..147ea44f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -238,7 +238,7 @@ services: - | if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \ + --wsgi-file /root/cic_cache/runnable/daemons/server.py \ --http :8000 \ --pyargv "-vv" From 452047b900391798c355e57acfafad8bcd4e1f71 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Tue, 17 Aug 2021 16:52:17 +0000 Subject: [PATCH 6/6] Fix configs after cic-base remove merges --- apps/cic-cache/requirements.txt | 6 ++++-- apps/cic-eth/requirements.txt | 2 +- apps/cic-eth/scripts/migrate.py | 22 +++++++++------------- apps/contract-migration/docker/Dockerfile | 3 ++- apps/contract-migration/requirements.txt | 6 +++--- apps/contract-migration/reset.sh | 6 +++++- docker-compose.yml | 1 + 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/apps/cic-cache/requirements.txt b/apps/cic-cache/requirements.txt index 40e9bdeb..890594bf 100644 --- a/apps/cic-cache/requirements.txt +++ b/apps/cic-cache/requirements.txt @@ -10,6 +10,8 @@ celery==4.4.7 redis==3.5.3 chainsyncer[sql]>=0.0.6a1,<0.1.0 erc20-faucet>=0.2.4a2, <0.3.0 -chainlib-eth>=0.0.7a3,<0.1.0 -chainlib>=0.0.7a3,<0.1.0 +#chainlib-eth==0.0.7a5,<0.1.0 +chainlib-eth==0.0.7a5 +#chainlib==0.0.7a4,<0.1.0 +chainlib==0.0.7a4 eth-address-index>=0.1.4a1,<0.2.0 diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 968deb53..078438ae 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -1,3 +1,3 @@ celery==4.4.7 -chainlib-eth>=0.0.7a5,<0.1.0 +chainlib-eth>=0.0.7a7,<0.1.0 semver==2.13.0 diff --git a/apps/cic-eth/scripts/migrate.py b/apps/cic-eth/scripts/migrate.py index d3602989..a316967e 100644 --- a/apps/cic-eth/scripts/migrate.py +++ b/apps/cic-eth/scripts/migrate.py @@ -10,6 +10,7 @@ from alembic.config import Config as AlembicConfig import confini from cic_eth.db import dsn_from_config +import cic_eth.cli logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() @@ -19,25 +20,20 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) dbdir = os.path.join(rootdir, 'cic_eth', 'db') migrationsdir = os.path.join(dbdir, 'migrations') -config_dir = os.path.join('/usr/local/etc/cic-eth') +arg_flags = cic_eth.cli.argflag_std_base -argparser = argparse.ArgumentParser() -argparser.add_argument('-c', type=str, default=config_dir, help='config file') -argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +argparser = cic_eth.cli.ArgumentParser(arg_flags) argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading') argparser.add_argument('-f', action='store_true', help='force action') -argparser.add_argument('-v', action='store_true', help='be verbose') -argparser.add_argument('-vv', action='store_true', help='be more verbose') args = argparser.parse_args() -if args.vv: - logging.getLogger().setLevel(logging.DEBUG) -elif args.v: - logging.getLogger().setLevel(logging.INFO) - -config = confini.Config(args.c, args.env_prefix) -config.process() +extra_args = { + 'migrations_dir': None, + 'reset': None, + 'f': '_FORCE_ACTION', + } +config = cic_eth.cli.Config.from_args(args, arg_flags, 0, extra_args=extra_args) config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'SSL') logg.debug('config:\n{}'.format(config)) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index e90e6995..5f2fd37c 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -30,13 +30,14 @@ COPY requirements.txt . ARG pip_index_url=https://pypi.org/simple ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG EXTRA_PIP_ARGS="" ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple" ARG pip_trusted_host=pypi.org RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url https://pypi.org/simple \ --force-reinstall \ --trusted-host $pip_trusted_host \ - --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \ + --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt COPY . . diff --git a/apps/contract-migration/requirements.txt b/apps/contract-migration/requirements.txt index d374e8d5..f71cb33b 100644 --- a/apps/contract-migration/requirements.txt +++ b/apps/contract-migration/requirements.txt @@ -1,10 +1,10 @@ cic-eth[tools]==0.12.2a4 eth-erc20>=0.0.12a1,<0.1.0 -erc20-demurrage-token>=0.0.3a1,<0.1.0 +erc20-demurrage-token>=0.0.3a4,<0.1.0 eth-address-index>=0.1.4a1,<0.2.0 eth-accounts-index>=0.0.14a1,<0.1.0 cic-eth-registry>=0.5.8a1,<0.6.0 erc20-faucet>=0.2.4a1,<0.3.0 erc20-transfer-authorization>=0.3.4a1,<0.4.0 -sarafu-faucet>=0.0.5a5,<0.1.0 -chainlib-eth>=0.0.7a1,<0.1.0 +sarafu-faucet>=0.0.5a7,<0.1.0 +chainlib-eth>=0.0.7a8,<0.1.0 diff --git a/apps/contract-migration/reset.sh b/apps/contract-migration/reset.sh index 3a9d1b7c..990e1dae 100755 --- a/apps/contract-migration/reset.sh +++ b/apps/contract-migration/reset.sh @@ -99,11 +99,15 @@ if [[ -n "${ETH_PROVIDER}" ]]; then >&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m" fi fi - DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww` + export _CONFINI_DIR=$CONFINI_DIR + unset CONFINI_DIR + DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` + export CONFINI_DIR=$_CONFINI_DIR else >&2 echo unknown token type $TOKEN_TYPE exit 1 fi + echo "giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT" giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT >&2 echo "deploy account index contract" diff --git a/docker-compose.yml b/docker-compose.yml index 147ea44f..08010b38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,6 +80,7 @@ services: pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple} pip_extra_args: $PIP_EXTRA_ARGS EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433} + EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest environment: CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS