From cebdb24ed6d313f7dca24beb3eadfad6901df31f Mon Sep 17 00:00:00 2001 From: Blair Vanderlugt Date: Sun, 7 Feb 2021 02:29:24 +0000 Subject: [PATCH] Cic notify build --- .gitlab-ci.yml | 1 + .gitmodules | 3 - apps/cic-notify | 1 - apps/cic-notify/.config/africastalking.ini | 4 + apps/cic-notify/.config/celery.ini | 3 + apps/cic-notify/.config/database.ini | 10 ++ apps/cic-notify/.config/tasks.ini | 4 + .../.config/test/africastalking.ini | 4 + apps/cic-notify/.config/test/celery.ini | 3 + apps/cic-notify/.config/test/database.ini | 10 ++ apps/cic-notify/.config/test/tasks.ini | 4 + apps/cic-notify/.gitignore | 4 + apps/cic-notify/.gitlab-ci.yml | 23 ++++ apps/cic-notify/CHANGELOG | 24 ++++ apps/cic-notify/cic_notify/__init__.py | 1 + apps/cic-notify/cic_notify/api.py | 55 +++++++++ apps/cic-notify/cic_notify/db/__init__.py | 34 ++++++ apps/cic-notify/cic_notify/db/enum.py | 7 ++ .../cic_notify/db/migrations/default/README | 1 + .../db/migrations/default/alembic.ini | 85 ++++++++++++++ .../cic_notify/db/migrations/default/env.py | 77 ++++++++++++ .../db/migrations/default/script.py.mako | 24 ++++ .../versions/b2aedf79b0b2_notify_sms_log.py | 48 ++++++++ .../cic_notify/db/models/__init__.py | 0 apps/cic-notify/cic_notify/db/models/base.py | 40 +++++++ .../cic_notify/db/models/notification.py | 27 +++++ apps/cic-notify/cic_notify/error.py | 11 ++ apps/cic-notify/cic_notify/runnable/tasker.py | 110 ++++++++++++++++++ apps/cic-notify/cic_notify/tasks/__init__.py | 1 + .../cic_notify/tasks/sms/__init__.py | 12 ++ .../cic_notify/tasks/sms/africastalking.py | 69 +++++++++++ apps/cic-notify/cic_notify/tasks/sms/db.py | 26 +++++ apps/cic-notify/cic_notify/tasks/sms/log.py | 26 +++++ apps/cic-notify/cic_notify/version.py | 47 ++++++++ apps/cic-notify/doc/texinfo/Makefile | 5 + apps/cic-notify/doc/texinfo/api.texi | 14 +++ apps/cic-notify/doc/texinfo/html/API.html | 61 ++++++++++ .../cic-notify/doc/texinfo/html/Overview.html | 72 ++++++++++++ apps/cic-notify/doc/texinfo/html/Run.html | 54 +++++++++ apps/cic-notify/doc/texinfo/html/Setup.html | 64 ++++++++++ apps/cic-notify/doc/texinfo/html/index.html | 94 +++++++++++++++ apps/cic-notify/doc/texinfo/index.texi | 28 +++++ apps/cic-notify/doc/texinfo/overview.texi | 22 ++++ apps/cic-notify/doc/texinfo/run.texi | 4 + apps/cic-notify/doc/texinfo/setup.texi | 17 +++ apps/cic-notify/docker/Dockerfile | 39 +++++++ apps/cic-notify/docker/Dockerfile.service | 13 +++ apps/cic-notify/docker/db.sh | 3 + apps/cic-notify/docker/start_tasker.sh | 5 + apps/cic-notify/requirements.txt | 5 + apps/cic-notify/scripts/migrate.py | 57 +++++++++ apps/cic-notify/scripts/send_sms.py | 12 ++ apps/cic-notify/setup.cfg | 47 ++++++++ apps/cic-notify/setup.py | 30 +++++ apps/cic-notify/test_requirements.txt | 5 + apps/cic-notify/tests/conftest.py | 31 +++++ apps/cic-notify/tests/fixtures_celery.py | 48 ++++++++ apps/cic-notify/tests/fixtures_config.py | 20 ++++ apps/cic-notify/tests/fixtures_database.py | 48 ++++++++ apps/cic-notify/tests/test_sms.py | 34 ++++++ docker-compose.yml | 46 ++++---- 61 files changed, 1650 insertions(+), 27 deletions(-) delete mode 160000 apps/cic-notify create mode 100644 apps/cic-notify/.config/africastalking.ini create mode 100644 apps/cic-notify/.config/celery.ini create mode 100644 apps/cic-notify/.config/database.ini create mode 100644 apps/cic-notify/.config/tasks.ini create mode 100644 apps/cic-notify/.config/test/africastalking.ini create mode 100644 apps/cic-notify/.config/test/celery.ini create mode 100644 apps/cic-notify/.config/test/database.ini create mode 100644 apps/cic-notify/.config/test/tasks.ini create mode 100644 apps/cic-notify/.gitignore create mode 100644 apps/cic-notify/.gitlab-ci.yml create mode 100644 apps/cic-notify/CHANGELOG create mode 100644 apps/cic-notify/cic_notify/__init__.py create mode 100644 apps/cic-notify/cic_notify/api.py create mode 100644 apps/cic-notify/cic_notify/db/__init__.py create mode 100644 apps/cic-notify/cic_notify/db/enum.py create mode 100644 apps/cic-notify/cic_notify/db/migrations/default/README create mode 100644 apps/cic-notify/cic_notify/db/migrations/default/alembic.ini create mode 100644 apps/cic-notify/cic_notify/db/migrations/default/env.py create mode 100644 apps/cic-notify/cic_notify/db/migrations/default/script.py.mako create mode 100644 apps/cic-notify/cic_notify/db/migrations/default/versions/b2aedf79b0b2_notify_sms_log.py create mode 100644 apps/cic-notify/cic_notify/db/models/__init__.py create mode 100644 apps/cic-notify/cic_notify/db/models/base.py create mode 100644 apps/cic-notify/cic_notify/db/models/notification.py create mode 100644 apps/cic-notify/cic_notify/error.py create mode 100644 apps/cic-notify/cic_notify/runnable/tasker.py create mode 100644 apps/cic-notify/cic_notify/tasks/__init__.py create mode 100644 apps/cic-notify/cic_notify/tasks/sms/__init__.py create mode 100644 apps/cic-notify/cic_notify/tasks/sms/africastalking.py create mode 100644 apps/cic-notify/cic_notify/tasks/sms/db.py create mode 100644 apps/cic-notify/cic_notify/tasks/sms/log.py create mode 100644 apps/cic-notify/cic_notify/version.py create mode 100644 apps/cic-notify/doc/texinfo/Makefile create mode 100644 apps/cic-notify/doc/texinfo/api.texi create mode 100644 apps/cic-notify/doc/texinfo/html/API.html create mode 100644 apps/cic-notify/doc/texinfo/html/Overview.html create mode 100644 apps/cic-notify/doc/texinfo/html/Run.html create mode 100644 apps/cic-notify/doc/texinfo/html/Setup.html create mode 100644 apps/cic-notify/doc/texinfo/html/index.html create mode 100644 apps/cic-notify/doc/texinfo/index.texi create mode 100644 apps/cic-notify/doc/texinfo/overview.texi create mode 100644 apps/cic-notify/doc/texinfo/run.texi create mode 100644 apps/cic-notify/doc/texinfo/setup.texi create mode 100644 apps/cic-notify/docker/Dockerfile create mode 100644 apps/cic-notify/docker/Dockerfile.service create mode 100644 apps/cic-notify/docker/db.sh create mode 100755 apps/cic-notify/docker/start_tasker.sh create mode 100644 apps/cic-notify/requirements.txt create mode 100644 apps/cic-notify/scripts/migrate.py create mode 100644 apps/cic-notify/scripts/send_sms.py create mode 100644 apps/cic-notify/setup.cfg create mode 100644 apps/cic-notify/setup.py create mode 100644 apps/cic-notify/test_requirements.txt create mode 100644 apps/cic-notify/tests/conftest.py create mode 100644 apps/cic-notify/tests/fixtures_celery.py create mode 100644 apps/cic-notify/tests/fixtures_config.py create mode 100644 apps/cic-notify/tests/fixtures_database.py create mode 100644 apps/cic-notify/tests/test_sms.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3053e25..62d59be4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ include: - local: 'apps/contract-migration/.gitlab-ci.yml' - local: 'apps/cic-eth/.gitlab-ci.yml' - local: 'apps/cic-ussd/.gitlab-ci.yml' + - local: 'apps/cic-notify/.gitlab-ci.yml' stages: - build diff --git a/.gitmodules b/.gitmodules index cb0dedb2..89a43eeb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "apps/cic-notify"] - path = apps/cic-notify - url = git@gitlab.com:grassrootseconomics/cic-notify.git [submodule "apps/cic-cache"] path = apps/cic-cache url = git@gitlab.com:grassrootseconomics/cic-cache.git diff --git a/apps/cic-notify b/apps/cic-notify deleted file mode 160000 index ba8e5989..00000000 --- a/apps/cic-notify +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ba8e5989fa369782ed086e8b10bdd3524e353fe4 diff --git a/apps/cic-notify/.config/africastalking.ini b/apps/cic-notify/.config/africastalking.ini new file mode 100644 index 00000000..a6f6814a --- /dev/null +++ b/apps/cic-notify/.config/africastalking.ini @@ -0,0 +1,4 @@ +[AFRICASTALKING] +api_username = foo +api_key = bar +api_sender_id = baz diff --git a/apps/cic-notify/.config/celery.ini b/apps/cic-notify/.config/celery.ini new file mode 100644 index 00000000..af9fe5b1 --- /dev/null +++ b/apps/cic-notify/.config/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url = redis:// +result_url = redis:// diff --git a/apps/cic-notify/.config/database.ini b/apps/cic-notify/.config/database.ini new file mode 100644 index 00000000..bd32185b --- /dev/null +++ b/apps/cic-notify/.config/database.ini @@ -0,0 +1,10 @@ +[DATABASE] +user = postgres +password = +host = localhost +port = 5432 +name = /tmp/cic-notify.db +#engine = postgresql +#driver = psycopg2 +engine = sqlite +driver = pysqlite diff --git a/apps/cic-notify/.config/tasks.ini b/apps/cic-notify/.config/tasks.ini new file mode 100644 index 00000000..a0110422 --- /dev/null +++ b/apps/cic-notify/.config/tasks.ini @@ -0,0 +1,4 @@ +[TASKS] +africastalking = cic_notify.tasks.sms.africastalking +db = cic_notify.tasks.sms.db +log = cic_notify.tasks.sms.log diff --git a/apps/cic-notify/.config/test/africastalking.ini b/apps/cic-notify/.config/test/africastalking.ini new file mode 100644 index 00000000..a6f6814a --- /dev/null +++ b/apps/cic-notify/.config/test/africastalking.ini @@ -0,0 +1,4 @@ +[AFRICASTALKING] +api_username = foo +api_key = bar +api_sender_id = baz diff --git a/apps/cic-notify/.config/test/celery.ini b/apps/cic-notify/.config/test/celery.ini new file mode 100644 index 00000000..30a519c7 --- /dev/null +++ b/apps/cic-notify/.config/test/celery.ini @@ -0,0 +1,3 @@ +[celery] +broker_url = filesystem:// +result_url = filesystem:// diff --git a/apps/cic-notify/.config/test/database.ini b/apps/cic-notify/.config/test/database.ini new file mode 100644 index 00000000..bd32185b --- /dev/null +++ b/apps/cic-notify/.config/test/database.ini @@ -0,0 +1,10 @@ +[DATABASE] +user = postgres +password = +host = localhost +port = 5432 +name = /tmp/cic-notify.db +#engine = postgresql +#driver = psycopg2 +engine = sqlite +driver = pysqlite diff --git a/apps/cic-notify/.config/test/tasks.ini b/apps/cic-notify/.config/test/tasks.ini new file mode 100644 index 00000000..a0110422 --- /dev/null +++ b/apps/cic-notify/.config/test/tasks.ini @@ -0,0 +1,4 @@ +[TASKS] +africastalking = cic_notify.tasks.sms.africastalking +db = cic_notify.tasks.sms.db +log = cic_notify.tasks.sms.log diff --git a/apps/cic-notify/.gitignore b/apps/cic-notify/.gitignore new file mode 100644 index 00000000..f47db3aa --- /dev/null +++ b/apps/cic-notify/.gitignore @@ -0,0 +1,4 @@ +__pycache__ +*.pyc +venv/ +.idea/ \ No newline at end of file diff --git a/apps/cic-notify/.gitlab-ci.yml b/apps/cic-notify/.gitlab-ci.yml new file mode 100644 index 00000000..ca34ea61 --- /dev/null +++ b/apps/cic-notify/.gitlab-ci.yml @@ -0,0 +1,23 @@ +.cic_notify_variables: + variables: + APP_NAME: cic-notify + DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile + +.this_changes_target: + rules: + - changes: + - $CONTEXT/$APP_NAME/* + +build-mr-cic-notify: + extends: + - .this_changes_target + - .py_build_merge_request + - .cic_notify_variables + +build-push-cic-notify: + extends: + - .this_changes_target + - .py_build_push + - .cic_notify_variables + + diff --git a/apps/cic-notify/CHANGELOG b/apps/cic-notify/CHANGELOG new file mode 100644 index 00000000..830d70f5 --- /dev/null +++ b/apps/cic-notify/CHANGELOG @@ -0,0 +1,24 @@ +- 0.3.2 + * Relax dependencies to compatible +- 0.3.1 + * Upgrade dependencies +- 0.3.0 + * Rehabilitate africastalking script + * Add queue identifiers +- 0.2.0 + * Remove unnecessarily complicated code +- 0.1.0 + * Simplify tasks +- 0.0.4 + * Add python api + * Add send sms tool + * Add database to tasker script + * Rename tasker script + * Fix no-commit bug in db notification +- 0.0.3 + * Add sms notify to db +- 0.0.2 + * Re-introduce log sms notify handler +- 0.0.1 + * Port code from cic-ussd + * Make dynamic task module loader diff --git a/apps/cic-notify/cic_notify/__init__.py b/apps/cic-notify/cic_notify/__init__.py new file mode 100644 index 00000000..0a0e47b0 --- /dev/null +++ b/apps/cic-notify/cic_notify/__init__.py @@ -0,0 +1 @@ +from .api import * diff --git a/apps/cic-notify/cic_notify/api.py b/apps/cic-notify/cic_notify/api.py new file mode 100644 index 00000000..789eb835 --- /dev/null +++ b/apps/cic-notify/cic_notify/api.py @@ -0,0 +1,55 @@ +# standard imports +import logging +import re + +# third-party imports +import celery + +# local imports +from cic_notify.tasks import sms + +app = celery.current_app +logging.basicConfig(level=logging.DEBUG) +logg = logging.getLogger() + +sms_tasks_matcher = r"^(cic_notify.tasks.sms)(\.\w+)?" + + +class Api: + # TODO: Implement callback strategy + def __init__(self, queue='cic-notify'): + """ + :param queue: The queue on which to execute notification tasks + :type queue: str + """ + registered_tasks = app.tasks + self.sms_tasks = [] + + for task in registered_tasks.keys(): + logg.debug(f'Found: {task} {registered_tasks[task]}') + match = re.match(sms_tasks_matcher, task) + if match: + self.sms_tasks.append(task) + + self.queue = queue + logg.info(f'api using queue: {self.queue}') + + def sms(self, message, recipient): + """This function chains all sms tasks in order to send a message, log and persist said data to disk + :param message: The message to be sent to the recipient. + :type message: str + :param recipient: The phone number of the recipient. + :type recipient: str + :return: a celery Task + :rtype: Celery.Task + """ + signatures = [] + for task in self.sms_tasks: + signature = celery.signature(task) + signatures.append(signature) + signature_group = celery.group(signatures) + result = signature_group.apply_async( + args=[message, recipient], + queue=self.queue + ) + return result diff --git a/apps/cic-notify/cic_notify/db/__init__.py b/apps/cic-notify/cic_notify/db/__init__.py new file mode 100644 index 00000000..890147ad --- /dev/null +++ b/apps/cic-notify/cic_notify/db/__init__.py @@ -0,0 +1,34 @@ +# standard imports +import os +import logging + +# local imports +from cic_notify.db.models.base import SessionBase + +logg = logging.getLogger() + + +def dsn_from_config(config): + scheme = config.get('DATABASE_ENGINE') + if config.get('DATABASE_DRIVER') != None: + scheme += '+{}'.format(config.get('DATABASE_DRIVER')) + + dsn = '' + if config.get('DATABASE_ENGINE') == 'sqlite': + dsn = '{}:///{}'.format( + scheme, + config.get('DATABASE_NAME'), + ) + + else: + dsn = '{}://{}:{}@{}:{}/{}'.format( + scheme, + config.get('DATABASE_USER'), + config.get('DATABASE_PASSWORD'), + config.get('DATABASE_HOST'), + config.get('DATABASE_PORT'), + config.get('DATABASE_NAME'), + ) + logg.debug('parsed dsn from config: {}'.format(dsn)) + return dsn + diff --git a/apps/cic-notify/cic_notify/db/enum.py b/apps/cic-notify/cic_notify/db/enum.py new file mode 100644 index 00000000..a7fdf4df --- /dev/null +++ b/apps/cic-notify/cic_notify/db/enum.py @@ -0,0 +1,7 @@ +import enum + +class NotificationStatusEnum(enum.Enum): + UNKNOWN = 'UNKNOWN' + +class NotificationTransportEnum(enum.Enum): + SMS = 'SMS' diff --git a/apps/cic-notify/cic_notify/db/migrations/default/README b/apps/cic-notify/cic_notify/db/migrations/default/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/apps/cic-notify/cic_notify/db/migrations/default/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/apps/cic-notify/cic_notify/db/migrations/default/alembic.ini b/apps/cic-notify/cic_notify/db/migrations/default/alembic.ini new file mode 100644 index 00000000..5b6e42b2 --- /dev/null +++ b/apps/cic-notify/cic_notify/db/migrations/default/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgres+psycopg2://postgres@localhost/cic-notify + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/apps/cic-notify/cic_notify/db/migrations/default/env.py b/apps/cic-notify/cic_notify/db/migrations/default/env.py new file mode 100644 index 00000000..70518a2e --- /dev/null +++ b/apps/cic-notify/cic_notify/db/migrations/default/env.py @@ -0,0 +1,77 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/apps/cic-notify/cic_notify/db/migrations/default/script.py.mako b/apps/cic-notify/cic_notify/db/migrations/default/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/apps/cic-notify/cic_notify/db/migrations/default/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/apps/cic-notify/cic_notify/db/migrations/default/versions/b2aedf79b0b2_notify_sms_log.py b/apps/cic-notify/cic_notify/db/migrations/default/versions/b2aedf79b0b2_notify_sms_log.py new file mode 100644 index 00000000..3bef4ccb --- /dev/null +++ b/apps/cic-notify/cic_notify/db/migrations/default/versions/b2aedf79b0b2_notify_sms_log.py @@ -0,0 +1,48 @@ +"""Notify sms log + +Revision ID: b2aedf79b0b2 +Revises: +Create Date: 2020-10-11 15:59:02.765157 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b2aedf79b0b2' +down_revision = None +branch_labels = None +depends_on = None + +status_enum = sa.Enum( + 'UNKNOWN', # the state of the message is not known + name='notification_status', + ) + +transport_enum = sa.Enum( + 'SMS', + name='notification_transport', + ) + +def upgrade(): + op.create_table('notification', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('transport', transport_enum, nullable=False), + sa.Column('status', status_enum, nullable=False), + sa.Column('status_code', sa.String(), nullable=True), + sa.Column('status_serial', sa.Integer(), nullable=False, server_default='0'), + sa.Column('recipient', sa.String(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('message', sa.String(), nullable=False), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('notification_recipient_transport_idx', 'notification', ['transport', 'recipient'], schema=None, unique=False) + + +def downgrade(): + op.drop_index('notification_recipient_transport_idx') + op.drop_table('notification') + status_enum.drop(op.get_bind(), checkfirst=False) + transport_enum.drop(op.get_bind(), checkfirst=False) diff --git a/apps/cic-notify/cic_notify/db/models/__init__.py b/apps/cic-notify/cic_notify/db/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apps/cic-notify/cic_notify/db/models/base.py b/apps/cic-notify/cic_notify/db/models/base.py new file mode 100644 index 00000000..d72b0e72 --- /dev/null +++ b/apps/cic-notify/cic_notify/db/models/base.py @@ -0,0 +1,40 @@ +# third-party imports +from sqlalchemy import Column, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +Model = declarative_base(name='Model') + + +class SessionBase(Model): + __abstract__ = True + + id = Column(Integer, primary_key=True) + + engine = None + session = None + query = None + + + @staticmethod + def create_session(): + session = sessionmaker(bind=SessionBase.engine) + SessionBase.session = session() + return SessionBase.session + + + @staticmethod + def _set_engine(engine): + SessionBase.engine = engine + + + @staticmethod + def build(): + Model.metadata.create_all(bind=SessionBase.engine) + + + @staticmethod + def connect(dsn): + e = create_engine(dsn) + SessionBase._set_engine(e) diff --git a/apps/cic-notify/cic_notify/db/models/notification.py b/apps/cic-notify/cic_notify/db/models/notification.py new file mode 100644 index 00000000..b5d1e195 --- /dev/null +++ b/apps/cic-notify/cic_notify/db/models/notification.py @@ -0,0 +1,27 @@ +# standard imports +import datetime + +# third-party imports +from sqlalchemy import Enum, Column, String, DateTime + +# local imports +from .base import SessionBase +from ..enum import NotificationStatusEnum, NotificationTransportEnum + + +class Notification(SessionBase): + __tablename__ = 'notification' + + transport = Column(Enum(NotificationTransportEnum)) + status = Column(Enum(NotificationStatusEnum)) + recipient = Column(String) + message = Column(String) + created = Column(DateTime, default=datetime.datetime.utcnow) + updated = Column(DateTime, default=datetime.datetime.utcnow) + + def __init__(self, transport, recipient, message, **kwargs): + super(Notification, self).__init__(**kwargs) + self.transport = transport + self.recipient = recipient + self.message = message + self.status = NotificationStatusEnum.UNKNOWN diff --git a/apps/cic-notify/cic_notify/error.py b/apps/cic-notify/cic_notify/error.py new file mode 100644 index 00000000..a2a68f61 --- /dev/null +++ b/apps/cic-notify/cic_notify/error.py @@ -0,0 +1,11 @@ +class NotInitializedError(Exception): + pass + + +class AlreadyInitializedError(Exception): + pass + + +class PleaseCommitFirstError(Exception): + """Raised when there exists uncommitted changes in the code while trying to build out the package.""" + pass diff --git a/apps/cic-notify/cic_notify/runnable/tasker.py b/apps/cic-notify/cic_notify/runnable/tasker.py new file mode 100644 index 00000000..286b68f4 --- /dev/null +++ b/apps/cic-notify/cic_notify/runnable/tasker.py @@ -0,0 +1,110 @@ +# standard imports +import os +import logging +import importlib +import argparse +import tempfile + +# third-party imports +import celery +import confini + +# local imports +from cic_notify.db.models.base import SessionBase +from cic_notify.db import dsn_from_config + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + +config_dir = os.path.join('/usr/local/etc/cic-notify') + +argparser = argparse.ArgumentParser() +argparser.add_argument('-c', type=str, default=config_dir, help='config file') +argparser.add_argument('-q', type=str, default='cic-notify', help='queue name for worker tasks') +argparser.add_argument('-v', action='store_true', help='be verbose') +argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +argparser.add_argument('-vv', action='store_true', help='be more verbose') +args = argparser.parse_args() + +if args.vv: + logging.getLogger().setLevel(logging.DEBUG) +elif args.v: + logging.getLogger().setLevel(logging.INFO) + +config = confini.Config(args.c, args.env_prefix) +config.process() +config.censor('PASSWORD', 'DATABASE') + +# connect to database +dsn = dsn_from_config(config) +SessionBase.connect(dsn) + +# verify database connection with minimal sanity query +session = SessionBase.create_session() +session.execute('select version_num from alembic_version') +session.close() + +# set up celery +app = celery.Celery(__name__) + +broker = config.get('CELERY_BROKER_URL') +if broker[:4] == 'file': + bq = tempfile.mkdtemp() + bp = tempfile.mkdtemp() + app.conf.update({ + 'broker_url': broker, + 'broker_transport_options': { + 'data_folder_in': bq, + 'data_folder_out': bq, + 'data_folder_processed': bp, + }, + }, + ) + logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp)) +else: + app.conf.update({ + 'broker_url': broker, + }) + +result = config.get('CELERY_RESULT_URL') +if result[:4] == 'file': + rq = tempfile.mkdtemp() + app.conf.update({ + 'result_backend': 'file://{}'.format(rq), + }) + logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq)) +else: + app.conf.update({ + 'result_backend': result, + }) + + +for key in config.store.keys(): + if key[:5] == 'TASKS': + logg.info(f'adding sms task from {key}') + module = importlib.import_module(config.store[key]) + if key == 'TASKS_AFRICASTALKING': + africastalking_notifier = module.AfricasTalkingNotifier + africastalking_notifier.initialize( + config.get('AFRICASTALKING_API_USERNAME'), + config.get('AFRICASTALKING_API_KEY'), + config.get('AFRICASTALKING_API_SENDER_ID') + ) + + +def main(): + argv = ['worker'] + if args.vv: + argv.append('--loglevel=DEBUG') + elif args.v: + argv.append('--loglevel=INFO') + argv.append('-Q') + argv.append(args.q) + argv.append('-n') + argv.append(args.q) + + app.worker_main(argv) + + +if __name__ == '__main__': + main() diff --git a/apps/cic-notify/cic_notify/tasks/__init__.py b/apps/cic-notify/cic_notify/tasks/__init__.py new file mode 100644 index 00000000..f7dcf98c --- /dev/null +++ b/apps/cic-notify/cic_notify/tasks/__init__.py @@ -0,0 +1 @@ +from . import sms diff --git a/apps/cic-notify/cic_notify/tasks/sms/__init__.py b/apps/cic-notify/cic_notify/tasks/sms/__init__.py new file mode 100644 index 00000000..9ca51081 --- /dev/null +++ b/apps/cic-notify/cic_notify/tasks/sms/__init__.py @@ -0,0 +1,12 @@ +# standard imports + +# third-party imports + +# local imports +import celery + +celery_app = celery.current_app + +from .africastalking import send +from .db import persist_notification +from .log import log diff --git a/apps/cic-notify/cic_notify/tasks/sms/africastalking.py b/apps/cic-notify/cic_notify/tasks/sms/africastalking.py new file mode 100644 index 00000000..4ee9d47b --- /dev/null +++ b/apps/cic-notify/cic_notify/tasks/sms/africastalking.py @@ -0,0 +1,69 @@ +# standard imports +import logging + +# third party imports +import celery +import africastalking + +# local imports +from cic_notify.error import NotInitializedError, AlreadyInitializedError + +logg = logging.getLogger() +celery_app = celery.current_app + + +class AfricasTalkingNotifier: + initiated = None + sender_id = None + + def __init__(self): + if not self.initiated: + raise NotInitializedError() + self.api_client = africastalking.SMS + + @staticmethod + def initialize(api_username, api_key, sender_id=None): + """ + :param api_username: + :type api_username: + :param api_key: + :type api_key: + :param sender_id: + :type sender_id: + """ + if AfricasTalkingNotifier.initiated: + raise AlreadyInitializedError() + africastalking.initialize(username=api_username, api_key=api_key) + + AfricasTalkingNotifier.sender_id = sender_id + AfricasTalkingNotifier.initiated = True + + def send(self, message, recipient): + """ + :param message: + :type message: + :param recipient: + :type recipient: + :return: + :rtype: + """ + if self.sender_id: + response = self.api_client.send(message=message, recipients=[recipient], sender_id=self.sender_id) + logg.debug(f'Africastalking response sender-id {response}') + else: + response = self.api_client.send(message=message, recipients=[recipient]) + logg.debug(f'africastalking response no-sender-id {response}') + + +@celery_app.task +def send(message, recipient): + """ + :param message: + :type message: + :param recipient: + :type recipient: + :return: + :rtype: + """ + africastalking_notifier = AfricasTalkingNotifier() + africastalking_notifier.send(message=message, recipient=recipient) diff --git a/apps/cic-notify/cic_notify/tasks/sms/db.py b/apps/cic-notify/cic_notify/tasks/sms/db.py new file mode 100644 index 00000000..ca623154 --- /dev/null +++ b/apps/cic-notify/cic_notify/tasks/sms/db.py @@ -0,0 +1,26 @@ +# standard imports + +# third-party imports +import celery + +# local imports +from cic_notify.db.models.notification import Notification +from cic_notify.db.enum import NotificationTransportEnum + +celery_app = celery.current_app + + +@celery_app.task +def persist_notification(recipient, message): + """ + :param recipient: + :type recipient: + :param message: + :type message: + :return: + :rtype: + """ + Notification.create_session() + notification = Notification(transport=NotificationTransportEnum.SMS, recipient=recipient, message=message) + Notification.session.add(notification) + Notification.session.commit() diff --git a/apps/cic-notify/cic_notify/tasks/sms/log.py b/apps/cic-notify/cic_notify/tasks/sms/log.py new file mode 100644 index 00000000..30de7741 --- /dev/null +++ b/apps/cic-notify/cic_notify/tasks/sms/log.py @@ -0,0 +1,26 @@ +# standard imports +import logging +import time + +# third-party imports +import celery + +celery_app = celery.current_app +logg = celery_app.log.get_default_logger() +local_logg = logging.getLogger(__name__) + + +@celery_app.task +def log(recipient, message): + """ + :param recipient: + :type recipient: + :param message: + :type message: + :return: + :rtype: + """ + timestamp = time.time() + log_string = f'[{timestamp}] {__name__} message to {recipient}: {message}' + logg.info(log_string) + local_logg.info(log_string) diff --git a/apps/cic-notify/cic_notify/version.py b/apps/cic-notify/cic_notify/version.py new file mode 100644 index 00000000..6c1fe32d --- /dev/null +++ b/apps/cic-notify/cic_notify/version.py @@ -0,0 +1,47 @@ +# standard imports +import logging +import time + +# third-party imports +import semver + +# local imports +from cic_notify.error import PleaseCommitFirstError + +logg = logging.getLogger() + +version = (0, 4, 0, 'alpha.2') + +version_object = semver.VersionInfo( + major=version[0], + minor=version[1], + patch=version[2], + prerelease=version[3], + ) + +version_string = str(version_object) + + +def git_hash(): + import subprocess + git_diff = subprocess.run(['git', 'diff'], capture_output=True) + if len(git_diff.stdout) > 0: + raise PleaseCommitFirstError() + + git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True) + git_hash_brief = git_hash.stdout.decode('utf-8')[:8] + return git_hash_brief + + +try: + version_git = git_hash() + version_string += '.build.{}'.format(version_git) +except FileNotFoundError: + time_string_pair = str(time.time()).split('.') + version_string += '+build.{}{:<09d}'.format( + time_string_pair[0], + int(time_string_pair[1]), + ) +logg.info(f'Final version string will be {version_string}') + +__version_string__ = version_string diff --git a/apps/cic-notify/doc/texinfo/Makefile b/apps/cic-notify/doc/texinfo/Makefile new file mode 100644 index 00000000..0944d469 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/Makefile @@ -0,0 +1,5 @@ +all: html + +.PHONY: html +html: + makeinfo --html --output html index.texi diff --git a/apps/cic-notify/doc/texinfo/api.texi b/apps/cic-notify/doc/texinfo/api.texi new file mode 100644 index 00000000..36a9c2e2 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/api.texi @@ -0,0 +1,14 @@ +@node API +@chapter API + +A setuptools file is provided to install @code{cic-notify} as a normal python package. + +A single Python API method is provided so far: + +@itemize @code +@item sms(recipient, content) +@end itemize + +@code{recipient} is a string with phone number in @code{msisdn} format with a @code{+} prefix. + +@code{content} is a UTF-8 string containing the message to be sent. diff --git a/apps/cic-notify/doc/texinfo/html/API.html b/apps/cic-notify/doc/texinfo/html/API.html new file mode 100644 index 00000000..843e28a8 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/html/API.html @@ -0,0 +1,61 @@ + + + + + + +API (Grassroots Economics CIC notify 0.2.0) + + + + + + + + + + + + + + + + +
+

+Previous: , Up: Top   [Contents]

+
+
+

4 API

+ +

A single Python API method is provided so far: +

+ + +

recipient is a string with phone number in msisdn format with a + prefix. +

+

content is a UTF-8 string containing the message to be sent. +

+ + + + diff --git a/apps/cic-notify/doc/texinfo/html/Overview.html b/apps/cic-notify/doc/texinfo/html/Overview.html new file mode 100644 index 00000000..102cc83b --- /dev/null +++ b/apps/cic-notify/doc/texinfo/html/Overview.html @@ -0,0 +1,72 @@ + + + + + + +Overview (Grassroots Economics CIC notify 0.2.0) + + + + + + + + + + + + + + + + + +
+

+Next: , Up: Top   [Contents]

+
+
+

1 Overview

+ +

This system is in early stages of development. It is intended to be a flexible notification broker where additional notification targets easily can be plugged in. +

+

The framework is designed to asynchronously execute all tasks belonging to a specific context, based on the name of the task. +

+

Currently, only handlers for sms notifications are implemented. Any task with a notify.sms. prefix registered in the celery task worker pool will be executed upon the high-level "send sms" task. +

+

Similary, any other notification category can be implemented. e.g. notify.email., notify.telegram., notify.mattermost. etc. +

+

1.1 Contents

+ +

The only implementations so far are three sms notification tasks: +

+ + +

NOTE: The Africas Talking API will be removed from the suite, and provided as an add-on package down the road. It will illustrate how to include arbitrary tasks to a asynchronous group of notification targets. +

+ + + + diff --git a/apps/cic-notify/doc/texinfo/html/Run.html b/apps/cic-notify/doc/texinfo/html/Run.html new file mode 100644 index 00000000..827c0f02 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/html/Run.html @@ -0,0 +1,54 @@ + + + + + + +Run (Grassroots Economics CIC notify 0.2.0) + + + + + + + + + + + + + + + + + +
+

+Next: , Previous: , Up: Top   [Contents]

+
+
+

3 Running the Application

+ +

A convenience task runner is provided in scripts/cic-notify-tasker.py. +

+ + + + diff --git a/apps/cic-notify/doc/texinfo/html/Setup.html b/apps/cic-notify/doc/texinfo/html/Setup.html new file mode 100644 index 00000000..57d7491e --- /dev/null +++ b/apps/cic-notify/doc/texinfo/html/Setup.html @@ -0,0 +1,64 @@ + + + + + + +Setup (Grassroots Economics CIC notify 0.2.0) + + + + + + + + + + + + + + + + + +
+

+Next: , Previous: , Up: Top   [Contents]

+
+
+

2 Setup

+ + +

2.1 Handlers

+ +

Notification tasks in this package are intended to be loaded dynamically, making no assumptions on which handlers that should be connected to the different tasks. The tasks provided are: +

+ + + + + + + diff --git a/apps/cic-notify/doc/texinfo/html/index.html b/apps/cic-notify/doc/texinfo/html/index.html new file mode 100644 index 00000000..53ee37e0 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/html/index.html @@ -0,0 +1,94 @@ + + + + + + +Top (Grassroots Economics CIC notify 0.2.0) + + + + + + + + + + + + + + + +

Grassroots Economics CIC notify 0.2.0

+ + + + +

Table of Contents

+ +
+ + +
+ + +
+

+   [Contents]

+
+
+

Grassroots Economics CIC ETH

+ +

This document describes microservices that broker external notifications from the CIC network +

+ + + + + + +
• Overview   +
• Setup   +
• Running the Application   +
• API   +
+ +
+
+

+   [Contents]

+
+ + + + + diff --git a/apps/cic-notify/doc/texinfo/index.texi b/apps/cic-notify/doc/texinfo/index.texi new file mode 100644 index 00000000..9abd6334 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/index.texi @@ -0,0 +1,28 @@ +\input texinfo +@settitle Grassroots Economics CIC notify 0.2.0 + +@copying +Released 2020 under GPL3 +@end copying + +@titlepage +@title Community Inclusion Currency Network +@author Louis Holbrook + +@end titlepage + +@c +@contents + +@ifnottex +@node Top +@top Grassroots Economics CIC ETH + +This document describes microservices that broker external notifications from the CIC network +@end ifnottex + + +@include overview.texi +@include setup.texi +@include run.texi +@include api.texi diff --git a/apps/cic-notify/doc/texinfo/overview.texi b/apps/cic-notify/doc/texinfo/overview.texi new file mode 100644 index 00000000..7c789c14 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/overview.texi @@ -0,0 +1,22 @@ +@node Overview +@chapter Overview + +This system is in early stages of development. It is intended to be a flexible notification broker where additional notification targets easily can be plugged in. + +The framework is designed to asynchronously execute all tasks belonging to a specific context, based on the name of the task. + +Currently, only handlers for @strong{sms} notifications are implemented. Any task with a @code{notify.sms.} prefix registered in the celery task worker pool will be executed upon the high-level "send sms" task. + +Similary, any other notification category can be implemented. e.g. @code{notify.email.}, @code{notify.telegram.}, @code{notify.mattermost.} etc. + +@section Contents + +The only implementations so far are three @code{sms} notification tasks: + +@itemize +@item log (as in python logger, thus more or less noop) +@item db (postgres) +@item Africas Talking API +@end itemize + +@strong{NOTE}: The Africas Talking API will be removed from the suite, and provided as an add-on package down the road. It will illustrate how to include arbitrary tasks to a asynchronous group of notification targets. diff --git a/apps/cic-notify/doc/texinfo/run.texi b/apps/cic-notify/doc/texinfo/run.texi new file mode 100644 index 00000000..c901289a --- /dev/null +++ b/apps/cic-notify/doc/texinfo/run.texi @@ -0,0 +1,4 @@ +@node Run +@chapter Running the Application + +A convenience task runner is provided in @file{scripts/cic-notify-tasker.py}. diff --git a/apps/cic-notify/doc/texinfo/setup.texi b/apps/cic-notify/doc/texinfo/setup.texi new file mode 100644 index 00000000..d7f8d6f2 --- /dev/null +++ b/apps/cic-notify/doc/texinfo/setup.texi @@ -0,0 +1,17 @@ +@node Setup +@chapter Setup + + +@section Handlers + +Notification tasks in this package are intended to be loaded dynamically, making no assumptions on which handlers that should be connected to the different tasks. The tasks provided are: + +@itemize +@item +@strong{notify.sms.log}: (none) +@item +@strong{notify.sms.db}: Postgresql, along with python packages alembic, SQLAlchemy and psycopg2. +@item +@strong{notify.sms.africastalking}: africastalking python package. +@end itemize + diff --git a/apps/cic-notify/docker/Dockerfile b/apps/cic-notify/docker/Dockerfile new file mode 100644 index 00000000..ec77a552 --- /dev/null +++ b/apps/cic-notify/docker/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.8.6 + +RUN apt-get update && \ + apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps + +WORKDIR /usr/src/cic-notify + +ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433' +ARG root_requirement_file='requirements.txt' +RUN echo "copying root req file ${root_requirement_file}" +COPY $root_requirement_file . +RUN pip install -r $root_requirement_file $pip_extra_index_url_flag + +COPY cic-notify/setup.cfg \ + cic-notify/setup.py \ + ./ +COPY cic-notify/cic_notify/ ./cic_notify/ +COPY cic-notify/requirements.txt \ + cic-notify/test_requirements.txt \ + ./ + +COPY cic-notify/scripts/ scripts/ +RUN pip install $pip_extra_index_url_flag .[africastalking,notifylog] + +COPY cic-notify/tests/ tests/ +COPY cic-notify/docker/db.sh \ + cic-notify/docker/start_tasker.sh \ + /root/ + +#RUN apk add postgresql-client +#RUN apk add bash + +# ini files in config directory defines the configurable parameters for the application +# they can all be overridden by environment variables +# to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) +COPY cic-notify/.config/ /usr/local/etc/cic-notify/ +COPY cic-notify/cic_notify/db/migrations/ /usr/local/share/cic-notify/alembic/ + +WORKDIR /root diff --git a/apps/cic-notify/docker/Dockerfile.service b/apps/cic-notify/docker/Dockerfile.service new file mode 100644 index 00000000..cf2bb38f --- /dev/null +++ b/apps/cic-notify/docker/Dockerfile.service @@ -0,0 +1,13 @@ +FROM grassrootseconomics:cic-notify + +#FROM python:3.8.6-alpine + +#RUN apk update && \ +# apk add gnupg libpq + +#COPY --from=0 /usr/local/ /usr/local/ +#COPY --from=0 /root/ /root/ + +#RUN apk add bash + +WORKDIR /root diff --git a/apps/cic-notify/docker/db.sh b/apps/cic-notify/docker/db.sh new file mode 100644 index 00000000..ccb03f9e --- /dev/null +++ b/apps/cic-notify/docker/db.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +migrate.py -c /usr/local/etc/cic-notify --migrations-dir /usr/local/share/cic-notify/alembic -vv diff --git a/apps/cic-notify/docker/start_tasker.sh b/apps/cic-notify/docker/start_tasker.sh new file mode 100755 index 00000000..b58f0e9e --- /dev/null +++ b/apps/cic-notify/docker/start_tasker.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +. ./db.sh + +/usr/local/bin/cic-notify-tasker -vv $@ diff --git a/apps/cic-notify/requirements.txt b/apps/cic-notify/requirements.txt new file mode 100644 index 00000000..0ab7977c --- /dev/null +++ b/apps/cic-notify/requirements.txt @@ -0,0 +1,5 @@ +celery~=4.4.7 +confini~=0.3.6a1 +alembic~=1.4.2 +redis~=3.5.3 +semver==2.13.0 \ No newline at end of file diff --git a/apps/cic-notify/scripts/migrate.py b/apps/cic-notify/scripts/migrate.py new file mode 100644 index 00000000..9b490189 --- /dev/null +++ b/apps/cic-notify/scripts/migrate.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# standard imports +import argparse +import os +import logging + +# third party imports +import alembic +from alembic.config import Config as AlembicConfig +import confini + +from cic_notify.db import dsn_from_config + +logging.basicConfig(level=logging.WARNING) +logg = logging.getLogger() + +rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +dbdir = os.path.join(rootdir, 'cic_notify', 'db') +migrationsdir = os.path.join(dbdir, 'migrations') + +config_dir = os.path.join('/usr/local/etc/cic-notify') + +argparser = argparse.ArgumentParser() +argparser.add_argument('-c', type=str, default=config_dir, help='config file') +argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration') +argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory') +argparser.add_argument('-v', action='store_true', help='be verbose') +argparser.add_argument('-vv', action='store_true', help='be more verbose') +args = argparser.parse_args() + +if args.vv: + logging.getLogger().setLevel(logging.DEBUG) +elif args.v: + logging.getLogger().setLevel(logging.INFO) + +config = confini.Config(args.c, args.env_prefix) +config.process() +config.censor('PASSWORD', 'DATABASE') +#config.censor('PASSWORD', 'SSL') +logg.debug('config:\n{}'.format(config)) + +migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE')) +if not os.path.isdir(migrations_dir): + logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE'))) + migrations_dir = os.path.join(args.migrations_dir, 'default') + +# connect to database +dsn = dsn_from_config(config) + + +logg.info('using migrations dir {}'.format(migrations_dir)) +logg.info('using db {}'.format(dsn)) +ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini')) +ac.set_main_option('sqlalchemy.url', dsn) +ac.set_main_option('script_location', migrations_dir) + +alembic.command.upgrade(ac, 'head') diff --git a/apps/cic-notify/scripts/send_sms.py b/apps/cic-notify/scripts/send_sms.py new file mode 100644 index 00000000..4a7614c1 --- /dev/null +++ b/apps/cic-notify/scripts/send_sms.py @@ -0,0 +1,12 @@ +# local imports +import cic_notify + +# third-part imports +import celery + + +# TODO: Add configurable backend +celery_app = celery.Celery(broker='redis:///', backend='redis:///') + +api = cic_notify.Api('cic-notify') +print(api.sms('+25412121212', 'foo')) diff --git a/apps/cic-notify/setup.cfg b/apps/cic-notify/setup.cfg new file mode 100644 index 00000000..d421d373 --- /dev/null +++ b/apps/cic-notify/setup.cfg @@ -0,0 +1,47 @@ +[metadata] +name = cic-notify +version= attr: cic_notify.version.__version_string__ +description = CIC notifications service +author = Louis Holbrook +author_email = dev@holbrook.no +url = https://gitlab.com/grassrootseconomics/cic-eth +keywords = + cic + cryptocurrency + ethereum + sms +classifiers = + Programming Language :: Python :: 3 + Operating System :: OS Independent + Development Status :: 3 - Alpha + Environment :: No Input/Output (Daemon) + Intended Audience :: Developers + License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) + Topic :: Internet + Topic :: Blockchain :: EVM +license = GPL3 +licence_files = + LICENSE.txt + +[options] +python_requires = >= 3.6 +packages = + cic_notify + cic_notify.db + cic_notify.db.models + cic_notify.tasks.sms + cic_notify.runnable +scripts = + scripts/migrate.py +[options.extras_require] +africastalking = africastalking==1.2.3 +notifylog = psycopg2==2.8.6 +testing = + pytest==6.0.1 + pytest-celery==0.0.0a1 + pytest-mock==3.3.1 + pysqlite3==0.4.3 + +[options.entry_points] +console_scripts = + cic-notify-tasker = cic_notify.runnable.tasker:main diff --git a/apps/cic-notify/setup.py b/apps/cic-notify/setup.py new file mode 100644 index 00000000..848466f9 --- /dev/null +++ b/apps/cic-notify/setup.py @@ -0,0 +1,30 @@ +# standard imports +from setuptools import setup + +# third-party imports + +# local imports + + +requirements = [] +requirements_file = open('requirements.txt', 'r') +while True: + requirement = requirements_file.readline() + if requirement == '': + break + requirements.append(requirement.rstrip()) +requirements_file.close() + +test_requirements = [] +test_requirements_file = open('test_requirements.txt', 'r') +while True: + test_requirement = test_requirements_file.readline() + if test_requirement == '': + break + test_requirements.append(test_requirement.rstrip()) +test_requirements_file.close() + +setup( + install_requires=requirements, + tests_require=test_requirements, +) diff --git a/apps/cic-notify/test_requirements.txt b/apps/cic-notify/test_requirements.txt new file mode 100644 index 00000000..fda03d35 --- /dev/null +++ b/apps/cic-notify/test_requirements.txt @@ -0,0 +1,5 @@ +pytest~=6.0.1 +pytest-celery~=0.0.0a1 +pytest-mock~=3.3.1 +pysqlite3~=0.4.3 + diff --git a/apps/cic-notify/tests/conftest.py b/apps/cic-notify/tests/conftest.py new file mode 100644 index 00000000..4817ea3e --- /dev/null +++ b/apps/cic-notify/tests/conftest.py @@ -0,0 +1,31 @@ +# standard imports +import sys +import os +import pytest +import logging + +# third party imports +import confini + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) +sys.path.insert(0, root_dir) + +# local imports +from cic_notify.db.models.base import SessionBase +#from transport.notification import AfricastalkingNotification + +# fixtures +from tests.fixtures_config import * +from tests.fixtures_celery import * +from tests.fixtures_database import * + +logg = logging.getLogger() + + +#@pytest.fixture(scope='session') +#def africastalking_notification( +# load_config, +# ): +# return AfricastalkingNotificationTransport(load_config) +# diff --git a/apps/cic-notify/tests/fixtures_celery.py b/apps/cic-notify/tests/fixtures_celery.py new file mode 100644 index 00000000..20cfc444 --- /dev/null +++ b/apps/cic-notify/tests/fixtures_celery.py @@ -0,0 +1,48 @@ +# third-party imports +import pytest +import tempfile +import logging +import shutil + +logg = logging.getLogger(__file__) + + +# celery fixtures +@pytest.fixture(scope='session') +def celery_includes(): + return [ + 'cic_notify.tasks.sms', + ] + + +@pytest.fixture(scope='session') +def celery_config(): + bq = tempfile.mkdtemp() + bp = tempfile.mkdtemp() + rq = tempfile.mkdtemp() + logg.debug('celery broker queue {} processed {}'.format(bq, bp)) + logg.debug('celery backend store {}'.format(rq)) + yield { + 'broker_url': 'filesystem://', + 'broker_transport_options': { + 'data_folder_in': bq, + 'data_folder_out': bq, + 'data_folder_processed': bp, + }, + 'result_backend': 'file://{}'.format(rq), + } + logg.debug('cleaning up celery filesystem backend files {} {} {}'.format(bq, bp, rq)) + shutil.rmtree(bq) + shutil.rmtree(bp) + shutil.rmtree(rq) + + +@pytest.fixture(scope='session') +def celery_worker_parameters(): + return { +# 'queues': ('cic-notify'), + } + +@pytest.fixture(scope='session') +def celery_enable_logging(): + return True diff --git a/apps/cic-notify/tests/fixtures_config.py b/apps/cic-notify/tests/fixtures_config.py new file mode 100644 index 00000000..723fff1c --- /dev/null +++ b/apps/cic-notify/tests/fixtures_config.py @@ -0,0 +1,20 @@ +# standard imports +import os +import logging + +# third-party imports +import pytest +import confini + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.dirname(script_dir) +logg = logging.getLogger(__file__) + + +@pytest.fixture(scope='session') +def load_config(): + config_dir = os.path.join(root_dir, '.config/test') + conf = confini.Config(config_dir, 'CICTEST') + conf.process() + logg.debug('config {}'.format(conf)) + return conf diff --git a/apps/cic-notify/tests/fixtures_database.py b/apps/cic-notify/tests/fixtures_database.py new file mode 100644 index 00000000..eb3aad69 --- /dev/null +++ b/apps/cic-notify/tests/fixtures_database.py @@ -0,0 +1,48 @@ +# standard imports +import os + +# third-party imports +import pytest +import alembic +from alembic.config import Config as AlembicConfig + +# local imports +from cic_notify.db import SessionBase +from cic_notify.db import dsn_from_config + + +@pytest.fixture(scope='session') +def database_engine( + load_config, + ): + dsn = dsn_from_config(load_config) + SessionBase.connect(dsn) + return dsn + + +@pytest.fixture(scope='function') +def init_database( + load_config, + database_engine, + ): + + rootdir = os.path.dirname(os.path.dirname(__file__)) + dbdir = os.path.join(rootdir, 'cic_notify', 'db') + migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE')) + if not os.path.isdir(migrationsdir): + migrationsdir = os.path.join(dbdir, 'migrations', 'default') + + session = SessionBase.create_session() + + ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini')) + ac.set_main_option('sqlalchemy.url', database_engine) + ac.set_main_option('script_location', migrationsdir) + + alembic.command.downgrade(ac, 'base') + alembic.command.upgrade(ac, 'head') + + yield session + session.commit() + session.close() + + diff --git a/apps/cic-notify/tests/test_sms.py b/apps/cic-notify/tests/test_sms.py new file mode 100644 index 00000000..6019ab5f --- /dev/null +++ b/apps/cic-notify/tests/test_sms.py @@ -0,0 +1,34 @@ +# standard imports +import json + +# third party imports +import pytest +import celery + +# local imports +from cic_notify.tasks.sms import db +from cic_notify.tasks.sms import log + +def test_log_notification( + celery_session_worker, + ): + + recipient = '+25412121212' + content = 'bar' + s_log = celery.signature('cic_notify.tasks.sms.log.log') + t = s_log.apply_async(args=[recipient, content]) + + r = t.get() + + +def test_db_notification( + init_database, + celery_session_worker, + ): + + recipient = '+25412121213' + content = 'foo' + s_db = celery.signature('cic_notify.tasks.sms.db.persist_notification') + t = s_db.apply_async(args=[recipient, content]) + + r = t.get() diff --git a/docker-compose.yml b/docker-compose.yml index 8242a0a5..57c6bd73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -404,29 +404,29 @@ services: -# cic-notify-tasker: -# image: grassrootseconomics:cic-notify-service -# environment: -# DATABASE_USER: $DATABASE_USER -# DATABASE_HOST: $DATABASE_HOST -# DATABASE_PORT: $DATABASE_PORT -# DATABASE_PASSWORD: $DATABASE_PASSWORD -# DATABASE_NAME: $DATABASE_NAME_CIC_NOTIFY -# DATABASE_ENGINE: $DATABASE_ENGINE -# DATABASE_DRIVER: $DATABASE_DRIVER -# PGPASSWORD: $DATABASE_PASSWORD -# CELERY_BROKER_URL: $CELERY_BROKER_URL -# CELERY_RESULT_URL: $CELERY_RESULT_URL -# TASKS_AFRICASTALKING: $TASKS_AFRICASTALKING -# TASKS_SMS_DB: $TASKS_SMS_DB -# TASKS_LOG: $TASKS_LOG -# depends_on: -# - postgres -# - redis -# deploy: -# restart_policy: -# condition: on-failure -# command: "/root/start_tasker.sh -q cic-notify" + cic-notify-tasker: + image: grassrootseconomics:cic-notify-service + environment: + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} + DATABASE_NAME: ${DATABASE_NAME_CIC_NOTIFY:-cic_notify} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + PGPASSWORD: ${DATABASE_PASSWORD:-tralala} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis} + CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis} + TASKS_AFRICASTALKING: $TASKS_AFRICASTALKING + TASKS_SMS_DB: $TASKS_SMS_DB + TASKS_LOG: $TASKS_LOG + depends_on: + - postgres + - redis + deploy: + restart_policy: + condition: on-failure + command: "/root/start_tasker.sh -q cic-notify" # cic-meta-server: