fix sbot path

Update aux deps

See merge request grassrootseconomics/cic-internal-integration!264

.gitlab-ci.yml

@@ -10,6 +10,7 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
   - deploy
@@ -20,9 +21,39 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
 
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
+  script:
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -30,12 +61,17 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - TAG=latest sh ./scripts/build-push.sh
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 deploy-dev:
   stage: deploy

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

README.md

@@ -4,6 +4,7 @@
 
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
+
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,4 +1,5 @@
 celery==4.4.7
-erc20-demurrage-token~=0.0.3a1
-cic-eth-registry>=0.6.1a2,<0.7.0
-cic-eth[services]~=0.12.4a8
+erc20-demurrage-token~=0.0.5a3
+cic-eth-registry~=0.6.1a5
+chainlib~=0.0.9rc3
+cic_eth~=0.12.4a9

apps/cic-eth/cic_eth/eth/erc20.py

@@ -524,8 +524,11 @@ def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_cal
                     ],
                 queue=queue,
                 )
-        s.link(success_callback)
-        s.on_error(error_callback)
+
+        if success_callback != None:
+            s.link(success_callback)
+        if error_callback != None:
+            s.on_error(error_callback)
         s.apply_async()
 
     return tokens

apps/cic-ussd/cic_ussd/metadata/__init__.py

@@ -3,7 +3,6 @@
 # external imports
 
 # local imports
-from .base import Metadata
 from .custom import CustomMetadata
 from .person import PersonMetadata
 from .phone import PhonePointerMetadata

apps/cic-ussd/cic_ussd/metadata/base.py

@@ -1,99 +1,30 @@
 # standard imports
-import json
 import logging
-import os
-from typing import Dict, Union
 
-# third-part imports
-from cic_types.models.person import generate_metadata_pointer, Person
+# external imports
+from cic_types.condiments import MetadataPointer
+from cic_types.ext.metadata import MetadataRequestsHandler
+from cic_types.processor import generate_metadata_pointer
 
 # local imports
 from cic_ussd.cache import cache_data, get_cached_data
-from cic_ussd.http.requests import error_handler, make_request
-from cic_ussd.metadata.signer import Signer
 
 logg = logging.getLogger(__file__)
 
 
-class Metadata:
-    """
-    :cvar base_url: The base url or the metadata server.
-    :type base_url: str
-    """
+class UssdMetadataHandler(MetadataRequestsHandler):
+    def __init__(self, cic_type: MetadataPointer, identifier: bytes):
+        super().__init__(cic_type, identifier)
 
-    base_url = None
-
-
-class MetadataRequestsHandler(Metadata):
-
-    def __init__(self, cic_type: str, identifier: bytes, engine: str = 'pgp'):
-        """"""
-        self.cic_type = cic_type
-        self.engine = engine
-        self.headers = {
-            'X-CIC-AUTOMERGE': 'server',
-            'Content-Type': 'application/json'
-        }
-        self.identifier = identifier
-        self.metadata_pointer = generate_metadata_pointer(
-            identifier=self.identifier,
-            cic_type=self.cic_type
-        )
-        if self.base_url:
-            self.url = os.path.join(self.base_url, self.metadata_pointer)
-
-    def create(self, data: Union[Dict, str]):
-        """"""
-        data = json.dumps(data).encode('utf-8')
-        result = make_request(method='POST', url=self.url, data=data, headers=self.headers)
-
-        error_handler(result=result)
-        metadata = result.json()
-        return self.edit(data=metadata)
-
-    def edit(self, data: Union[Dict, str]):
-        """"""
-        cic_meta_signer = Signer()
-        signature = cic_meta_signer.sign_digest(data=data)
-        algorithm = cic_meta_signer.get_operational_key().get('algo')
-        formatted_data = {
-            'm': json.dumps(data),
-            's': {
-                'engine': self.engine,
-                'algo': algorithm,
-                'data': signature,
-                'digest': data.get('digest'),
-            }
-        }
-        formatted_data = json.dumps(formatted_data)
-        result = make_request(method='PUT', url=self.url, data=formatted_data, headers=self.headers)
-        logg.info(f'signed metadata submission status: {result.status_code}.')
-        error_handler(result=result)
-        try:
-            decoded_identifier = self.identifier.decode("utf-8")
-        except UnicodeDecodeError:
-            decoded_identifier = self.identifier.hex()
-        logg.info(f'identifier: {decoded_identifier}. metadata pointer: {self.metadata_pointer} set to: {data}.')
-        return result
-
-    def query(self):
-        """"""
-        result = make_request(method='GET', url=self.url)
-        error_handler(result=result)
-        result_data = result.json()
-        if not isinstance(result_data, dict):
-            raise ValueError(f'Invalid result data object: {result_data}.')
-        if result.status_code == 200:
-            if self.cic_type == ':cic.person':
-                person = Person()
-                person_data = person.deserialize(person_data=result_data)
-                serialized_person_data = person_data.serialize()
-                data = json.dumps(serialized_person_data)
-            else:
-                data = json.dumps(result_data)
-            cache_data(key=self.metadata_pointer, data=data)
-            logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
-        return result_data
+    def cache_metadata(self, data: str):
+        """
+        :param data:
+        :type data:
+        :return:
+        :rtype:
+        """
+        cache_data(self.metadata_pointer, data)
+        logg.debug(f'caching: {data} with key: {self.metadata_pointer}')
 
     def get_cached_metadata(self):
         """"""

apps/cic-ussd/cic_ussd/metadata/custom.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class CustomMetadata(MetadataRequestsHandler):
+class CustomMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.custom', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.CUSTOM, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/person.py

@@ -1,12 +1,13 @@
 # standard imports
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PersonMetadata(MetadataRequestsHandler):
+class PersonMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.person', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PERSON, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -2,12 +2,13 @@
 import logging
 
 # external imports
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PhonePointerMetadata(MetadataRequestsHandler):
+class PhonePointerMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.phone', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PHONE, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/preferences.py

@@ -1,13 +1,13 @@
 # standard imports
 
 # external imports
-import celery
+from cic_types.condiments import MetadataPointer
 
 # local imports
-from .base import MetadataRequestsHandler
+from .base import UssdMetadataHandler
 
 
-class PreferencesMetadata(MetadataRequestsHandler):
+class PreferencesMetadata(UssdMetadataHandler):
 
     def __init__(self, identifier: bytes):
-        super().__init__(cic_type=':cic.preferences', identifier=identifier)
+        super().__init__(cic_type=MetadataPointer.PREFERENCES, identifier=identifier)

apps/cic-ussd/cic_ussd/metadata/signer.py

@@ -1,60 +0,0 @@
-# standard imports
-import json
-import logging
-from typing import Optional
-from urllib.request import Request, urlopen
-
-# third-party imports
-import gnupg
-
-# local imports
-
-logg = logging.getLogger()
-
-
-class Signer:
-    """
-    :cvar gpg_path:
-    :type gpg_path:
-    :cvar gpg_passphrase:
-    :type gpg_passphrase:
-    :cvar key_file_path:
-    :type key_file_path:
-
-    """
-    gpg_path: str = None
-    gpg_passphrase: str = None
-    key_file_path: str = None
-
-    def __init__(self):
-        self.gpg = gnupg.GPG(gnupghome=self.gpg_path)
-
-        with open(self.key_file_path, 'r') as key_file:
-            self.key_data = key_file.read()
-
-    def get_operational_key(self):
-        """
-        :return:
-        :rtype:
-        """
-        # import key data into keyring
-        self.gpg.import_keys(key_data=self.key_data)
-        gpg_keys = self.gpg.list_keys()
-        key_algorithm = gpg_keys[0].get('algo')
-        key_id = gpg_keys[0].get("keyid")
-        logg.debug(f'using signing key: {key_id}, algorithm: {key_algorithm}')
-        return gpg_keys[0]
-
-    def sign_digest(self, data: dict):
-        """
-        :param data:
-        :type data:
-        :return:
-        :rtype:
-        """
-        digest = data['digest']
-        key_id = self.get_operational_key().get('keyid')
-        signature = self.gpg.sign(digest, passphrase=self.gpg_passphrase, keyid=key_id)
-        return str(signature)
-
-

apps/cic-ussd/cic_ussd/tasks/metadata.py

@@ -1,15 +1,17 @@
 # standard imports
+import json
 import logging
 
 # third-party imports
 import celery
+from cic_types.models.person import Person
 
 # local imports
 from cic_ussd.metadata import CustomMetadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
 from cic_ussd.tasks.base import CriticalMetadataTask
 
 celery_app = celery.current_app
-logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger(__file__)
 
 
 @celery_app.task
@@ -22,7 +24,13 @@ def query_person_metadata(blockchain_address: str):
     """
     identifier = bytes.fromhex(blockchain_address)
     person_metadata_client = PersonMetadata(identifier=identifier)
-    person_metadata_client.query()
+    response = person_metadata_client.query()
+    data = response.json()
+    person = Person()
+    person_data = person.deserialize(person_data=data)
+    serialized_person_data = person_data.serialize()
+    data = json.dumps(serialized_person_data)
+    person_metadata_client.cache_metadata(data=data)
 
 
 @celery_app.task
@@ -76,6 +84,9 @@ def query_preferences_metadata(blockchain_address: str):
     :type blockchain_address: str | Ox-hex
     """
     identifier = bytes.fromhex(blockchain_address)
-    logg.debug(f'Retrieving preferences metadata for address: {blockchain_address}.')
-    person_metadata_client = PreferencesMetadata(identifier=identifier)
-    return person_metadata_client.query()
+    logg.debug(f'retrieving preferences metadata for address: {blockchain_address}.')
+    preferences_metadata_client = PreferencesMetadata(identifier=identifier)
+    response = preferences_metadata_client.query()
+    data = json.dumps(response.json())
+    preferences_metadata_client.cache_metadata(data)
+    return data

apps/cic-ussd/requirements.txt

@@ -4,10 +4,10 @@ billiard==3.6.4.0
 bcrypt==3.2.0
 celery==4.4.7
 cffi==1.14.6
-cic-eth[services]~=0.12.4a7
+cic-eth[services]~=0.12.4a11
 cic-notify~=0.4.0a10
-cic-types~=0.1.0a15
-confini>=0.4.1a1,<0.5.0
+cic-types~=0.2.0a3
+confini>=0.3.6rc4,<0.5.0
 phonenumbers==8.12.12
 psycopg2==2.8.6
 python-i18n[YAML]==0.3.9

apps/cic-ussd/tests/cic_ussd/metadata/test_base.py

@@ -5,24 +5,25 @@ import os
 # external imports
 import requests_mock
 from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
-from cic_ussd.metadata.base import MetadataRequestsHandler
+from cic_ussd.metadata.base import UssdMetadataHandler
 
 
 # external imports
 
 
-def test_metadata_requests_handler(activated_account,
-                                   init_cache,
-                                   load_config,
-                                   person_metadata,
-                                   setup_metadata_request_handler,
-                                   setup_metadata_signer):
+def test_ussd_metadata_handler(activated_account,
+                               init_cache,
+                               load_config,
+                               person_metadata,
+                               setup_metadata_request_handler,
+                               setup_metadata_signer):
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    cic_type = ':cic.person'
-    metadata_client = MetadataRequestsHandler(cic_type, identifier)
+    cic_type = MetadataPointer.PERSON
+    metadata_client = UssdMetadataHandler(cic_type, identifier)
     assert metadata_client.cic_type == cic_type
     assert metadata_client.engine == 'pgp'
     assert metadata_client.identifier == identifier
@@ -38,7 +39,5 @@ def test_metadata_requests_handler(activated_account,
         assert result.status_code == 200
         person_metadata.pop('digest')
         request_mocker.register_uri('GET', metadata_client.url, status_code=200, reason='OK', json=person_metadata)
-        result = metadata_client.query()
+        result = metadata_client.query().json()
         assert result == person_metadata
-        cached_metadata = metadata_client.get_cached_metadata()
-        assert json.loads(cached_metadata) == person_metadata

apps/cic-ussd/tests/cic_ussd/metadata/test_custom.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import CustomMetadata
 
 
 def test_custom_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.custom'
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    cic_type = MetadataPointer.CUSTOM
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     custom_metadata_client = CustomMetadata(identifier)
     assert custom_metadata_client.cic_type == cic_type
     assert custom_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_person.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import PersonMetadata
 
 
 def test_person_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.person'
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    cic_type = MetadataPointer.PERSON
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     person_metadata_client = PersonMetadata(identifier)
     assert person_metadata_client.cic_type == cic_type
     assert person_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_phone.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -12,8 +12,8 @@ from cic_ussd.metadata import PhonePointerMetadata
 
 
 def test_phone_pointer_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.phone'
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    cic_type = MetadataPointer.PHONE
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     phone_pointer_metadata = PhonePointerMetadata(identifier)
     assert phone_pointer_metadata.cic_type == cic_type
     assert phone_pointer_metadata.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_preferences.py

@@ -1,7 +1,7 @@
 # standard imports
 import os
 # external imports
-from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
@@ -11,8 +11,8 @@ from cic_ussd.metadata import PreferencesMetadata
 
 
 def test_preferences_metadata(activated_account, load_config, setup_metadata_request_handler, setup_metadata_signer):
-    cic_type = ':cic.preferences'
-    identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
+    cic_type = MetadataPointer.PREFERENCES
+    identifier = bytes.fromhex(activated_account.blockchain_address)
     preferences_metadata_client = PreferencesMetadata(identifier)
     assert preferences_metadata_client.cic_type == cic_type
     assert preferences_metadata_client.engine == 'pgp'

apps/cic-ussd/tests/cic_ussd/metadata/test_signer.py

@@ -1,17 +0,0 @@
-# standard imports
-import shutil
-
-# third-party imports
-
-# local imports
-from cic_ussd.metadata.signer import Signer
-
-
-def test_client(load_config, setup_metadata_signer, person_metadata):
-    signer = Signer()
-    gpg = signer.gpg
-    assert signer.key_data is not None
-    gpg.import_keys(key_data=signer.key_data)
-    gpg_keys = gpg.list_keys()
-    assert signer.get_operational_key() == gpg_keys[0]
-    shutil.rmtree(Signer.gpg_path)

apps/cic-ussd/tests/fixtures/metadata.py

@@ -6,33 +6,19 @@ import tempfile
 # external imports
 import pytest
 from chainlib.hash import strip_0x
+from cic_types.condiments import MetadataPointer
 from cic_types.processor import generate_metadata_pointer
 
 # local imports
-from cic_ussd.metadata import Metadata, PersonMetadata, PhonePointerMetadata, PreferencesMetadata
-from cic_ussd.metadata.signer import Signer
+from cic_ussd.metadata import PersonMetadata, PhonePointerMetadata, PreferencesMetadata
 
 logg = logging.getLogger(__name__)
 
 
-@pytest.fixture(scope='function')
-def setup_metadata_signer(load_config):
-    temp_dir = tempfile.mkdtemp(dir='/tmp')
-    logg.debug(f'Created temp dir: {temp_dir}')
-    Signer.gpg_path = temp_dir
-    Signer.gpg_passphrase = load_config.get('PGP_PASSPHRASE')
-    Signer.key_file_path = os.path.join(load_config.get('PGP_KEYS_PATH'), load_config.get('PGP_PRIVATE_KEYS'))
-
-
-@pytest.fixture(scope='function')
-def setup_metadata_request_handler(load_config):
-    Metadata.base_url = load_config.get('CIC_META_URL')
-
-
 @pytest.fixture(scope='function')
 def account_phone_pointer(activated_account):
     identifier = bytes.fromhex(strip_0x(activated_account.blockchain_address))
-    return generate_metadata_pointer(identifier, ':cic.phone')
+    return generate_metadata_pointer(identifier, MetadataPointer.PERSON)
 
 
 @pytest.fixture(scope='function')

scripts/build.sh

@@ -6,4 +6,7 @@ set -e
 TAG=${TAG?Variable not set} \
 docker-compose \
 -f docker-compose.yml \
-build
+build \
+--no-cache \
+--parallel \
+--progress plain