Adds cic-eth-tasker to dependencies on bootstrap step.

bug: False 404 in cic-meta logging

See merge request grassrootseconomics/cic-internal-integration!302

.gitlab-ci.yml

@@ -10,6 +10,7 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
   - deploy
@@ -20,9 +21,39 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
 
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
+  script:
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -30,12 +61,17 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - TAG=latest sh ./scripts/build-push.sh
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 deploy-dev:
   stage: deploy

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

README.md

@@ -4,6 +4,7 @@
 
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
+
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1

apps/cic-meta/scripts/server/server.ts

@@ -193,6 +193,7 @@ async function processRequest(req, res) {
 						res.end();
 						return;
 					}
+					content = '';
 					break;
 
 				case 'get:automerge:client':
@@ -251,7 +252,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty content', data);
+			console.error('empty content', mod, digest, data);
 			res.writeHead(404, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/contract-migration/3_deploy_token.sh

@@ -154,6 +154,9 @@ fi
 
 check_wait 3
 
+>&2 echo -e "\033[;96mWriting token metadata and proofs\033[;39m"
+python scripts/proofs.py --token-symbol $TOKEN_SYMBOL -e $TOKEN_ADDRESS --address-declarator $DEV_ADDRESS_DECLARATOR --signer-address $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+
 >&2 echo -e "\033[;96mWriting env_reset file\033[;39m"
 confini-dump --schema-dir ./config > ${DEV_DATA_DIR}/env_reset
 

apps/contract-migration/config/proofs/config.ini

@@ -0,0 +1,8 @@
+[pgp]
+export_dir = pgp/keys/
+keys_path = /tmp/cic/pgp/
+private_keys = privatekeys_meta.asc
+passphrase =
+
+[meta]
+url =

apps/contract-migration/docker/Dockerfile

@@ -11,9 +11,10 @@ RUN cat /etc/apt/sources.list.d/ethereum.list
 RUN apt-key adv --keyserver keyserver.ubuntu.com  --recv-keys 2A518C819BE37D2C2031944D1C52189C923F6CA9
 
 RUN mkdir -vp /usr/local/etc/cic
+# create secrets directory
+RUN mkdir -vp pgp/keys
 
-COPY config_template/ /usr/local/etc/cic/
-COPY requirements.txt . 
+COPY requirements.txt .
 
 #RUN apt-get install libffi-dev
 

apps/contract-migration/requirements.txt

@@ -10,3 +10,4 @@ sarafu-faucet>=0.0.7a2,<0.1.0
 confini>=0.4.2rc3,<1.0.0
 eth-token-index>=0.2.4a1,<=0.3.0
 okota>=0.2.4a15,<0.3.0
+cic-types~=0.2.1a2

apps/contract-migration/scripts/proofs.py

@@ -0,0 +1,114 @@
+# standard imports
+import hashlib
+import json
+import logging
+import os
+from typing import Union
+
+# external imports
+import cic_eth.cli
+from chainlib.chain import ChainSpec
+from chainlib.eth.connection import EthHTTPConnection
+from chainlib.eth.gas import OverrideGasOracle
+from chainlib.eth.nonce import RPCNonceOracle
+from cic_types.condiments import MetadataPointer
+from cic_types.ext.metadata import MetadataRequestsHandler, Signer
+from eth_address_declarator import Declarator
+from eth_address_declarator.declarator import AddressDeclarator
+from funga.eth.signer import EIP155Signer
+from funga.eth.keystore.dict import DictKeystore
+from hexathon import add_0x, strip_0x
+
+# local imports
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.join(script_dir, '..')
+base_config_dir = os.path.join(root_dir, 'config', 'proofs')
+token_data_dir = os.path.join(root_dir, 'token_data')
+
+arg_flags = cic_eth.cli.argflag_std_base
+arg_parser = cic_eth.cli.ArgumentParser(arg_flags)
+arg_parser.add_argument('--token-symbol', type=str, help='Token symbol whose metadata is being processed.')
+arg_parser.add_argument('--address-declarator', type=str, help='Address to address declarator contract')
+arg_parser.add_argument('--signer-address', type=str, help='Wallet keyfile address')
+arg_parser.add_argument('-e', type=str, help='Token address.')
+args = arg_parser.parse_args()
+config = cic_eth.cli.Config.from_args(args, arg_flags, 0, base_config_dir=base_config_dir)
+
+token_metadata = os.path.join(token_data_dir, 'meta.json')
+token_proof = os.path.join(token_data_dir, 'proof.json')
+
+
+def hash_proof(data: bytes) -> hex:
+    hash_object = hashlib.sha256()
+    hash_object.update(data)
+    return hash_object.digest().hex()
+
+
+def init_meta():
+    MetadataRequestsHandler.base_url = config.get('META_URL')
+    Signer.gpg_path = config.get('PGP_EXPORT_DIR')
+    Signer.key_file_path = f"{config.get('PGP_KEYS_PATH')}{config.get('PGP_PRIVATE_KEYS')}"
+    Signer.gpg_passphrase = config.get('PGP_PASSPHRASE')
+
+
+def wrapper(chain_spec: ChainSpec, rpc: EthHTTPConnection) -> Declarator:
+    gas_oracle = OverrideGasOracle(limit=AddressDeclarator.gas(), conn=rpc)
+    nonce_oracle = RPCNonceOracle(address=add_0x(args.signer_address.lower()), conn=rpc)
+    keystore = DictKeystore()
+    keystore.import_keystore_file(keystore_file=config.get('WALLET_KEY_FILE'))
+    signer = EIP155Signer(keystore)
+    return Declarator(chain_spec, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle, signer=signer)
+
+
+def write_to_declarator(contract_address: hex, contract_wrapper: Declarator, proof: any, rpc: EthHTTPConnection, signer_address: hex, token_address: hex):
+    operation = contract_wrapper.add_declaration(contract_address, signer_address, token_address, proof)
+    results = rpc.do(operation[1])
+    rpc.wait(results)
+
+
+def write_metadata(writer: MetadataRequestsHandler, data: Union[dict, str]):
+    writer.create(data)
+
+
+if __name__ == '__main__':
+    init_meta()
+    chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
+
+    token_address_bytes = bytes.fromhex(strip_0x(args.e))
+    token_symbol_bytes = args.token_symbol.encode('utf-8')
+    token_meta_file = open(token_metadata, 'r')
+    token_proof_file = open(token_proof, 'r')
+    token_meta_data = json.load(token_meta_file)
+    token_proof_data = json.load(token_proof_file)
+    token_meta_file.close()
+    token_proof_file.close()
+
+    token_meta_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_META, identifier=token_address_bytes)
+    write_metadata(token_meta_writer, token_meta_data)
+
+    token_meta_symbol_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_META_SYMBOL, identifier=token_symbol_bytes)
+    write_metadata(token_meta_symbol_writer, token_meta_data)
+
+    token_proof_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_PROOF, identifier=token_address_bytes)
+    write_metadata(token_proof_writer, token_proof_data)
+
+    token_proof_symbol_writer = MetadataRequestsHandler(cic_type=MetadataPointer.TOKEN_PROOF_SYMBOL, identifier=token_symbol_bytes)
+    write_metadata(token_proof_symbol_writer, token_proof_data)
+
+    rpc = EthHTTPConnection(url=config.get('RPC_PROVIDER'), chain_spec=chain_spec)
+    contract_wrapper = wrapper(chain_spec, rpc)
+
+    hashed_token_proof = hash_proof(data=json.dumps(token_proof_data).encode('utf-8'))
+    identifier = bytes.fromhex(hashed_token_proof)
+    token_immutable_proof_writer = MetadataRequestsHandler(cic_type=MetadataPointer.NONE, identifier=identifier)
+    write_metadata(token_immutable_proof_writer, token_proof_data)
+    write_to_declarator(contract_address=args.address_declarator,
+                        contract_wrapper=contract_wrapper,
+                        proof=hashed_token_proof,
+                        rpc=rpc,
+                        signer_address=args.signer_address,
+                        token_address=args.e)

apps/contract-migration/token_data/meta.json

@@ -0,0 +1,6 @@
+{
+	"contact": {"phone":  "+254757628885", "email":  "info@grassrootseconomics.org"},
+	"country_code": "KE",
+	"location": "Kilifi",
+	"name": "GRASSROOTS ECONOMICS"
+}

apps/contract-migration/token_data/proof.json

@@ -0,0 +1,7 @@
+{
+	"description": "Community support",
+	"issuer": "Grassroots Economics",
+	"namespace": "ge",
+	"proofs": [],
+	"version": 0
+}

docker-compose.yml

@@ -78,14 +78,18 @@ services:
       REDIS_PORT_CALLBACK: ${REDIS_PORT_CALLBACK:-6379}
       FAUCET_AMOUNT: ${FAUCET_AMOUNT:-0}
       WALLET_KEY_FILE: ${WALLET_KEY_FILE:-/root/keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c}
+      PGP_PASSPHRASE: merman
+      META_URL: http://meta:8000
     command: ["./run_job.sh"]
     depends_on:
       - evm
       - postgres
       - redis
-        #- cic-eth-tasker
+      - cic-meta-server
+      - cic-eth-tasker
     volumes:
       - contract-config:/tmp/cic/config
+      - ./apps/contract-migration/testdata/pgp/:/tmp/cic/pgp
 
 
 

scripts/build.sh

@@ -6,4 +6,7 @@ set -e
 TAG=${TAG?Variable not set} \
 docker-compose \
 -f docker-compose.yml \
-build
+build \
+--no-cache \
+--parallel \
+--progress plain