Eliminate false 404 from cic-meta in put automerge client

feat: (automation) add semver

See merge request grassrootseconomics/cic-internal-integration!266

.gitlab-ci.yml

@@ -10,6 +10,7 @@ include:
     #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
+  - version
   - build
   - test
   - deploy
@@ -20,9 +21,39 @@ variables:
   DOCKER_BUILDKIT: "1"
   COMPOSE_DOCKER_CLI_BUILD: "1"
   CI_DEBUG_TRACE: "true"
+  SEMVERBOT_VERSION: "0.2.0"
 
-before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+  #before_script:
+  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+
+version:
+  #image: python:3.7-stretch
+  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
+  stage: version
+  script:
+    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
+    - eval $(ssh-agent -s)
+    - ssh-add <(echo "$SSH_PRIVATE_KEY")
+    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
+    - export TAG=$(sbot predict version -m auto)
+    - |
+      if [[ -z $TAG ]]
+      then
+        echo "tag could not be set $@"
+        exit 1
+      fi
+    - echo $TAG > version
+    - git tag -a v$TAG -m "ci tagged"
+    - git push origin v$TAG
+  artifacts:
+    paths:
+      - version
+  rules:
+  - if: $CI_COMMIT_REF_PROTECTED == "true"
+    when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 # runs on protected branches and pushes to repo
 build-push:
@@ -30,12 +61,17 @@ build-push:
   tags:
     - integration
   #script:
-  #  - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
+  #  - TAG=$CI_Cbefore_script:
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
   script:
-    - TAG=latest sh ./scripts/build-push.sh
+    - TAG=latest ./scripts/build-push.sh
+    - TAG=$(cat ./version) ./scripts/build-push.sh
   rules:
   - if: $CI_COMMIT_REF_PROTECTED == "true"
     when: always
+  - if: $CI_COMMIT_REF_NAME == "master"
+    when: always
 
 deploy-dev:
   stage: deploy

.semverbot.toml

@@ -0,0 +1,16 @@
+[git]
+
+[git.config]
+email = "semverbot@grassroots.org"
+name = "semvervot"
+
+[git.tags]
+prefix = "v"
+
+[semver]
+mode = "git-commit"
+
+[semver.detection]
+patch = ["fix", "[fix]", "patch", "[patch]"]
+minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
+major = ["BREAKING CHANGE"]

README.md

@@ -4,6 +4,7 @@
 
 This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
 
+
 ```
 export COMPOSE_DOCKER_CLI_BUILD=1
 export DOCKER_BUILDKIT=1

apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py

@@ -23,7 +23,7 @@ def upgrade():
     op.create_table(
             'lock',
             sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String, nullable=True),
+            sa.Column("address", sa.String(42), nullable=True),
             sa.Column('blockchain', sa.String),
             sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
             sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py

@@ -1,31 +0,0 @@
-"""Add gas cache
-
-Revision ID: c91cafc3e0c1
-Revises: aee12aeb47ec
-Create Date: 2021-10-28 20:45:34.239865
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'c91cafc3e0c1'
-down_revision = 'aee12aeb47ec'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'gas_cache',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column("address", sa.String, nullable=False),
-            sa.Column("tx_hash", sa.String, nullable=True),
-            sa.Column("method", sa.String, nullable=True),
-            sa.Column("value", sa.BIGINT(), nullable=False),
-            )
-
-
-def downgrade():
-    op.drop_table('gas_cache')

apps/cic-eth/cic_eth/db/models/gas_cache.py

@@ -1,27 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from sqlalchemy import Column, String, NUMERIC
-
-# local imports
-from .base import SessionBase
-
-logg = logging.getLogger(__name__)
-
-
-class GasCache(SessionBase):
-    """Provides gas budget cache for token operations
-    """
-    __tablename__ = 'gas_cache'
-
-    address = Column(String())
-    tx_hash = Column(String())
-    method = Column(String())
-    value = Column(NUMERIC())
-
-    def __init__(self, address, method, value, tx_hash):
-        self.address = address
-        self.tx_hash = tx_hash
-        self.method = method
-        self.value = value

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -12,7 +12,7 @@ from cic_eth.error import (
         IntegrityError,
         )
 
-logg = logging.getLogger(__name__)
+logg = logging.getLogger()
 
 
 class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
     __tablename__ = 'nonce'
 
     nonce = Column(Integer)
-    address_hex = Column(String(40))
+    address_hex = Column(String(42))
 
 
     @staticmethod

apps/cic-eth/cic_eth/eth/erc20.py

@@ -10,9 +10,6 @@ from chainlib.eth.tx import (
         TxFormat,
         unpack,
         )
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        )
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
 from hexathon import (
@@ -158,7 +155,7 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
 
     session = self.create_session()
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-    gas_oracle = self.create_gas_oracle(rpc, t['address'], MaxGasOracle.gas)
+    gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:
         (tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -229,11 +226,6 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
 
     session = self.create_session()
     nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
-
-    enc = ABIContractEncoder()
-    enc.method('transferFrom')
-    method = enc.get()
-
     gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
     c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
     try:

apps/cic-eth/cic_eth/eth/gas.py

@@ -41,7 +41,6 @@ from chainqueue.db.models.tx import TxCache
 from chainqueue.db.models.otx import Otx
 
 # local imports
-from cic_eth.db.models.gas_cache import GasCache
 from cic_eth.db.models.role import AccountRole
 from cic_eth.db.models.base import SessionBase
 from cic_eth.error import (
@@ -71,33 +70,12 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
+MAXIMUM_FEE_UNITS = 8000000
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def apply_gas_value_cache(address, method, value, tx_hash):
-    return apply_gas_value_cache_local(address, method, value, tx_hash)
+class MaxGasOracle:
 
-
-def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
-    address = tx_normalize.executable_address(address)
-    tx_hash = tx_normalize.tx_hash(tx_hash)
-    value = int(value)
-
-    session = SessionBase.bind_session(session)
-    q = session.query(GasCache)
-    q = q.filter(GasCache.address==address)
-    q = q.filter(GasCache.method==method)
-    o = q.first()
-
-    if o == None:
-        o = GasCache(address, method, value, tx_hash)
-    elif tx.gas_used > o.value:
-        o.value = value
-        o.tx_hash = strip_0x(tx_hash)
-
-    session.add(o)
-    session.commit()
-
-    SessionBase.release_session(session)
+    def gas(code=None):
+        return MAXIMUM_FEE_UNITS
 
 
 def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):

apps/cic-eth/cic_eth/runnable/daemons/filters/token.py

@@ -1,63 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from eth_erc20 import ERC20
-from chainlib.eth.contract import (
-        ABIContractEncoder,
-        ABIContractType,
-        )
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.address import is_same_address
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.erc20 import ERC20Token
-from eth_token_index import TokenUniqueSymbolIndex
-import celery
-
-# local imports
-from .base import SyncFilter
-
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
-
-
-class TokenFilter(SyncFilter):
-
-    def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
-        self.queue = queue
-        self.chain_spec = chain_spec
-        self.caller_address = call_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        if not tx.payload:
-            return (None, None)
-
-        try:
-            r = ERC20.parse_transfer_request(tx.payload)
-        except RequestMismatchException:
-            return (None, None)
-
-        token_address = tx.inputs[0]
-        token = ERC20Token(self.chain_spec, conn, token_address)
-
-        registry = CICRegistry(self.chain_spec, conn)
-        r = registry.by_name(token.symbol, sender_address=self.caller_address)
-        if is_same_address(r, ZERO_ADDRESS):
-            return None
-
-        enc = ABIContractEncoder()
-        enc.method('transfer')
-        method = enc.get()
-
-        s = celery.signature(
-                'cic_eth.eth.gas.apply_gas_value_cache',
-                [
-                    token_address,
-                    method,
-                    tx.gas_used,
-                    tx.hash,
-                    ],
-                queue=self.queue,
-                )
-        return s.apply_async()

apps/cic-eth/cic_eth/task.py

@@ -10,6 +10,7 @@ from chainlib.chain import ChainSpec
 from chainlib.connection import RPCConnection
 from chainlib.eth.constant import ZERO_ADDRESS
 from chainlib.eth.nonce import RPCNonceOracle
+from chainlib.eth.gas import RPCGasOracle
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 
@@ -29,7 +30,7 @@ class BaseTask(celery.Task):
     call_address = ZERO_ADDRESS
     trusted_addresses = []
     create_nonce_oracle = RPCNonceOracle
-    create_gas_oracle = CacheGasOracle
+    create_gas_oracle = RPCGasOracle
     default_token_address = None
     default_token_symbol = None
     default_token_name = None

apps/cic-eth/tests/filters/test_token_filter.py

@@ -1,96 +0,0 @@
-# external imports
-from eth_erc20 import ERC20
-from chainlib.connection import RPCConnection
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.gas import (
-        Gas,
-        OverrideGasOracle,
-        )
-from chainlib.eth.tx import (
-        TxFormat,
-        receipt,
-        raw,
-        unpack,
-        Tx,
-        )
-from chainlib.eth.block import (
-        Block,
-        block_latest,
-        block_by_number,
-        )
-from chainlib.eth.contract import ABIContractEncoder
-from hexathon import strip_0x
-from eth_token_index import TokenUniqueSymbolIndex
-
-# local imports
-from cic_eth.runnable.daemons.filters.token import TokenFilter
-from cic_eth.db.models.gas_cache import GasCache
-from cic_eth.db.models.base import SessionBase
-
-
-def test_filter_gas(
-        default_chain_spec,
-        init_database,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        agent_roles,
-        token_roles,
-        foo_token,
-        token_registry,
-        register_lookups,
-        celery_worker,
-        cic_registry,
-    ):
-
-    rpc = RPCConnection.connect(default_chain_spec, 'default')
-    nonce_oracle = RPCNonceOracle(token_roles['FOO_TOKEN_OWNER'], eth_rpc)
-    gas_oracle = OverrideGasOracle(price=1000000000, limit=1000000)
-    c = ERC20(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
-    (tx_hash_hex, tx_signed_raw_hex) = c.transfer(foo_token, token_roles['FOO_TOKEN_OWNER'], agent_roles['ALICE'], 100, tx_format=TxFormat.RLP_SIGNED)
-    o = raw(tx_signed_raw_hex)
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    rcpt = eth_rpc.do(o)
-    assert rcpt['status'] == 1
-
-    fltr = TokenFilter(default_chain_spec, queue=None, call_address=agent_roles['ALICE'])
-
-    o = block_latest()
-    r = eth_rpc.do(o)
-    o = block_by_number(r, include_tx=False)
-    r = eth_rpc.do(o)
-    block = Block(r)
-    block.txs = [tx_hash_hex]
-
-    tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
-    tx_src = unpack(tx_signed_raw_bytes, default_chain_spec)
-    tx = Tx(tx_src, block=block)
-    tx.apply_receipt(rcpt)
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert t == None
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
-    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex_register, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], foo_token)
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    r = t.get_leaf()
-    assert t.successful()
-
-    q = init_database.query(GasCache)
-    q = q.filter(GasCache.tx_hash==strip_0x(tx_hash_hex))
-    o = q.first()
-
-    assert o.address == strip_0x(foo_token)
-    assert o.value > 0
-
-    enc = ABIContractEncoder()
-    enc.method('transfer')
-    method = enc.get()
-
-    assert o.method == method

apps/cic-meta/scripts/server/server.ts

@@ -193,6 +193,7 @@ async function processRequest(req, res) {
 						res.end();
 						return;
 					}
+					content = '';
 					break;
 
 				case 'get:automerge:client':
@@ -251,7 +252,7 @@ async function processRequest(req, res) {
 		}
 
 		if (content === undefined) {
-			console.error('empty content', data);
+			console.error('empty content', mod, digest, data);
 			res.writeHead(404, {"Content-Type": "text/plain"});
 			res.end();
 			return;

apps/contract-migration/4_init_custodial.sh

@@ -18,18 +18,6 @@ fi
 must_address "$CIC_REGISTRY_ADDRESS" "registry"
 must_eth_rpc
 
-
-
-default_token=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw DefaultToken`
-h=`erc20-transfer -u -e $default_token -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -y $WALLET_KEY_FILE -s 10`
-r=`eth-receipt 
-
-set +e
-set +a
-
-exit 0
-
-
 # get required addresses from registries
 token_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw TokenRegistry`
 accounts_index_address=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS $DEV_DEBUG_FLAG --raw AccountRegistry`

scripts/build.sh

@@ -6,4 +6,7 @@ set -e
 TAG=${TAG?Variable not set} \
 docker-compose \
 -f docker-compose.yml \
-build
+build \
+--no-cache \
+--parallel \
+--progress plain