cic-cache migrated

.gitignore

@@ -1,16 +1,2 @@
 service-configs/*
 !service-configs/.gitkeep
-**/node_modules/
-__pycache__
-*.pyc
-*.o
-gmon.out
-*.egg-info
-dist/
-build/
-**/*sqlite
-**/.nyc_output
-**/coverage
-**/.venv
-.idea
-**/.vim

.gitlab-ci.yml

@@ -5,8 +5,6 @@ include:
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
-  - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build

apps/cic-cache/.config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/.config/cic.ini

@@ -1,2 +0,0 @@
-[cic]
-registry_address = 

apps/cic-cache/.config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=

apps/cic-cache/.config/eth.ini

@@ -1,6 +0,0 @@
-[eth]
-provider = ws://localhost:8545
-#ttp_provider = http://localhost:8545
-#provider = http://localhost:8545
-gas_provider_address = 
-#chain_id = 

apps/cic-cache/.config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/.config/test/cic.ini

@@ -1,2 +0,0 @@
-[cic]
-registry_address = 

apps/cic-cache/.config/test/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic-cache-test
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite
-DEBUG=

apps/cic-cache/.config/test/eth.ini

@@ -1,5 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-#chain_id = 

apps/cic-cache/.coveragerc

@@ -1,6 +0,0 @@
-[report]
-omit = 
-	.venv/*
-	scripts/*
-	cic_cache/db/migrations/*
-	cic_cache/version.py

apps/cic-cache/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-cache/.envrc_example

@@ -1,7 +0,0 @@
-set -a
-CICTEST_DATABASE_ENGINE=postgresql
-CICTEST_DATABASE_DRIVER=psycopg2
-CICTEST_DATABASE_HOST=localhost
-CICTEST_DATABASE_PORT=5432
-CICTEST_DATABASE_NAME=cic-eth-test
-set +a

apps/cic-cache/.gitignore

@@ -1,8 +0,0 @@
-.envrc
-.envrc_dev
-.venv
-__pycache__
-*.pyc
-_build
-doc/**/*.png
-doc/**/html

apps/cic-cache/.gitlab-ci.yml

@@ -1,52 +0,0 @@
-.cic_cache_variables:
-    variables:
-        APP_NAME: cic-cache
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
-
-build-mr-cic-cache:
-    extends:
-        - .py_build_merge_request
-        - .cic_cache_variables
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-
-test-mr-cic-cache:
-    stage: test
-    extends:
-        - .cic_cache_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-cache"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/$APP_NAME/**/*
-      when: always
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-cache/**/*
-      when: always
-        
-

apps/cic-cache/CHANGELOG

@@ -1,13 +0,0 @@
-- 0.1.2
-	* Revert to alembic migrations
-- 0.1.1
-	* Add missing modules to setup 
-- 0.1.0
-	* Remove old APIs
-	* Add bloom filter output APIs for all txs and per-account txs
-- 0.0.2
-	* UWSGI server endpoint example
-	* OpenAPI spec
-	* stored procedures, test fixture for database schema
-- 0.0.1
-	* Add json translators of transaction_list and balances stored procedure queries

apps/cic-cache/cic_cache/__init__.py

@@ -1 +0,0 @@
-from .cache import BloomCache

apps/cic-cache/cic_cache/api.py

@@ -1,73 +0,0 @@
-"""API for cic-cache celery tasks
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-
-"""
-# standard imports
-import logging
-
-# third-party imports 
-import celery
-
-
-app = celery.current_app
-
-logg = logging.getLogger(__name__)
-
-
-class Api:
-    """Creates task chains to perform well-known CIC operations.
-
-    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
-
-    :param callback_param: Static value to pass to callback
-    :type callback_param: str
-    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
-    :type callback_task: string
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, queue='cic-cache', callback_param=None, callback_task='cic_cache.callbacks.noop.noop', callback_queue=None):
-        self.callback_param = callback_param
-        self.callback_task = callback_task
-        self.queue = queue
-        logg.info('api using queue {}'.format(self.queue))
-        self.callback_success = None
-        self.callback_error = None
-        if callback_queue == None:
-            callback_queue=self.queue
-
-        if callback_param != None:
-            self.callback_success = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        0,
-                        ],
-                    queue=callback_queue,
-                    )
-            self.callback_error = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        1,
-                        ],
-                    queue=callback_queue,
-                    )       
-
-    def list(self, offset, limit, address=None):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter',
-        [
-            0,
-            100,
-            address,
-            ],
-            queue=None
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t

apps/cic-cache/cic_cache/cache.py

@@ -1,138 +0,0 @@
-# standard imports
-import logging
-import datetime
-
-# external imports
-import moolb
-
-# local imports
-from cic_cache.db.list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        list_transactions_mined_with_data,
-        )
-
-logg = logging.getLogger()
-
-
-DEFAULT_FILTER_SIZE = 8192 * 8
-
-class Cache:
-
-    def __init__(self, session):
-        self.session = session
-
-
-class BloomCache(Cache):
-
-    @staticmethod
-    def __get_filter_size(n):
-        n = DEFAULT_FILTER_SIZE
-        logg.warning('filter size hardcoded to {}'.format(n))
-        return n
-
-
-    def load_transactions(self, offset, limit):
-        """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
-
-        Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
-
-        For example, if the block number is 13 and the transaction index is 42, the input are:
-
-        block filter:       0x0d000000
-        block+tx filter:    0x0d0000002a0000000
-
-        :param offset: Offset in data set to return transactions from
-        :type offset: int
-        :param limit: Max number of transactions to retrieve
-        :type limit: int
-        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
-        :rtype: tuple
-        """
-        rows = list_transactions_mined(self.session, offset, limit) 
-
-        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        highest_block = -1
-        lowest_block = -1
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r[0]
-            lowest_block = r[0]
-            block = r[0].to_bytes(4, byteorder='big')
-            tx = r[1].to_bytes(4, byteorder='big')
-            f_block.add(block)
-            f_blocktx.add(block + tx)
-            logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
-        return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
-
-
-    def load_transactions_account(self, address, offset, limit):
-        """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
-
-        :param address: Address to retrieve transactions for.
-        :type address: str, 0x-hex
-        :param offset: Offset in data set to return transactions from
-        :type offset: int
-        :param limit: Max number of transactions to retrieve
-        :type limit: int
-        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
-        :rtype: tuple
-        """
-        rows = list_transactions_account_mined(self.session, address, offset, limit) 
-
-        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        highest_block = -1;
-        lowest_block = -1;
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r[0]
-            lowest_block = r[0]
-            block = r[0].to_bytes(4, byteorder='big')
-            tx = r[1].to_bytes(4, byteorder='big')
-            f_block.add(block)
-            f_blocktx.add(block + tx)
-            logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
-        return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
-
-
-class DataCache(Cache):
-
-    def load_transactions_with_data(self, offset, end):
-        rows = list_transactions_mined_with_data(self.session, offset, end) 
-        tx_cache = []
-        highest_block = -1;
-        lowest_block = -1;
-        date_is_str = None # stick this in startup
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r['block_number']
-            lowest_block = r['block_number']
-            tx_type = 'unknown'
-
-            if r['value'] != None:
-                tx_type = '{}.{}'.format(r['domain'], r['value'])
-
-            if date_is_str == None:
-                date_is_str = type(r['date_block']).__name__ == 'str'
-
-            o = {
-                'block_number': r['block_number'],
-                'tx_hash': r['tx_hash'],
-                'date_block': r['date_block'],
-                'sender': r['sender'],
-                'recipient': r['recipient'],
-                'from_value': int(r['from_value']),
-                'to_value': int(r['to_value']),
-                'source_token': r['source_token'],
-                'destination_token': r['destination_token'],
-                'success': r['success'],
-                'tx_type': tx_type,
-            }
-
-            if date_is_str:
-                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
-
-            tx_cache.append(o)
-        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/db/__init__.py

@@ -1,40 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from .list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        add_transaction,
-        tag_transaction,
-        add_tag,
-    )
-
-
-logg = logging.getLogger()
-
-
-def dsn_from_config(config):
-    scheme = config.get('DATABASE_ENGINE')
-    if config.get('DATABASE_DRIVER') != None:
-        scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
-
-    dsn = ''
-    if config.get('DATABASE_ENGINE') == 'sqlite':
-        dsn = '{}:///{}'.format(
-                scheme,
-                config.get('DATABASE_NAME'),    
-            )
-
-    else:
-        dsn = '{}://{}:{}@{}:{}/{}'.format(
-                scheme,
-                config.get('DATABASE_USER'),
-                config.get('DATABASE_PASSWORD'),
-                config.get('DATABASE_HOST'),
-                config.get('DATABASE_PORT'),
-                config.get('DATABASE_NAME'),    
-            )
-    logg.debug('parsed dsn from config: {}'.format(dsn))
-    return dsn
-

apps/cic-cache/cic_cache/db/list.py

@@ -1,199 +0,0 @@
-# standard imports
-import logging
-import datetime
-
-# external imports
-from cic_cache.db.models.base import SessionBase
-from sqlalchemy import text
-
-logg = logging.getLogger()
-
-
-def list_transactions_mined(
-        session,
-        offset,
-        limit,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data(
-        session,
-        offset,
-        end,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined(
-        session,
-        address,
-        offset,
-        limit,
-        ):
-    """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
-
-    :param address: Address to retrieve transactions for.
-    :type address: str, 0x-hex
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
-    r = session.execute(s)
-    return r
-
-
-def add_transaction(
-        session,
-        tx_hash,
-        block_number,
-        tx_index,
-        sender,
-        receiver,
-        source_token,
-        destination_token,
-        from_value,
-        to_value,
-        success,
-        timestamp,
-        ):
-    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param block_number: Block number
-    :type block_number: int
-    :param tx_index: Transaction index in block
-    :type tx_index: int
-    :param sender: Ethereum address of effective sender
-    :type sender: str, 0x-hex
-    :param receiver: Ethereum address of effective recipient
-    :type receiver: str, 0x-hex
-    :param source_token: Ethereum address of token used by sender
-    :type source_token: str, 0x-hex
-    :param destination_token: Ethereum address of token received by recipient
-    :type destination_token: str, 0x-hex
-    :param from_value: Source token value spent in transaction
-    :type from_value: int
-    :param to_value: Destination token value received in transaction
-    :type to_value: int
-    :param success: True if code execution on network was successful
-    :type success: bool
-    :param date_block: Block timestamp
-    :type date_block: datetime
-    """
-    date_block = datetime.datetime.fromtimestamp(timestamp)
-    s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
-            tx_hash,
-            block_number,
-            tx_index,
-            sender,
-            receiver,
-            source_token,
-            destination_token,
-            from_value,
-            to_value,
-            success,
-            date_block,
-            )
-    session.execute(s)
-
-
-
-def tag_transaction(
-        session,
-        tx_hash,
-        name,
-        domain=None,
-        ):
-    """Tag a single transaction with a single tag.
-
-    Tag must already exist in storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises ValueError: Unknown tag or transaction hash
-
-    """
-
-    s = text("SELECT id from tx where tx_hash = :a")
-    r = session.execute(s, {'a': tx_hash}).fetchall()
-    tx_id = r[0].values()[0]
-
-    if tx_id == None:
-        raise ValueError('unknown tx hash {}'.format(tx_hash))
-
-    #s = text("SELECT id from tag where value = :a and domain = :b")
-    if domain == None:
-        s = text("SELECT id from tag where value = :a")
-    else:
-        s = text("SELECT id from tag where value = :a and domain = :b")
-    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
-    tag_id = r[0].values()[0]
-
-    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
-
-    if tag_id == None:
-        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
-
-    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
-    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
-
-
-def add_tag(
-        session,
-        name,
-        domain=None,
-        ):
-    """Add a single tag to storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises sqlalchemy.exc.IntegrityError: Tag already exists
-    """
-
-    s = None
-    if domain == None: 
-        s = text("INSERT INTO tag (value) VALUES (:b)")
-    else:
-        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
-    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/README

@@ -1 +0,0 @@
-Generic single-database configuration.

apps/cic-cache/cic_cache/db/migrations/default/versions/63b629f14a85_.py

@@ -1,52 +0,0 @@
-"""Base tables
-
-Revision ID: 63b629f14a85
-Revises: 
-Create Date: 2020-12-04 08:16:00.412189
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '63b629f14a85'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tx',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('date_registered', sa.DateTime, nullable=False, server_default=sa.func.current_timestamp()),
-            sa.Column('block_number', sa.Integer, nullable=False),
-            sa.Column('tx_index', sa.Integer, nullable=False),
-            sa.Column('tx_hash', sa.String(66), nullable=False),
-            sa.Column('sender', sa.String(42), nullable=False),
-            sa.Column('recipient', sa.String(42), nullable=False),
-            sa.Column('source_token', sa.String(42), nullable=False),
-            sa.Column('destination_token', sa.String(42), nullable=False),
-            sa.Column('success', sa.Boolean, nullable=False),
-            sa.Column('from_value', sa.NUMERIC(), nullable=False),
-            sa.Column('to_value', sa.NUMERIC(), nullable=False),
-            sa.Column('date_block', sa.DateTime, nullable=False),
-            )
-    op.create_table(
-            'tx_sync',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tx', sa.String(66), nullable=False),
-            )
-
-    op.execute("INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');")
-
-    op.create_index('sender_token_idx', 'tx', ['sender', 'source_token'])
-    op.create_index('recipient_token_idx', 'tx', ['recipient', 'destination_token'])
-            
-
-def downgrade():
-    op.drop_index('recipient_token_idx')
-    op.drop_index('sender_token_idx')
-    op.drop_table('tx_sync')
-    op.drop_table('tx')

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -1,28 +0,0 @@
-"""Add chain syncer
-
-Revision ID: 6604de4203e2
-Revises: 63b629f14a85
-Create Date: 2021-04-01 08:10:29.156243
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from chainsyncer.db.migrations.sqlalchemy import (
-        chainsyncer_upgrade,
-        chainsyncer_downgrade,
-        )
-
-
-# revision identifiers, used by Alembic.
-revision = '6604de4203e2'
-down_revision = '63b629f14a85'
-branch_labels = None
-depends_on = None
-
-def upgrade():
-    chainsyncer_upgrade(0, 0, 1)
-
-
-def downgrade():
-    chainsyncer_downgrade(0, 0, 1)
-

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -1,38 +0,0 @@
-"""Transaction tags
-
-Revision ID: aaf2bdce7d6e
-Revises: 6604de4203e2
-Create Date: 2021-05-01 09:20:20.775082
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'aaf2bdce7d6e'
-down_revision = '6604de4203e2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tag',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('domain', sa.String(), nullable=True),
-            sa.Column('value', sa.String(), nullable=False),
-            )
-    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
-
-    op.create_table(
-            'tag_tx_link',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
-            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
-                )
-
-def downgrade():
-    op.drop_table('tag_tx_link')
-    op.drop_index('idx_tag_domain_value')
-    op.drop_table('tag')

apps/cic-cache/cic_cache/db/models/base.py

@@ -1,103 +0,0 @@
-# stanard imports
-import logging
-
-# third-party imports
-from sqlalchemy import Column, Integer
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-logg = logging.getLogger()
-
-Model = declarative_base(name='Model')
-
-
-class SessionBase(Model):
-    """The base object for all SQLAlchemy enabled models. All other models must extend this.
-    """
-    __abstract__ = True
- 
-    id = Column(Integer, primary_key=True)
-
-    engine = None
-    """Database connection engine of the running aplication"""
-    sessionmaker = None
-    """Factory object responsible for creating sessions from the  connection pool"""
-    transactional = True
-    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
-    poolable = True
-    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
-    procedural = True
-    """Whether the database backend supports stored procedures"""
-    localsessions = {}
-    """Contains dictionary of sessions initiated by db model components"""
-
-
-    @staticmethod
-    def create_session():
-        """Creates a new database session.
-        """
-        return SessionBase.sessionmaker()
-
-
-    @staticmethod
-    def _set_engine(engine):
-        """Sets the database engine static property
-        """
-        SessionBase.engine = engine
-        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
-
-
-    @staticmethod
-    def connect(dsn, debug=False):
-        """Create new database connection engine and connect to database backend.
-
-        :param dsn: DSN string defining connection.
-        :type dsn: str
-        """
-        e = None
-        if SessionBase.poolable:
-            e = create_engine(
-                    dsn,
-                    max_overflow=50,
-                    pool_pre_ping=True,
-                    pool_size=20,
-                    pool_recycle=10,
-                    echo=debug,
-                )
-        else:
-            e = create_engine(
-                    dsn,
-                    echo=debug,
-                    )
-
-        SessionBase._set_engine(e)
-
-
-    @staticmethod
-    def disconnect():
-        """Disconnect from database and free resources.
-        """
-        SessionBase.engine.dispose()
-        SessionBase.engine = None
-
-
-    @staticmethod
-    def bind_session(session=None):
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
-            localsession_key = str(id(localsession))
-            logg.debug('creating new session {}'.format(localsession_key))
-            SessionBase.localsessions[localsession_key] = localsession
-        return localsession
-
-
-    @staticmethod
-    def release_session(session=None):
-        session_key = str(id(session))
-        if SessionBase.localsessions.get(session_key) != None:
-            logg.debug('destroying session {}'.format(session_key))
-            session.commit()
-            session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1,2 +0,0 @@
-from .erc20 import *
-from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,27 +0,0 @@
-class TagSyncFilter:
-    """Holds tag name and domain for an implementing filter.
-
-    :param name: Tag value 
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    """
-
-    def __init__(self, name, domain=None):
-        self.tag_name = name
-        self.tag_domain = domain
-
-
-    def tag(self):
-        """Return tag value/domain.
-
-        :rtype: Tuple
-        :returns: tag value/domain.
-        """
-        return (self.tag_name, self.tag_domain)
-
-
-    def __str__(self):
-        if self.tag_domain == None:
-            return self.tag_name
-        return '{}.{}'.format(self.tag_domain, self.tag_name)

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -1,83 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.eth.address import (
-        to_checksum_address,
-        )
-from chainlib.eth.error import RequestMismatchException
-from chainlib.status import Status
-from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import (
-        NotAContractError,
-        ContractMismatchError,
-        )
-from eth_erc20 import ERC20
-
-# local imports
-from .base import TagSyncFilter
-from cic_cache import db as cic_cache_db
-
-logg = logging.getLogger().getChild(__name__)
-
-
-class ERC20TransferFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec):
-        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
-        self.chain_spec = chain_spec
-
-
-    # TODO: Verify token in declarator / token index
-    def filter(self, conn, block, tx, db_session=None):
-        logg.debug('filter {} {}'.format(block, tx))
-        token = None
-        try:
-            token = ERC20Token(self.chain_spec, conn, tx.inputs[0])
-        except NotAContractError:
-            logg.debug('not a contract {}'.format(tx.inputs[0]))
-            return False
-        except ContractMismatchError:
-            logg.debug('not an erc20 token  {}'.format(tx.inputs[0]))
-            return False
-
-        transfer_data = None
-        try:
-            transfer_data = ERC20.parse_transfer_request(tx.payload)
-        except RequestMismatchException:
-            logg.debug('erc20 match but not a transfer, skipping')
-            return False
-        except ValueError:
-            logg.debug('erc20 match but bogus data, skipping')
-            return False
-
-        token_sender = tx.outputs[0]
-        token_recipient = transfer_data[0]
-        token_value = transfer_data[1]
-
-        logg.debug('matched erc20 token transfer {} ({}) to {} value {}'.format(token.name, token.address, transfer_data[0], transfer_data[1]))
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token.address,
-                token.address,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -1,73 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from erc20_faucet import Faucet
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.status import Status
-from hexathon import strip_0x
-
-# local imports
-import cic_cache.db as cic_cache_db
-from .base import TagSyncFilter
-
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
-
-
-class FaucetFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
-        super(FaucetFilter, self).__init__('give_to', domain='faucet')
-        self.chain_spec = chain_spec
-        self.sender_address = sender_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        try:
-            data = strip_0x(tx.payload)
-        except ValueError:
-            return False
-        logg.debug('data {}'.format(data))
-        if Faucet.method_for(data[:8]) == None:
-            return False
-
-        token_sender = tx.inputs[0]
-        token_recipient = data[64+8-40:]
-        logg.debug('token recipient {}'.format(token_recipient))
- 
-        f = Faucet(self.chain_spec)
-        o = f.token(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token = f.parse_token(r)
-
-        f = Faucet(self.chain_spec)
-        o = f.token_amount(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token_value = f.parse_token_amount(r)
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token,
-                token,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -1,114 +0,0 @@
-# standard imports
-import logging
-import json
-import re
-import base64
-
-# external imports
-from hexathon import add_0x
-
-# local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-    )
-
-logg = logging.getLogger(__name__)
-#logg = logging.getLogger()
-
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
-
-DEFAULT_LIMIT = 100
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    address = r[1]
-    if r[2] == None:
-        address = add_0x(address)
-    offset = 0
-    if r.lastindex > 2:
-        offset = r[4]
-    limit = DEFAULT_LIMIT
-    if r.lastindex > 4:
-        limit = r[6]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_data(session, env):
-    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-        return None
-
-    offset = r[1]
-    end = r[2]
-    if int(r[2]) < int(r[1]):
-        raise ValueError('cart before the horse, dude')
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,83 +0,0 @@
-# standard imports
-import os
-import logging
-import argparse
-import base64
-
-# external imports
-import confini
-
-# local imports
-from cic_cache.db import dsn_from_config
-from cic_cache.db.models.base import SessionBase
-from cic_cache.runnable.daemons.query import (
-        process_transactions_account_bloom,
-        process_transactions_all_bloom,
-        process_transactions_all_data,
-        )
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
-
-
-# uwsgi application
-def application(env, start_response):
-
-    headers = []
-    content = b''
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_transactions_all_data,
-            process_transactions_all_bloom,
-            process_transactions_account_bloom,
-            ]:
-        r = None
-        try:
-            r = handler(session, env)
-        except ValueError as e:
-            start_response('400 {}'.format(str(e)))
-            return []
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -1,98 +0,0 @@
-# standard imports
-import logging
-import os
-import sys
-import argparse
-
-# third-party imports
-import celery
-import confini
-
-# local imports
-from cic_cache.db import dsn_from_config
-from cic_cache.db.models.base import SessionBase
-from cic_cache.tasks.tx import *
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-
-# connect to database
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-# verify database connection with minimal sanity query
-#session = SessionBase.create_session()
-#session.execute('select version_num from alembic_version')
-#session.close()
-
-# set up celery
-current_app = celery.Celery(__name__)
-
-broker = config.get('CELERY_BROKER_URL')
-if broker[:4] == 'file':
-    bq = tempfile.mkdtemp()
-    bp = tempfile.mkdtemp()
-    current_app.conf.update({
-            'broker_url': broker,
-            'broker_transport_options': {
-                'data_folder_in': bq,
-                'data_folder_out': bq,
-                'data_folder_processed': bp,
-            },
-            },
-            )
-    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
-else:
-    current_app.conf.update({
-        'broker_url': broker,
-        })
-
-result = config.get('CELERY_RESULT_URL')
-if result[:4] == 'file':
-    rq = tempfile.mkdtemp()
-    current_app.conf.update({
-        'result_backend': 'file://{}'.format(rq),
-        })
-    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
-else:
-    current_app.conf.update({
-        'result_backend': result,
-        })
-
-
-def main():
-    argv = ['worker']
-    if args.vv:
-        argv.append('--loglevel=DEBUG')
-    elif args.v:
-        argv.append('--loglevel=INFO')
-    argv.append('-Q')
-    argv.append(args.q)
-    argv.append('-n')
-    argv.append(args.q)
-
-    current_app.worker_main(argv)
-
-
-if __name__ == '__main__':
-    main()

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -1,152 +0,0 @@
-# standard imports
-import os
-import sys
-import logging
-import time
-import argparse
-import sys
-import re
-
-# external imports
-import confini
-import celery
-import sqlalchemy
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.error import UnknownContractError
-from chainlib.chain import ChainSpec
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.connection import RPCConnection
-from chainlib.eth.block import (
-        block_latest,
-        )
-from hexathon import (
-        strip_0x,
-        )
-from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
-from chainsyncer.db.models.base import SessionBase
-
-# local imports
-from cic_cache.db import (
-        dsn_from_config,
-        add_tag,
-        )
-from cic_cache.runnable.daemons.filters import (
-        ERC20TransferFilter,
-        FaucetFilter,
-        )
-
-script_dir = os.path.realpath(os.path.dirname(__file__))
-
-def add_block_args(argparser):
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
-
-
-logg = cic_base.log.create()
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
-dsn = dsn_from_config(config)
-
-SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
-
-
-def register_filter_tags(filters, session):
-    for f in filters:
-        tag = f.tag()
-        try:
-            add_tag(session, tag[0], domain=tag[1])
-            session.commit()
-            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-        except sqlalchemy.exc.IntegrityError:
-            session.rollback()
-            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-
-
-def main():
-    # Connect to blockchain with chainlib
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    o = block_latest()
-    r = rpc.do(o)
-    block_offset = int(strip_0x(r), 16) + 1
-
-    logg.debug('current block height {}'.format(block_offset))
-
-    syncers = []
-
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
-    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
-
-    if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
-        initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
-            initial_block_start = block_offset
-            initial_block_offset += 1
-        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
-        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
-    else:
-        for syncer_backend in syncer_backends:
-            logg.info('resuming sync session {}'.format(syncer_backend))
-
-    for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
-
-    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
-
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-
-    erc20_transfer_filter = ERC20TransferFilter(chain_spec)
-    faucet_filter = FaucetFilter(chain_spec)
-
-    filters = [
-        erc20_transfer_filter,
-        faucet_filter,
-            ]
-
-    session = SessionBase.create_session()
-    register_filter_tags(filters, session)
-    session.close()
-
-    i = 0
-    for syncer in syncers:
-        logg.debug('running syncer index {}'.format(i))
-        for f in filters:
-            syncer.add_filter(f)
-        r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
-        sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
-
-        i += 1
-
-
-if __name__ == '__main__':
-    main()

apps/cic-cache/cic_cache/tasks/tx.py

@@ -1,38 +0,0 @@
-# third-party imports
-import celery
-
-# local imports
-from cic_cache.cache import BloomCache
-from cic_cache.db.models.base import SessionBase
-
-celery_app = celery.current_app
-
-
-@celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, encoding='hex'):
-    queue = self.request.delivery_info.get('routing_key')
-
-    session = SessionBase.create_session()
-
-    c = BloomCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-    else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    session.close()
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': bloom_filter_block.hex(), 
-        'blocktx_filter': bloom_filter_tx.hex(),
-        'filter_rounds': 3,
-            }
-
-    return o
-
-
-

apps/cic-cache/cic_cache/version.py

@@ -1,18 +0,0 @@
-import os
-import semver
-
-version = (
-        0,
-        2,
-        0,
-        'alpha.2',
-        )
-
-version_object = semver.VersionInfo(
-        major=version[0],
-        minor=version[1],
-        patch=version[2],
-        prerelease=version[3],
-        )
-
-version_string = str(version_object)

apps/cic-cache/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec =
-trust_address = 

apps/cic-cache/config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 
-dir = /usr/local/share/bancor

apps/cic-cache/config/docker/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379

apps/cic-cache/config/docker/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-chain_spec =
-registry_address =  
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 1
-history_start = 0

apps/cic-cache/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 5
-history_start = 0

apps/cic-cache/config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/test/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = 
-trust_address =

apps/cic-cache/config/test/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic-cache-test
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite
-DEBUG=1

apps/cic-cache/config/test/eth.ini

@@ -1,5 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-#chain_id = 

apps/cic-cache/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[syncer]
-loop_interval = 1

apps/cic-cache/db/psycopg2/db.sql

@@ -1,22 +0,0 @@
-CREATE TABLE tx (
-	id SERIAL PRIMARY KEY,
-	date_registered TIMESTAMP NOT NULL default CURRENT_TIMESTAMP,
-	block_number INTEGER NOT NULL,
-	tx_index INTEGER NOT NULL,
-	tx_hash VARCHAR(66) NOT NULL,
-	sender VARCHAR(42) NOT NULL, 
-	recipient VARCHAR(42) NOT NULL, 
-	source_token VARCHAR(42) NOT NULL,
-	destination_token VARCHAR(42) NOT NULL,
-	from_value BIGINT NOT NULL,
-	to_value BIGINT NOT NULL,
-	success BOOLEAN NOT NULL, 
-	date_block TIMESTAMP NOT NULL
-);
-
-CREATE TABLE tx_sync (
-	id SERIAL PRIMARY KEY,
-	tx VARCHAR(66) NOT NULL
-);
-
-INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');

apps/cic-cache/db/pysqlite/db.sql

@@ -1,23 +0,0 @@
-CREATE TABLE tx (
-	id SERIAL PRIMARY KEY,
-	date_registered DATETIME NOT NULL default CURRENT_DATE, 
-	block_number INTEGER NOT NULL,
-	tx_index INTEGER NOT NULL,
-	tx_hash VARCHAR(66) NOT NULL,
-	sender VARCHAR(42) NOT NULL, 
-	recipient VARCHAR(42) NOT NULL, 
-	source_token VARCHAR(42) NOT NULL,
-	destination_token VARCHAR(42) NOT NULL,
-	from_value INTEGER NOT NULL,
-	to_value INTEGER NOT NULL,
-	success BOOLEAN NOT NULL, 
-	date_block DATETIME NOT NULL,
-	CHECK (success IN (0, 1))
-);
-
-CREATE TABLE tx_sync (
-	id SERIAL PRIMARY_KEY,
-	tx VARCHAR(66) NOT NULL
-);
-
-INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');

apps/cic-cache/doc/openapi/server.yml

@@ -1,102 +0,0 @@
-openapi: "3.0.3"
-info:
-  title: Grassroots Economics CIC Cache
-  description: Cache of processed transaction data from Ethereum blockchain and worker queues
-  termsOfService: bzz://grassrootseconomics.eth/terms
-  contact:
-    name: Grassroots Economics
-    url: https://www.grassrootseconomics.org
-    email: will@grassecon.org
-  license:
-    name: GPLv3
-  version: 0.1.0
-
-paths:
-  /tx/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-
-    parameters:
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
-    - name: limit
-      in: path
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions by account
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
-    - name: limit
-      in: path
-      schema:
-        type: integer
-        format: int32
-
-components:
-  schemas:
-    BlocksBloom:
-      type: object
-      properties:
-        low:
-          type: int
-          format: int32
-          description: The lowest block number included in the filter
-        block_filter:
-          type: string
-          format: byte
-          description: Block number filter
-        blocktx_filter:
-          type: string
-          format: byte
-          description: Block and tx index filter
-        alg:
-          type: string
-          description: Hashing algorithm (currently only using sha256)
-        filter_rounds:
-          type: int
-          format: int32
-          description: Number of hash rounds used to create the filter

apps/cic-cache/docker/Dockerfile

@@ -1,38 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/Dockerfile_ci

@@ -1,37 +0,0 @@
-# syntax = docker/dockerfile:1.2
-FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
- 
-# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY requirements.txt .
-#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-#RUN pip install $pip_extra_index_url_flag .
-#RUN pip install .[server]
-
-ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
-RUN pip install --index-url https://pypi.org/simple  \
-	  --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
-    -r requirements.txt
-
-COPY . . 
-
-RUN python setup.py install
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/db.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
->&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
-set +e

apps/cic-cache/docker/start_tracker.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. ./db.sh
-
-if [ $? -ne "0" ]; then
-	>&2 echo db migrate fail
-	exit 1
-fi
-
-/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/examples/bloom_client/parse.js

@@ -1,40 +0,0 @@
-let xmlhttprequest = require('xhr2');
-let moolb = require('moolb');
-
-let xhr = new xmlhttprequest();
-xhr.responseType = 'json';
-xhr.open('GET', 'http://localhost:5555/tx/0/100');
-xhr.addEventListener('load', (e) => {
-
-	d = xhr.response;
-
-	b_one = Buffer.from(d.block_filter, 'base64');
-	b_two = Buffer.from(d.blocktx_filter, 'base64');
-
-	for (let i = 0; i < 8192; i++) {
-		if (b_two[i] > 0) {
-			console.debug('value on', i, b_two[i]);
-		}
-	}
-	console.log(b_one, b_two);
-
-	let f_block = moolb.fromBytes(b_one, d.filter_rounds);
-	let f_blocktx = moolb.fromBytes(b_two, d.filter_rounds);
-	let a = new ArrayBuffer(8);
-	let w = new DataView(a);
-	for (let i = 410000; i < 430000; i++) {
-		w.setInt32(0, i);
-		let r = new Uint8Array(a.slice(0, 4));
-		if (f_block.check(r)) {
-			for (let j = 0; j < 200; j++) {
-				w = new DataView(a);
-				w.setInt32(4, j);
-				r = new Uint8Array(a);
-				if (f_blocktx.check(r)) {
-					console.log('true', i, j);
-				}
-			}
-		}
-	}
-});
-let r = xhr.send();

apps/cic-cache/requirements.txt

@@ -1,13 +0,0 @@
-cic-base~=0.2.0a4
-alembic==1.4.2
-confini>=0.3.6rc3,<0.5.0
-uwsgi==2.0.19.1
-moolb~=0.1.0
-cic-eth-registry~=0.5.6a2
-SQLAlchemy==1.3.20
-semver==2.13.0
-psycopg2==2.8.6
-celery==4.4.7
-redis==3.5.3
-chainsyncer[sql]~=0.0.3a5
-erc20-faucet~=0.2.2a2

apps/cic-cache/scripts/migrate.py

@@ -1,65 +0,0 @@
-#!/usr/bin/python
-import os
-import argparse
-import logging
-import re
-
-import alembic
-from alembic.config import Config as AlembicConfig
-import confini
-
-from cic_cache.db import dsn_from_config
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-# BUG: the dbdir doesn't work after script install
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
-if not os.path.isdir(migrations_dir):
-    logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
-    migrations_dir = os.path.join(args.migrations_dir, 'default')
-
-# connect to database
-dsn = dsn_from_config(config)
-
-
-logg.info('using migrations dir {}'.format(migrations_dir))
-logg.info('using db {}'.format(dsn))
-ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
-ac.set_main_option('sqlalchemy.url', dsn)
-ac.set_main_option('script_location', migrations_dir)
-
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
-alembic.command.upgrade(ac, 'head')

apps/cic-cache/setup.cfg

@@ -1,41 +0,0 @@
-[metadata]
-name = cic-cache
-description = CIC Cache API and server
-author = Louis Holbrook
-author_email = dev@holbrook.no
-url = https://gitlab.com/grassrootseconomics/cic-eth
-keywords =
-	cic
-	cryptocurrency
-	ethereum
-classifiers =
-	Programming Language :: Python :: 3
-	Operating System :: OS Independent
-	Development Status :: 3 - Alpha
-	Environment :: No Input/Output (Daemon)
-	Intended Audience :: Developers
-	License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
-	Topic :: Internet
-#	Topic :: Blockchain :: EVM
-license = GPL3
-licence_files =
-	LICENSE.txt
-
-[options]
-python_requires = >= 3.6
-packages = 
-	cic_cache
-	cic_cache.tasks
-	cic_cache.db
-	cic_cache.db.models
-	cic_cache.runnable
-	cic_cache.runnable.daemons
-	cic_cache.runnable.daemons.filters
-scripts =
-	./scripts/migrate.py
-
-[options.entry_points]
-console_scripts =
-	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
-	cic-cache-serverd = cic_cache.runnable.daemons.server:main
-	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main

apps/cic-cache/setup.py

@@ -1,60 +0,0 @@
-from setuptools import setup
-
-import configparser
-import os
-import time
-
-from cic_cache.version import (
-        version_object,
-        version_string
-        )
-
-class PleaseCommitFirstError(Exception):
-    pass
-
-def git_hash():
-    import subprocess
-    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-    if len(git_diff.stdout) > 0:
-        raise PleaseCommitFirstError()
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-version_string = str(version_object)
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-            time_string_pair[0],
-            int(time_string_pair[1]),
-            )
-print('final version string will be {}'.format(version_string))
-
-requirements = []
-f = open('requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    requirements.append(l.rstrip())
-f.close()
-
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
-setup(
-    version=version_string,
-    install_requires=requirements,
-    tests_require=test_requirements,
-        )

apps/cic-cache/test_requirements.txt

@@ -1,10 +0,0 @@
-pytest==6.0.1
-pytest-cov==2.10.1
-pytest-mock==3.3.1
-pysqlite3==0.4.3
-sqlparse==0.4.1
-pytest-celery==0.0.0a1
-eth_tester==0.5.0b3
-py-evm==0.3.0a20
-cic_base[full]==0.1.3a3+build.984b5cff
-sarafu-faucet~=0.0.4a1

apps/cic-cache/tests/conftest.py

@@ -1,110 +0,0 @@
-# standard imports
-import os
-import sys
-import datetime
-
-# external imports
-import pytest
-import moolb
-
-# local imports
-from cic_cache import db
-from cic_cache import BloomCache
-from cic_cache.cache import DEFAULT_FILTER_SIZE
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.dirname(script_dir)
-sys.path.insert(0, root_dir)
-
-# fixtures
-from tests.fixtures_config import *
-from tests.fixtures_database import *
-from tests.fixtures_celery import *
-
-
-@pytest.fixture(scope='session')
-def balances_dict_fields():
-    return {
-            'out_pending': 0,
-            'out_synced': 1,
-            'out_confirmed': 2,
-            'in_pending': 3,
-            'in_synced': 4,
-            'in_confirmed': 5,
-    }
-
-
-@pytest.fixture(scope='function')
-def txs(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        ):
-
-    session = init_database
-
-    tx_number = 13 
-    tx_hash_first = '0x' + os.urandom(32).hex()
-    val = 15000
-    nonce = 1
-    dt = datetime.datetime.utcnow()
-    db.add_transaction(
-        session,
-        tx_hash_first,
-        list_defaults['block'],
-        tx_number,
-        list_actors['alice'],
-        list_actors['bob'],
-        list_tokens['foo'],
-        list_tokens['foo'],
-        1024,
-        2048,
-        True,
-        dt.timestamp(),
-            )
-
-
-    tx_number = 42
-    tx_hash_second = '0x' + os.urandom(32).hex()
-    tx_signed_second = '0x' + os.urandom(128).hex()
-    nonce = 1
-    dt -= datetime.timedelta(hours=1)
-    db.add_transaction(
-        session,
-        tx_hash_second,
-        list_defaults['block']-1,
-        tx_number,
-        list_actors['diane'],
-        list_actors['alice'],
-        list_tokens['foo'],
-        list_tokens['foo'],
-        1024,
-        2048,
-        False,
-        dt.timestamp(),
-        )
-    
-    session.commit()
-
-    return [
-            tx_hash_first,
-            tx_hash_second,
-            ]
-
-
-@pytest.fixture(scope='function')
-def tag_txs(
-        init_database,
-        txs,
-        ):
-
-    db.add_tag(init_database, 'taag', domain='test')
-    init_database.commit()
-
-    db.tag_transaction(init_database, txs[1], 'taag', domain='test')
-
-
-@pytest.fixture(scope='session')
-def zero_filter():
-    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/conftest.py

@@ -1,3 +0,0 @@
-from chainlib.eth.pytest import *
-from cic_eth_registry.pytest.fixtures_tokens import *
-

apps/cic-cache/tests/filters/test_erc20.py

@@ -1,173 +0,0 @@
-# standard imports
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-from sqlalchemy import text
-from chainlib.eth.tx import Tx
-from chainlib.eth.block import Block
-from chainlib.chain import ChainSpec
-from chainlib.eth.error import RequestMismatchException
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-from cic_cache.runnable.daemons.filters.base import TagSyncFilter
-
-logg = logging.getLogger()
-
-
-def test_base_filter_str(
-        init_database,
-        ):
-        f = TagSyncFilter('foo')
-        assert 'foo' == str(f)
-        f = TagSyncFilter('foo', domain='bar')
-        assert 'bar.foo' == str(f)
-
-
-
-def test_erc20_filter(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash
-
-
-def test_erc20_filter_nocontract(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': os.urandom(20).hex(),
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-
-@pytest.mark.parametrize(
-        'contract_method,contract_input,expected_exception',
-        [
-            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
-            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
-            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
-            ],
-        )
-def test_erc20_filter_bogus(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        contract_method,
-        contract_input,
-        expected_exception,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = contract_method
-    data += contract_input 
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/filters/test_faucet.py

@@ -1,71 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.chain import ChainSpec
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_by_hash,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        unpack,
-        transaction,
-        Tx,
-        )
-from hexathon import strip_0x
-from erc20_faucet.faucet import SingleShotFaucet
-from sqlalchemy import text
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
-
-logg = logging.getLogger()
-
-
-def test_filter_faucet(
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        faucet_noregistry,
-        init_database,
-        list_defaults,
-        contract_roles,
-        agent_roles,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
-    c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
-    r = eth_rpc.do(o)
-
-    tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
-
-    o = receipt(r)
-    r = eth_rpc.do(o)
-    rcpt = Tx.src_normalize(r)
-
-    assert r['status'] == 1
-
-    o = block_by_hash(r['block_hash'])
-    r = eth_rpc.do(o)
-    block_object = Block(r)
-
-    tx = Tx(tx_src, block_object)
-    tx.apply_receipt(rcpt)
-
-    r = fltr.filter(eth_rpc, block_object, tx, init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash

apps/cic-cache/tests/fixtures_database.py

@@ -1,102 +0,0 @@
-# standard imports
-import os
-import logging
-import re
-
-# external imports
-import pytest
-import sqlparse
-import alembic
-from alembic.config import Config as AlembicConfig
-
-# local imports
-from cic_cache.db.models.base import SessionBase
-from cic_cache.db import dsn_from_config
-from cic_cache.db import add_tag
-
-logg = logging.getLogger(__file__)
-
-
-@pytest.fixture(scope='function')
-def database_engine(
-    load_config,
-        ):
-    if load_config.get('DATABASE_ENGINE') == 'sqlite':
-        SessionBase.transactional = False
-        SessionBase.poolable = False
-        try:
-            os.unlink(load_config.get('DATABASE_NAME'))
-        except FileNotFoundError:
-            pass
-    dsn = dsn_from_config(load_config)
-    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
-    return dsn
-
-
-@pytest.fixture(scope='function')
-def init_database(
-        load_config,
-        database_engine,
-        ):
-
-    rootdir = os.path.dirname(os.path.dirname(__file__))
-    dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-    migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
-    if not os.path.isdir(migrationsdir):
-        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
-    logg.info('using migrations directory {}'.format(migrationsdir))
-
-    session = SessionBase.create_session()
-
-    ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
-    ac.set_main_option('sqlalchemy.url', database_engine)
-    ac.set_main_option('script_location', migrationsdir)
-
-    alembic.command.downgrade(ac, 'base')
-    alembic.command.upgrade(ac, 'head')
-
-    session.commit()
-
-    yield session
-    session.commit()
-    session.close()
-
-
-@pytest.fixture(scope='function')
-def list_tokens(
-        ):
-    return {
-            'foo': '0x' + os.urandom(20).hex(),
-            'bar': '0x' + os.urandom(20).hex(),
-        }
-
-
-@pytest.fixture(scope='function')
-def list_actors(
-        ):
-    return {
-            'alice': '0x' + os.urandom(20).hex(),
-            'bob': '0x' + os.urandom(20).hex(),
-            'charlie': '0x' + os.urandom(20).hex(),
-            'diane': '0x' + os.urandom(20).hex(),
-            }
-
-
-@pytest.fixture(scope='function')
-def list_defaults(
-        ):
-
-    return {
-        'block': 420000,
-    }
-
-
-@pytest.fixture(scope='function')
-def tags(
-        init_database,
-        ):
-
-    add_tag(init_database, 'foo') 
-    add_tag(init_database, 'baz', domain='bar') 
-    add_tag(init_database, 'xyzzy', domain='bar') 
-    init_database.commit()

apps/cic-cache/tests/test_api.py

@@ -1,31 +0,0 @@
-# standard imports
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache.runnable.daemons.query import process_transactions_all_data
-
-
-def test_api_all_data(
-        init_database,
-        txs,
-        ):
-
-    env = {
-        'PATH_INFO': '/txa/410000/420000',
-        'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    j = process_transactions_all_data(init_database, env)
-    o = json.loads(j[1])
-    
-    assert len(o['data']) == 2
-
-    env = {
-        'PATH_INFO': '/txa/420000/410000',
-        'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-   
-    with pytest.raises(ValueError):
-        j = process_transactions_all_data(init_database, env)

apps/cic-cache/tests/test_cache.py

@@ -1,56 +0,0 @@
-# standard imports
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache import BloomCache
-from cic_cache.cache import DataCache
-
-logg = logging.getLogger()
-
-
-def test_cache(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        ):
-
-    session = init_database
-
-    c = BloomCache(session)
-    b = c.load_transactions(0, 100)
-
-    assert b[0] == list_defaults['block'] - 1
-
-    c = BloomCache(session)
-    c.load_transactions_account(list_actors['alice'],0, 100)
-
-    assert b[0] == list_defaults['block'] - 1
-
-
-def test_cache_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        tag_txs,
-        ):
-
-    session = init_database
-
-    c = DataCache(session)
-    b = c.load_transactions_with_data(410000, 420000)
-
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[1]
-    assert b[2][1]['tx_type'] == 'unknown'
-    assert b[2][0]['tx_type'] == 'test.taag'
-    

apps/cic-cache/tests/test_query.py

@@ -1,230 +0,0 @@
-# standard imports
-import logging
-import json
-import base64
-import copy
-import re
-
-# external imports
-import pytest
-from hexathon import strip_0x
-
-# local imports
-from cic_cache.runnable.daemons.query import *
-
-logg = logging.getLogger()
-
-
-
-@pytest.mark.parametrize(
-        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
-        [
-            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
-            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
-            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
-            ],
-        )
-def test_query_regex(
-        list_actors,
-        query_path_prefix,
-        query_role,
-        query_address_index,
-        query_offset,
-        query_offset_index,
-        query_limit,
-        query_limit_index,
-        match_re,
-        ):
-
-        paths = []
-        path = query_path_prefix
-        query_address = None
-        if query_role != None:
-            query_address = strip_0x(list_actors[query_role])
-            paths.append(path + '0x' + query_address)
-            paths.append(path + query_address)
-        if query_offset != None:
-            if query_limit != None:
-                for i in range(len(paths)-1):
-                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
-            else:
-                for i in range(len(paths)-1):
-                    paths[i] += '/' + str(query_offset)
-
-        for i in range(len(paths)):
-            paths.append(paths[i] + '/')
-
-        for p in paths:
-            logg.debug('testing path {} against {}'.format(p, match_re))
-            m = re.match(match_re, p)
-            l = len(m.groups())
-            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
-            for i in range(l+1):
-                logg.debug('group {} {}'.format(i, m[i]))
-            if m.lastindex >= query_offset_index:
-                assert query_offset == int(m[query_offset_index + 1])
-            if m.lastindex >= query_limit_index:
-                assert query_limit == int(m[query_limit_index + 1])
-            if query_address_index != None:
-                match_address = strip_0x(m[query_address_index + 1])
-                assert query_address == match_address
-
-
-
-@pytest.mark.parametrize(
-        'role_name, query_offset, query_limit, query_match',
-        [ 
-            ('alice', None, None, [(420000, 13), (419999, 42)]),
-            ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
-            ('alice', 2, None, []), # 420000 == list_defaults['block']
-            ],
-        )
-def test_query_process_txs_account(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        role_name,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    actor = None
-    try:
-        actor = list_actors[role_name]
-    except KeyError:
-        actor = os.urandom(20).hex()
-    path_info = '/tx/user/0x' + strip_0x(actor)
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_account_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_offset, query_limit, query_match',
-        [ 
-            (None, 2, [(420000, 13), (419999, 42)]),
-            (0, 1, [(420000, 13)]),
-            (1, 1, [(419999, 42)]),
-            (2, 0, []),
-            ],
-        )
-def test_query_process_txs_bloom(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    path_info = '/tx'
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_block_start, query_block_end, query_match_count',
-        [ 
-            (None, 42, 0),
-            (420000, 420001, 1),
-            (419999, 419999, 1), # matches are inclusive
-            (419999, 420000, 2),
-            (419999, 420001, 2),
-            ],
-        )
-def test_query_process_txs_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_block_start,
-        query_block_end,
-        query_match_count,
-        ):
-
-    path_info = '/txa'
-    if query_block_start != None:
-        path_info += '/' + str(query_block_start)
-    if query_block_end != None:
-        if query_block_start == None:
-            path_info += '/0'
-        path_info += '/' + str(query_block_end)
-    env = {
-            'PATH_INFO': path_info,
-            'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_data(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    assert len(o['data']) == query_match_count

apps/cic-cache/tests/test_tag.py

@@ -1,37 +0,0 @@
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache.db import tag_transaction
-
-logg = logging.getLogger()
-
-
-def test_cache(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        tags,
-        ):
-
-    tag_transaction(init_database, txs[0], 'foo')
-    tag_transaction(init_database, txs[0], 'baz', domain='bar')
-    tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
-    assert r[0][0] == txs[0]
-
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
-    assert r[0][0] == txs[0]
-
-    
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
-    assert r[0][0] == txs[1]

apps/cic-cache/tests/test_task.py

@@ -1,27 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-
-# local imports
-from cic_cache.api import Api
-
-logg = logging.getLogger()
-
-
-def test_task(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        celery_session_worker,
-        ):
-
-    api = Api(queue=None)
-    t = api.list(0, 100)
-    r = t.get()
-    logg.debug('r {}'.format(r))
-
-    assert r['low'] == list_defaults['block'] - 1

apps/cic-eth-aux/erc20-demurrage-token/MANIFEST.in

@@ -1 +0,0 @@
-include *requirements.txt

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/__init__.py

@@ -1,53 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-from erc20_demurrage_token.demurrage import DemurrageCalculator
-from chainlib.connection import RPCConnection
-from chainlib.chain import ChainSpec
-from chainlib.eth.constant import ZERO_ADDRESS
-from cic_eth_registry import CICRegistry
-
-logg = logging.getLogger(__name__)
-
-celery_app = celery.current_app
-
-
-class NoopCalculator:
-
-    def amount_since(self, amount, timestamp):
-        logg.debug('noopcalculator amount {} timestamp {}'.format(amount, timestamp))
-        return amount
-
-
-class DemurrageCalculationTask(celery.Task):
-
-    demurrage_token_calcs = {}
-
-    @classmethod
-    def register_token(cls, rpc, chain_spec, token_symbol, sender_address=ZERO_ADDRESS):
-        registry = CICRegistry(chain_spec, rpc)
-        token_address = registry.by_name(token_symbol, sender_address=sender_address)
-        try:
-            c = DemurrageCalculator.from_contract(rpc, chain_spec, token_address, sender_address=sender_address)
-            logg.info('found demurrage calculator for ERC20 {} @ {}'.format(token_symbol, token_address))
-        except:
-            logg.warning('Token {} at address {} does not appear to be a demurrage contract. Calls to balance adjust for this token will always return the same amount'.format(token_symbol, token_address))
-            c = NoopCalculator()
-
-        cls.demurrage_token_calcs[token_symbol] = c
-
-
-@celery_app.task(bind=True, base=DemurrageCalculationTask)
-def get_adjusted_balance(self, token_symbol, amount, timestamp):
-    c = self.demurrage_token_calcs[token_symbol]
-    return c.amount_since(amount, timestamp)
-
-
-def aux_setup(rpc, config, sender_address=ZERO_ADDRESS):
-    chain_spec_str = config.get('CIC_CHAIN_SPEC')
-    chain_spec = ChainSpec.from_chain_str(chain_spec_str)
-    token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
-    
-    DemurrageCalculationTask.register_token(rpc, chain_spec, token_symbol, sender_address=sender_address)

apps/cic-eth-aux/erc20-demurrage-token/cic_eth_aux/erc20_demurrage_token/api.py

@@ -1,30 +0,0 @@
-# standard imports
-import logging
-
-# external imports 
-import celery
-from cic_eth.api.base import ApiBase
-
-app = celery.current_app
-
-logg = logging.getLogger(__name__)
-
-
-class Api(ApiBase):
-   
-    def get_adjusted_balance(self, token_symbol, balance, timestamp):
-        s = celery.signature(
-                'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
-                [
-                    token_symbol,
-                    balance,
-                    timestamp,
-                    ],
-                queue=None,
-                )
-        if self.callback_param != None:
-            s.link(self.callback_success)
-            s.link.on_error(self.callback_error)
-
-        t = s.apply_async(queue=self.queue)
-        return t

apps/cic-eth-aux/erc20-demurrage-token/requirements.txt

@@ -1,5 +0,0 @@
-celery==4.4.7
-erc20-demurrage-token~=0.0.2a3
-cic-eth-registry~=0.5.6a1
-chainlib~=0.0.5a1
-cic_eth~=0.12.0a2

apps/cic-eth-aux/erc20-demurrage-token/setup.py

@@ -1,25 +0,0 @@
-from setuptools import setup
-
-requirements = []
-f = open('requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    requirements.append(l.rstrip())
-f.close()
-
-test_requirements = []
-f = open('test_requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    test_requirements.append(l.rstrip())
-f.close()
-
-
-setup(
-        install_requires=requirements,
-        tests_require=test_requirements,
-        )

apps/cic-eth-aux/erc20-demurrage-token/test_requirements.txt

@@ -1,12 +0,0 @@
-pytest==6.0.1
-pytest-celery==0.0.0a1
-pytest-mock==3.3.1
-pytest-cov==2.10.1
-eth-tester==0.5.0b3
-py-evm==0.3.0a20
-SQLAlchemy==1.3.20
-cic-eth~=0.12.0a1
-liveness~=0.0.1a7
-eth-accounts-index==0.0.12a1
-eth-contract-registry==0.5.6a1
-eth-address-index==0.1.2a1

apps/cic-eth-aux/erc20-demurrage-token/tests/conftest.py

@@ -1,88 +0,0 @@
-# external imports
-import celery
-from chainlib.eth.pytest.fixtures_chain import *
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from cic_eth_registry.pytest.fixtures_tokens import *
-from erc20_demurrage_token.unittest.base import TestTokenDeploy
-from erc20_demurrage_token.token import DemurrageToken
-from eth_token_index.index import TokenUniqueSymbolIndex
-from eth_address_declarator.declarator import AddressDeclarator
-
-# cic-eth imports
-from cic_eth.pytest.fixtures_celery import *
-from cic_eth.pytest.fixtures_token import *
-from cic_eth.pytest.fixtures_config import *
-
-
-@pytest.fixture(scope='function')
-def demurrage_token(
-        default_chain_spec,
-        eth_rpc,
-        token_registry,
-        contract_roles,
-        eth_signer,
-        ):
-    d = TestTokenDeploy(eth_rpc, token_symbol='BAR', token_name='Bar Token')
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    c = DemurrageToken(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    token_address =  d.deploy(eth_rpc, contract_roles['CONTRACT_DEPLOYER'], c, 'SingleNocap')
-    logg.debug('demurrage token contract "BAR" deployed to {}'.format(token_address))
-
-    return token_address
-    
-
-@pytest.fixture(scope='function')
-def demurrage_token_symbol(
-        default_chain_spec,
-        eth_rpc,
-        demurrage_token,
-        contract_roles,
-        ):
-
-    c = DemurrageToken(default_chain_spec)
-    o = c.symbol(demurrage_token, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-    r = eth_rpc.do(o)
-    return c.parse_symbol(r)
-
-
-@pytest.fixture(scope='function')
-def demurrage_token_declaration(
-        foo_token_declaration,
-        ):
-    return foo_token_declaration
-
-
-@pytest.fixture(scope='function')
-def register_demurrage_token(
-        default_chain_spec,
-        token_registry,
-        eth_rpc,
-        eth_signer,
-        register_lookups,
-        contract_roles,
-        demurrage_token_declaration,
-        demurrage_token,
-        address_declarator,
-        ):
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
-
-    c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.register(token_registry, contract_roles['CONTRACT_DEPLOYER'], demurrage_token)
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    nonce_oracle = RPCNonceOracle(contract_roles['TRUSTED_DECLARATOR'], eth_rpc)
-    c = AddressDeclarator(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.add_declaration(address_declarator, contract_roles['TRUSTED_DECLARATOR'], demurrage_token, demurrage_token_declaration)
-
-    eth_rpc.do(o)
-    o = receipt(tx_hash_hex)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    return token_registry
-

apps/cic-eth-aux/erc20-demurrage-token/tests/test_demurrage_task.py

@@ -1,69 +0,0 @@
-# standard imports
-import logging
-import copy
-import datetime
-
-# external imports
-import celery
-
-# cic-eth imports
-from cic_eth_aux.erc20_demurrage_token import (
-        DemurrageCalculationTask,
-        aux_setup,
-        )
-from cic_eth_aux.erc20_demurrage_token.api import Api as AuxApi
-
-logg = logging.getLogger()
-
-
-def test_demurrage_calulate_task(
-        default_chain_spec,
-        eth_rpc,
-        cic_registry,
-        celery_session_worker,
-        register_demurrage_token,
-        demurrage_token_symbol,
-        contract_roles,
-        load_config,
-        ):
- 
-    config = copy.copy(load_config)
-    config.add(str(default_chain_spec), 'CIC_CHAIN_SPEC', exists_ok=True)
-    config.add(demurrage_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
-    aux_setup(eth_rpc, load_config, sender_address=contract_roles['CONTRACT_DEPLOYER'])
-
-    since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
-    s = celery.signature(
-            'cic_eth_aux.erc20_demurrage_token.get_adjusted_balance',
-            [
-                demurrage_token_symbol,
-                1000,
-                since.timestamp(),
-                ],
-            queue=None,
-            )
-    t = s.apply_async()
-    r = t.get_leaf()
-    assert t.successful()
-    assert r == 980
-
-
-
-def test_demurrage_calculate_api(
-        default_chain_spec,
-        eth_rpc,
-        cic_registry,
-        celery_session_worker,
-        register_demurrage_token,
-        demurrage_token_symbol,
-        contract_roles,
-        load_config,
-        ):
-
-        api = AuxApi(str(default_chain_spec), queue=None)
-        since = datetime.datetime.utcnow() - datetime.timedelta(minutes=1)
-        t = api.get_adjusted_balance(demurrage_token_symbol, 1000, since.timestamp())
-        r = t.get_leaf()
-        assert t.successful()
-        assert r == 980
-

apps/cic-eth/.coveragerc

@@ -5,5 +5,3 @@ omit =
 	cic_eth/db/migrations/*
 	cic_eth/sync/head.py
 	cic_eth/sync/mempool.py
-	cic_eth/queue/state.py
-	*redis*.py

apps/cic-eth/.dockerignore

@@ -1,6 +0,0 @@
-.git
-.cache
-.dot
-**/doc
-**/.venv
-**/venv

apps/cic-eth/.gitlab-ci.yml

@@ -1,52 +1,22 @@
 .cic_eth_variables:
     variables:
         APP_NAME: cic-eth
-        DOCKERFILE_PATH: docker/Dockerfile_ci
-        CONTEXT: apps/$APP_NAME
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_eth_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
 
 build-mr-cic-eth:
     extends:
+        - .cic_eth_changes_target
+        - .py_build_merge_request
         - .cic_eth_variables
-        - .py_build_target_dev
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
-
-test-mr-cic-eth:
-    stage: test
-    extends:
-        - .cic_eth_variables
-    cache:
-        key:
-            files:
-                - test_requirements.txt
-        paths:
-        - /root/.cache/pip
-    image: $MR_IMAGE_TAG
-    script:
-        - cd apps/$APP_NAME/
-        - >
-          pip install --extra-index-url https://pip.grassrootseconomics.net:8433
-          --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-          -r admin_requirements.txt
-          -r services_requirements.txt
-          -r test_requirements.txt 
-        - export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
-    needs: ["build-mr-cic-eth"]
-    rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
 
 build-push-cic-eth:
     extends:
         - .py_build_push
         - .cic_eth_variables
-    rules:
-    - if:  $CI_COMMIT_BRANCH == "master"
-      changes:
-          - apps/cic-eth/**/*
-      when: always
+        
+

apps/cic-eth/MANIFEST.in

@@ -1,2 +0,0 @@
-include *requirements.txt config/test/*
-

apps/cic-eth/admin_requirements.txt

@@ -1,5 +0,0 @@
-SQLAlchemy==1.3.20
-cic-eth-registry>=0.5.6a2,<0.6.0
-hexathon~=0.0.1a7
-chainqueue>=0.0.3a1,<0.1.0
-eth-erc20>=0.0.10a3,<0.1.0

apps/cic-eth/cic_eth/admin/ctrl.py

@@ -2,36 +2,20 @@
 import datetime
 import logging
 
-# external imports
+# third-party imports
 import celery
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.chain import ChainSpec
-from hexathon import (
-        add_0x,
-        strip_0x,
-        uniform as hex_uniform,
-        )
+from cic_registry import zero_address
 
 # local imports
 from cic_eth.db.enum import LockEnum
-from cic_eth.db.models.base import SessionBase
 from cic_eth.db.models.lock import Lock
-from cic_eth.task import (
-        CriticalSQLAlchemyTask,
-        )
 from cic_eth.error import LockedError
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
-
-def normalize_address(a):
-    if a == None:
-        return None
-    return add_0x(hex_uniform(strip_0x(a)))
-
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
+@celery_app.task()
+def lock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL, tx_hash=None):
     """Task wrapper to set arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -43,17 +27,13 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
     logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
+@celery_app.task()
+def unlock(chained_input, chain_str, address=zero_address, flags=LockEnum.ALL):
     """Task wrapper to reset arbitrary locks
 
     :param chain_str: Chain spec string representation
@@ -65,17 +45,13 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, flags, address=address)
     logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+@celery_app.task()
+def lock_send(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set send lock
 
     :param chain_str: Chain spec string representation
@@ -85,15 +61,13 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
     logg.debug('Send locked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+@celery_app.task()
+def unlock_send(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset send lock
 
     :param chain_str: Chain spec string representation
@@ -103,15 +77,13 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.SEND, address=address)
     logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
+@celery_app.task()
+def lock_queue(chained_input, chain_str, address=zero_address, tx_hash=None):
     """Task wrapper to set queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -121,15 +93,13 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=Non
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
     logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
+@celery_app.task()
+def unlock_queue(chained_input, chain_str, address=zero_address):
     """Task wrapper to reset queue direct lock
 
     :param chain_str: Chain spec string representation
@@ -139,33 +109,17 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
     :returns: New lock state for address
     :rtype: number
     """
-    address = normalize_address(address)
-    chain_str = str(ChainSpec.from_dict(chain_spec_dict))
     r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
     logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
     return chained_input
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
-    address = normalize_address(address)
-    chain_str = '::'
-    if chain_spec_dict != None:
-        chain_str = str(ChainSpec.from_dict(chain_spec_dict))
-    session = SessionBase.create_session()
-    r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
+@celery_app.task()
+def check_lock(chained_input, chain_str, lock_flags, address=None):
+    r = Lock.check(chain_str, lock_flags, address=zero_address)
     if address != None:
-        r |= Lock.check(chain_str, lock_flags, address=address, session=session)
+        r |= Lock.check(chain_str, lock_flags, address=address)
     if r > 0:
         logg.debug('lock check {} has match {} for {}'.format(lock_flags, r, address))
-        session.close()
         raise LockedError(r)
-    session.flush()
-    session.close()
     return chained_input
-
-
-@celery_app.task()
-def shutdown(message):
-    logg.critical('shutdown called: {}'.format(message))
-    celery_app.control.shutdown() #broadcast('shutdown')

apps/cic-eth/cic_eth/admin/debug.py

@@ -1,25 +1,12 @@
-# standard imports
 import datetime
 
-# external imports
 import celery
 
-# local imports
-from cic_eth.db.models.debug import Debug
-from cic_eth.db.models.base import SessionBase
-from cic_eth.task import CriticalSQLAlchemyTask
-
 celery_app = celery.current_app
 
 
-@celery_app.task(base=CriticalSQLAlchemyTask)
-def alert(chained_input, tag, txt):
-    session = SessionBase.create_session()
-
-    o = Debug(tag, txt)
-    session.add(o)
-    session.commit()
-
-    session.close()
-    
-    return chained_input
+@celery_app.task()
+def out_tmp(tag, txt):
+    f = open('/tmp/err.{}.txt'.format(tag), "w")
+    f.write(txt)
+    f.close()

apps/cic-eth/cic_eth/admin/nonce.py

@@ -1,45 +1,32 @@
 # standard imports
 import logging
 
-# external imports
+# third-party imports
 import celery
-from chainlib.chain import ChainSpec
-from chainlib.connection import RPCConnection
-from chainlib.eth.tx import (
-        unpack,
-        TxFactory,
-        )
-from chainlib.eth.gas import OverrideGasOracle
-from chainqueue.sql.query import get_tx
-from chainqueue.sql.state import set_cancel
-from chainqueue.db.models.otx import Otx
-from chainqueue.db.models.tx import TxCache
-from hexathon import (
-        strip_0x,
-        add_0x,
-        uniform as hex_uniform,
-        )
-from potaahto.symbols import snake_and_camel
+from cic_registry.chain import ChainSpec
 
 # local imports
 from cic_eth.db.models.base import SessionBase
+from cic_eth.db.models.otx import Otx
+from cic_eth.db.models.tx import TxCache
 from cic_eth.db.models.nonce import Nonce
-from cic_eth.admin.ctrl import (
-        lock_send,
-        unlock_send,
-        lock_queue,
-        unlock_queue,
-        )
-from cic_eth.queue.tx import queue_create
-from cic_eth.eth.gas import create_check_gas_task
-from cic_eth.task import BaseTask
+from cic_eth.admin.ctrl import lock_send
+from cic_eth.admin.ctrl import unlock_send
+from cic_eth.admin.ctrl import lock_queue
+from cic_eth.admin.ctrl import unlock_queue
+from cic_eth.queue.tx import get_tx
+from cic_eth.queue.tx import set_cancel
+from cic_eth.queue.tx import create as queue_create
+from cic_eth.eth.util import unpack_signed_raw_tx
+from cic_eth.eth.task import sign_tx
+from cic_eth.eth.task import create_check_gas_and_send_task
 
 celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-@celery_app.task(bind=True, base=BaseTask)
-def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
+@celery_app.task(bind=True)
+def shift_nonce(self, chain_str, tx_hash_orig_hex, delta=1):
     """Shift all transactions with nonces higher than the offset by the provided position delta.
 
     Transactions who are replaced by transactions that move nonces will be marked as OVERRIDDEN.
@@ -50,109 +37,89 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
     :type tx_hash_orig_hex: str, 0x-hex
     :param delta: Amount
     """
-    chain_spec = ChainSpec.from_dict(chainspec_dict)
-    rpc = RPCConnection.connect(chain_spec, 'default')
-    rpc_signer = RPCConnection.connect(chain_spec, 'signer')
     queue = None
     try:
         queue = self.request.delivery_info.get('routing_key')
     except AttributeError:
         pass
 
-    session = BaseTask.session_func()
-    tx_brief = get_tx(chain_spec, tx_hash_orig_hex, session=session)
-    tx_raw = bytes.fromhex(strip_0x(tx_brief['signed_tx']))
-    tx = unpack(tx_raw, chain_spec)
+    chain_spec = ChainSpec.from_chain_str(chain_str)
+    tx_brief = get_tx(tx_hash_orig_hex)
+    tx_raw = bytes.fromhex(tx_brief['signed_tx'][2:])
+    tx = unpack_signed_raw_tx(tx_raw, chain_spec.chain_id())
     nonce = tx_brief['nonce']
     address = tx['from']
 
-    logg.debug('shifting nonce {} position(s) for address {}, offset {}, hash {}'.format(delta, address, nonce, tx['hash']))
+    logg.debug('shifting nonce {} position(s) for address {}, offset {}'.format(delta, address, nonce))
 
-    lock_queue(None, chain_spec.asdict(), address=address)
-    lock_send(None, chain_spec.asdict(), address=address)
+    lock_queue(None, chain_str, address)
+    lock_send(None, chain_str, address)
 
-    set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
-
-    query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
+    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
-    q = q.filter(TxCache.sender==query_address)
+    q = q.filter(TxCache.sender==address)
     q = q.filter(Otx.nonce>=nonce+delta)
     q = q.order_by(Otx.nonce.asc())
     otxs = q.all()
 
     tx_hashes = []
     txs = []
-    gas_total = 0
     for otx in otxs:
-        tx_raw = bytes.fromhex(strip_0x(otx.signed_tx))
-        tx_new = unpack(tx_raw, chain_spec)
-        tx_new = snake_and_camel(tx_new)
+        tx_raw = bytes.fromhex(otx.signed_tx[2:])
+        tx_new = unpack_signed_raw_tx(tx_raw, chain_spec.chain_id())
 
         tx_previous_hash_hex = tx_new['hash']
         tx_previous_nonce = tx_new['nonce']
 
-        tx_new['gas_price'] += 1
-        tx_new['gasPrice'] = tx_new['gas_price']
-        tx_new['nonce'] -= delta
-        gas_total += tx_new['gas_price'] * tx_new['gas']
-
-        logg.debug('tx_new {}'.format(tx_new))
-        logg.debug('gas running total {}'.format(gas_total))
-
         del(tx_new['hash'])
         del(tx_new['hash_unsigned'])
-        del(tx_new['hashUnsigned'])
+        tx_new['nonce'] -= delta
 
-        gas_oracle = OverrideGasOracle(limit=tx_new['gas'], price=tx_new['gas_price'] + 1) # TODO: it should be possible to merely set this price here and if missing in the existing struct then fill it in (chainlib.eth.tx)
-        c = TxFactory(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle)
-        (tx_hash_hex, tx_signed_raw_hex) = c.build_raw(tx_new)
+        (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx_new, chain_str)
         logg.debug('tx {} -> {} nonce {} -> {}'.format(tx_previous_hash_hex, tx_hash_hex, tx_previous_nonce, tx_new['nonce']))
 
         otx = Otx(
-            tx_new['nonce'],
-            tx_hash_hex,
-            tx_signed_raw_hex,
-            )
+            nonce=tx_new['nonce'],
+            address=tx_new['from'],
+            tx_hash=tx_hash_hex,
+            signed_tx=tx_signed_raw_hex,
+                )
         session.add(otx)
+        session.commit()
 
         # TODO: cancel all first, then replace. Otherwise we risk two non-locked states for two different nonces.
-        set_cancel(chain_spec, strip_0x(tx_previous_hash_hex), manual=True, session=session)
+        set_cancel(tx_previous_hash_hex, True)
 
-        TxCache.clone(tx_previous_hash_hex, tx_hash_hex, session=session)
+        TxCache.clone(tx_previous_hash_hex, tx_hash_hex)
 
         tx_hashes.append(tx_hash_hex)
         txs.append(tx_signed_raw_hex)
-        session.commit()
 
     session.close()
 
-    s = create_check_gas_task(
+    s = create_check_gas_and_send_task(
          txs, 
-         chain_spec,
-         #tx_new['from'],
-         address,
-         #gas=tx_new['gas'],
-         gas=gas_total,
-         tx_hashes_hex=tx_hashes, 
-         queue=queue,
+         chain_str,
+         tx_new['from'],
+         tx_new['gas'],
+         tx_hashes, 
+         queue,
         )
 
     s_unlock_send = celery.signature(
         'cic_eth.admin.ctrl.unlock_send',
         [
-            chain_spec.asdict(),
-            address,
-            #tx_new['from'],
+            chain_str,
+            tx_new['from'],
             ],
         queue=queue,
         )
     s_unlock_direct = celery.signature(
         'cic_eth.admin.ctrl.unlock_queue',
         [
-            chain_spec.asdict(),
-            address,
-            #tx_new['from'],
+            chain_str,
+            tx_new['from'],
             ],
         queue=queue,
         )

apps/cic-eth/cic_eth/admin/token.py

@@ -1,21 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-# local imports
-from cic_eth.task import BaseTask
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-
-@celery_app.task(bind=True, base=BaseTask)
-def default_token(self):
-    return {
-            'symbol': self.default_token_symbol,
-            'address': self.default_token_address,
-            'name': self.default_token_name,
-            'decimals': self.default_token_decimals,
-            }

apps/cic-eth/cic_eth/api/__init__.py

@@ -5,3 +5,4 @@
 """
 
 from .api_task import Api
+from .api_admin import AdminApi

apps/cic-eth/cic_eth/api/admin.py

@@ -1,644 +0,0 @@
-# standard imports
-import logging
-import sys
-
-# external imports
-import celery
-from chainlib.eth.constant import (
-        ZERO_ADDRESS,
-        )
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import UnknownContractError
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.contract import code
-from chainlib.eth.tx import (
-        transaction,
-        receipt,
-        unpack,
-        )
-from chainlib.hash import keccak256_hex_to_hex
-from hexathon import (
-        strip_0x,
-        add_0x,
-        uniform as hex_uniform,
-        )
-from chainlib.eth.gas import balance
-from chainqueue.db.enum import (
-        StatusEnum,
-        StatusBits,
-        is_alive,
-        is_error_status,
-        status_str,
-    )
-from chainqueue.error import TxStateChangeError
-from chainqueue.sql.query import get_tx
-from eth_erc20 import ERC20
-
-# local imports
-from cic_eth.db.models.base import SessionBase
-from cic_eth.db.models.role import AccountRole
-from cic_eth.db.models.nonce import Nonce
-from cic_eth.error import InitializationError
-
-app = celery.current_app
-
-#logg = logging.getLogger(__file__)
-logg = logging.getLogger()
-
-local_fail = StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.UNKNOWN_ERROR
-
-
-class AdminApi:
-    """Provides an interface to view and manipulate existing transaction tasks and system runtime settings.
-
-    :param rpc_client: Rpc client to use for blockchain connections.
-    :type rpc_client: cic_eth.eth.rpc.RpcClient
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, rpc, queue='cic-eth', call_address=ZERO_ADDRESS):
-        self.rpc = rpc
-        self.queue = queue
-        self.call_address = call_address
-
-
-    def proxy_do(self, chain_spec, o):
-        s_proxy = celery.signature(
-                'cic_eth.task.rpc_proxy',
-                [
-                    chain_spec.asdict(),
-                    o,
-                    'default',
-                    ],
-                queue=self.queue
-                )
-        return s_proxy.apply_async()
-
-
-    
-    def registry(self):
-        s_registry = celery.signature(
-                'cic_eth.task.registry',
-                [],
-                queue=self.queue
-                )
-        return s_registry.apply_async()
-
-
-    def unlock(self, chain_spec, address, flags=None):
-        s_unlock = celery.signature(
-            'cic_eth.admin.ctrl.unlock',
-            [
-                None,
-                chain_spec.asdict(),
-                address,
-                flags,
-                ],
-            queue=self.queue,
-            )
-        return s_unlock.apply_async()
-
-
-    def lock(self, chain_spec, address, flags=None):
-        s_lock = celery.signature(
-            'cic_eth.admin.ctrl.lock',
-            [
-                None,
-                chain_spec.asdict(),
-                address,
-                flags,
-                ],
-            queue=self.queue,
-            )
-        return s_lock.apply_async()
-
-
-    def get_lock(self):
-        s_lock = celery.signature(
-            'cic_eth.queue.lock.get_lock',
-            [],
-            queue=self.queue,
-            )
-        return s_lock.apply_async()
-
-
-    def tag_account(self, tag, address_hex, chain_spec):
-        """Persistently associate an address with a plaintext tag.
-
-        Some tags are known by the system and is used to resolve addresses to use for certain transactions. 
-
-        :param tag: Address tag
-        :type tag: str
-        :param address_hex: Ethereum address to tag
-        :type address_hex: str, 0x-hex
-        :raises ValueError: Invalid checksum address
-        """
-        s_tag = celery.signature(
-            'cic_eth.eth.account.set_role',
-            [
-                tag,
-                address_hex,
-                chain_spec.asdict(),
-                ],
-            queue=self.queue,
-            )
-        return s_tag.apply_async()
-
-
-    def have_account(self, address_hex, chain_spec):
-        s_have = celery.signature(
-            'cic_eth.eth.account.have',
-            [
-                address_hex,
-                chain_spec.asdict(),
-                ],
-            queue=self.queue,
-            )
-        return s_have.apply_async()
-
-
-    def resend(self, tx_hash_hex, chain_spec, in_place=True, unlock=False):
-
-        logg.debug('resend {}'.format(tx_hash_hex))
-        s_get_tx_cache = celery.signature(
-            'cic_eth.queue.query.get_tx_cache',
-            [
-                chain_spec.asdict(),
-                tx_hash_hex,
-                ],
-            queue=self.queue,
-            )
-
-        # TODO: This check should most likely be in resend task itself
-        tx_dict = s_get_tx_cache.apply_async().get()
-        if not is_alive(getattr(StatusEnum, tx_dict['status']).value):
-            raise TxStateChangeError('Cannot resend mined or obsoleted transaction'.format(txold_hash_hex))
-        
-        if not in_place:
-            raise NotImplementedError('resend as new not yet implemented')
-
-        s = celery.signature(
-            'cic_eth.eth.gas.resend_with_higher_gas',
-            [
-                chain_spec.asdict(),
-                None,
-                1.01,
-                ],
-            queue=self.queue,
-            )
-
-        s_manual = celery.signature(
-            'cic_eth.queue.state.set_manual',
-            [
-                chain_spec.asdict(),
-                tx_hash_hex,
-                ],
-            queue=self.queue,
-            )
-        s_manual.link(s)
-
-        if unlock:
-            s_gas = celery.signature(
-                'cic_eth.admin.ctrl.unlock_send',
-                [
-                    chain_spec.asdict(),
-                    tx_dict['sender'],
-                ],
-                queue=self.queue,
-                )
-            s.link(s_gas)
-
-        return s_manual.apply_async()
-                    
-
-    def check_nonce(self, chain_spec, address):
-        s = celery.signature(
-                'cic_eth.queue.query.get_account_tx',
-                [
-                    chain_spec.asdict(),
-                    address,
-                    True,
-                    False,
-                    ],
-                queue=self.queue,
-                )
-        txs = s.apply_async().get()
-
-        blocking_tx = None
-        blocking_nonce = None
-        nonce_otx = 0
-        last_nonce = -1
-        for k in txs.keys():
-            s_get_tx = celery.signature(
-                    'cic_eth.queue.query.get_tx',
-                    [
-                        chain_spec.asdict(),
-                        k,
-                        ],
-                    queue=self.queue,
-                    )
-            tx = s_get_tx.apply_async().get()
-            logg.debug('checking nonce {} (previous {})'.format(tx['nonce'], last_nonce))
-            nonce_otx = tx['nonce']
-            if not is_alive(tx['status']) and tx['status'] & local_fail > 0:
-                logg.info('permanently errored {} nonce {} status {}'.format(k, nonce_otx, status_str(tx['status'])))
-                blocking_tx = k
-                blocking_nonce = nonce_otx
-            elif nonce_otx - last_nonce > 1:
-                logg.debug('tx {}'.format(tx))
-                tx_obj = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
-                logg.error('nonce gap; {} followed {} for account {}'.format(nonce_otx, last_nonce, tx_obj['from']))
-                blocking_tx = k
-                blocking_nonce = nonce_otx
-                break
-            last_nonce = nonce_otx
-
-        return {
-            'nonce': {
-                #'network': nonce_cache,
-                'queue': nonce_otx,
-                #'cache': nonce_cache,
-                'blocking': blocking_nonce,
-            },
-            'tx': {
-                'blocking': add_0x(blocking_tx),
-            }
-        }
-
-
-    # TODO: is risky since it does not validate that there is actually a nonce problem?
-    def fix_nonce(self, chain_spec, address, nonce):
-        s = celery.signature(
-                'cic_eth.queue.query.get_account_tx',
-                [
-                    chain_spec.asdict(),
-                    address,
-                    True,
-                    False,
-                    ],
-                queue=self.queue,
-                )
-        txs = s.apply_async().get()
-
-        tx_hash_hex = None
-        session = SessionBase.create_session()
-        for k in txs.keys():
-            tx_dict = get_tx(chain_spec, k, session=session)
-            if tx_dict['nonce'] == nonce:
-                tx_hash_hex = k
-        session.close()
-
-        s_nonce = celery.signature(
-                'cic_eth.admin.nonce.shift_nonce',
-                [
-                    chain_spec.asdict(),
-                    tx_hash_hex, 
-                ],
-                queue=self.queue
-                )
-        return s_nonce.apply_async()
-
-
-    def account(self, chain_spec, address, include_sender=True, include_recipient=True, renderer=None, w=sys.stdout):
-        """Lists locally originated transactions for the given Ethereum address.
-
-        Performs a synchronous call to the Celery task responsible for performing the query.
-
-        :param address: Ethereum address to return transactions for
-        :type address: str, 0x-hex
-        """
-
-        address = add_0x(hex_uniform(strip_0x(address)))
-        last_nonce = -1
-        s = celery.signature(
-                'cic_eth.queue.query.get_account_tx',
-                [
-                    chain_spec.asdict(),
-                    address,
-                    ],
-                queue=self.queue,
-                )
-        txs = s.apply_async().get()
-
-        tx_dict_list = []
-        for tx_hash in txs.keys():
-            errors = []
-            s = celery.signature(
-                    'cic_eth.queue.query.get_tx_cache',
-                    [
-                        chain_spec.asdict(),
-                        tx_hash,
-                        ],
-                    queue=self.queue,
-                    )
-            tx_dict = s.apply_async().get()
-            if tx_dict['sender'] == address:
-                if tx_dict['nonce'] - last_nonce > 1:
-                    logg.error('nonce gap; {} followed {} for address {} tx {}'.format(tx_dict['nonce'], last_nonce, tx_dict['sender'], tx_hash))
-                    errors.append('nonce')
-                elif tx_dict['nonce'] == last_nonce:
-                    logg.info('nonce {} duplicate for address {} in tx {}'.format(tx_dict['nonce'], tx_dict['sender'], tx_hash))
-                last_nonce = tx_dict['nonce']
-                if not include_sender:
-                    logg.debug('skipping sender tx {}'.format(tx_dict['tx_hash']))
-                    continue
-            elif tx_dict['recipient'] == address and not include_recipient:
-                logg.debug('skipping recipient tx {}'.format(tx_dict['tx_hash']))
-                continue
-
-            o = {
-                'nonce': tx_dict['nonce'], 
-                'tx_hash': tx_dict['tx_hash'],
-                'status': tx_dict['status'],
-                'date_updated': tx_dict['date_updated'],
-                'errors': errors,
-                    }
-            if renderer != None:
-                r = renderer(o)
-                w.write(r + '\n')
-            else:
-                tx_dict_list.append(o)
-
-        return tx_dict_list
-
-
-    # TODO: Add exception upon non-existent tx aswell as invalid tx data to docstring 
-    # TODO: This method is WAY too long
-    def tx(self, chain_spec, tx_hash=None, tx_raw=None, registry=None, renderer=None, w=sys.stdout):
-        """Output local and network details about a given transaction with local origin.
-
-        If the transaction hash is given, the raw trasnaction data will be retrieved from the local transaction queue backend. Otherwise the raw transaction data must be provided directly. Only one of transaction hash and transaction data can be passed.
-
-        :param chain_spec: Chain spec of the transaction's chain context 
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :param tx_hash: Transaction hash of transaction to parse and view
-        :type tx_hash: str, 0x-hex
-        :param tx_raw: Signed raw transaction data to parse and view
-        :type tx_raw: str, 0x-hex
-        :raises ValueError: Both tx_hash and tx_raw are passed
-        :return: Transaction details
-        :rtype: dict
-        """
-        problems = []
-
-        if tx_hash != None and tx_raw != None:
-            ValueError('Specify only one of hash or raw tx')
-
-        if tx_raw != None:
-            tx_hash = add_0x(keccak256_hex_to_hex(tx_raw))
-
-        s = celery.signature(
-            'cic_eth.queue.query.get_tx_cache',
-            [
-                chain_spec.asdict(),
-                tx_hash,
-                ],
-            queue=self.queue,
-            )
-    
-        t = s.apply_async()
-        tx = t.get()
-
-        source_token = None
-        if tx['source_token'] != ZERO_ADDRESS:
-            source_token_declaration = None
-            if registry != None:
-                try:
-                    source_token_declaration = registry.by_address(tx['source_token'], sender_address=self.call_address)
-                except UnknownContractError:
-                    logg.warning('unknown source token contract {} (direct)'.format(tx['source_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['source_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                source_token_declaration = t.get()
-
-            if source_token_declaration != None:
-                logg.warning('found declarator record for source token {} but not checking validity'.format(tx['source_token']))
-                source_token = ERC20Token(chain_spec, self.rpc, tx['source_token'])
-                logg.debug('source token set tup {}'.format(source_token))
-
-
-
-        destination_token = None
-        if tx['destination_token'] != ZERO_ADDRESS:
-            destination_token_declaration = None
-            if registry != None:
-                try:
-                    destination_token_declaration = registry.by_address(tx['destination_token'], sender_address=self.call_address)
-                except UnknownContractError:
-                    logg.warning('unknown destination token contract {}'.format(tx['destination_token']))
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['destination_token'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                destination_token_declaration = t.get()
-            if destination_token_declaration != None:
-                logg.warning('found declarator record for destination token {} but not checking validity'.format(tx['destination_token']))
-                destination_token = ERC20Token(chain_spec, self.rpc, tx['destination_token'])
-
-        tx['sender_description'] = 'Custodial account'
-        tx['recipient_description'] = 'Custodial account'
-
-        o = code(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
-        if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    sender_contract = registry.by_address(tx['sender'], sender_address=self.call_address)
-                    tx['sender_description'] = 'Contract at {}'.format(tx['sender'])
-                except UnknownContractError:
-                    tx['sender_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['sender_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['sender'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['sender_description'] = t.get()
-                if tx['sender_description'] == None:
-                    tx['sender_description'] = 'Unknown contract'
-
-
-        else:
-            s = celery.signature(
-                    'cic_eth.eth.account.have',
-                    [
-                        tx['sender'],
-                        chain_spec.asdict(),
-                        ],
-                    queue=self.queue,
-                    )
-            t = s.apply_async()
-            account = t.get()
-            if account == None:
-                tx['sender_description'] = 'Unknown account'
-            else:
-                s = celery.signature(
-                    'cic_eth.eth.account.role',
-                    [
-                        tx['sender'],
-                        chain_spec.asdict(),
-                        ],
-                    queue=self.queue,
-                    )
-                t = s.apply_async()
-                role = t.get()
-                if role != None:
-                    tx['sender_description'] = role
-
-        o = code(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
-        if len(strip_0x(r, allow_empty=True)) > 0:
-            if registry != None:
-                try:
-                    recipient_contract = registry.by_address(tx['recipient'])
-                    tx['recipient_description'] = 'Contract at {}'.format(tx['recipient'])
-                except UnknownContractError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-                except KeyError as e:
-                    tx['recipient_description'] = 'Unknown contract'
-            else:
-                s = celery.signature(
-                        'cic_eth.task.registry_address_lookup',
-                        [
-                            chain_spec.asdict(),
-                            tx['recipient'],
-                            ],
-                        queue=self.queue
-                        )
-                t = s.apply_async()
-                tx['recipient_description'] = t.get()
-                if tx['recipient_description'] == None:
-                    tx['recipient_description'] = 'Unknown contract'
-
-        else:
-            s = celery.signature(
-                    'cic_eth.eth.account.have',
-                    [
-                        tx['recipient'],
-                        chain_spec.asdict(),
-                        ],
-                    queue=self.queue,
-                    )
-            t = s.apply_async()
-            account = t.get()
-            if account == None:
-                tx['recipient_description'] = 'Unknown account'
-            else:
-                s = celery.signature(
-                    'cic_eth.eth.account.role',
-                    [
-                        tx['recipient'],
-                        chain_spec.asdict(),
-                        ],
-                    queue=self.queue,
-                    )
-                t = s.apply_async()
-                role = t.get()
-                if role != None:
-                    tx['recipient_description'] = role
-
-        erc20_c = ERC20(chain_spec)
-        if source_token != None:
-            tx['source_token_symbol'] = source_token.symbol
-            o = erc20_c.balance_of(tx['source_token'], tx['sender'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['sender_token_balance'] = erc20_c.parse_balance(r)
-
-        if destination_token != None:
-            tx['destination_token_symbol'] = destination_token.symbol
-            o = erc20_c.balance_of(tx['destination_token'], tx['recipient'], sender_address=self.call_address)
-            r = self.rpc.do(o)
-            tx['recipient_token_balance'] = erc20_c.parse_balance(r)
-            #tx['recipient_token_balance'] = destination_token.function('balanceOf')(tx['recipient']).call()
-
-        # TODO: this can mean either not subitted or culled, need to check other txs with same nonce to determine which
-        tx['network_status'] = 'Not in node' 
-
-        r = None
-        try:
-            o = transaction(tx_hash)
-            t = self.proxy_do(chain_spec, o)
-            r = t.get()
-            if r != None:
-                tx['network_status'] = 'Mempool'
-        except Exception as e:
-            logg.warning('(too permissive exception handler, please fix!) {}'.format(e))
-
-        if r != None:
-            try:
-                o = receipt(tx_hash)
-                t = self.proxy_do(chain_spec, o)
-                r = t.get()
-                logg.debug('h {} o {}'.format(tx_hash, o))
-                if int(strip_0x(r['status'])) == 1:
-                    tx['network_status'] = 'Confirmed'
-                else:
-                    tx['network_status'] = 'Reverted'
-                tx['network_block_number'] = r.blockNumber
-                tx['network_tx_index'] = r.transactionIndex
-                if tx['block_number'] == None:
-                    problems.append('Queue is missing block number {} for mined tx'.format(r.blockNumber))
-            except Exception as e:
-                logg.warning('too permissive exception handler, please fix!')
-                pass
-
-        o = balance(tx['sender'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
-        tx['sender_gas_balance'] = r
-
-        o = balance(tx['recipient'])
-        t = self.proxy_do(chain_spec, o)
-        r = t.get()
-        tx['recipient_gas_balance'] = r
-
-        tx_unpacked = unpack(bytes.fromhex(strip_0x(tx['signed_tx'])), chain_spec)
-        tx['gas_price'] = tx_unpacked['gasPrice']
-        tx['gas_limit'] = tx_unpacked['gas']
-        tx['data'] = tx_unpacked['data']
-
-        s = celery.signature(
-            'cic_eth.queue.state.get_state_log',
-            [
-                chain_spec.asdict(),
-                tx_hash,
-                ],
-            queue=self.queue,
-            )
-        t = s.apply_async()
-        tx['status_log'] = t.get()
-
-        if len(problems) > 0:
-            sys.stderr.write('\n')
-            for p in problems:
-                sys.stderr.write('!!!{}\n'.format(p))
-
-        if renderer == None:
-            return tx
-
-        r = renderer(tx)
-        w.write(r + '\n')
-        return None