Upgrade transfer request auth contract, remove stale code in cic-eth

apps/cic-eth/cic_eth/api/api_admin.py

@@ -267,9 +267,7 @@ class AdminApi:
         """
         s = celery.signature(
                 'cic_eth.queue.tx.get_account_tx',
-                [
-                    address,
-                    ],
+                [address],
                 queue=self.queue,
                 )
         txs = s.apply_async().get()

apps/cic-eth/cic_eth/api/api_task.py

@@ -30,7 +30,7 @@ class Api:
     :param queue: Name of worker queue to submit tasks to
     :type queue: str
     """
-    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop.noop', callback_queue=None):
+    def __init__(self, chain_str, queue='cic-eth', callback_param=None, callback_task='cic_eth.callbacks.noop', callback_queue=None):
         self.chain_str = chain_str
         self.chain_spec = ChainSpec.from_chain_str(chain_str)
         self.callback_param = callback_param
@@ -301,15 +301,13 @@ class Api:
         return t
 
 
-    def balance(self, address, token_symbol, include_pending=True):
+    def balance(self, address, token_symbol):
         """Calls the provided callback with the current token balance of the given address.
 
         :param address: Ethereum address of holder
         :type address: str, 0x-hex
         :param token_symbol: ERC20 token symbol of token to send
         :type token_symbol: str
-        :param include_pending: If set, will include transactions that have not yet been fully processed
-        :type include_pending: bool
         :returns: uuid of root task
         :rtype: celery.Task
         """
@@ -332,45 +330,14 @@ class Api:
                     ],
                 queue=self.queue,
                 )
-        s_result = celery.signature(
-                'cic_eth.queue.balance.assemble_balances',
-                [],
-                queue=self.queue,
-                )
-
-        last_in_chain = s_balance
-        if include_pending:
-            s_balance_incoming = celery.signature(
-                    'cic_eth.queue.balance.balance_incoming',
-                    [
-                        address,
-                        self.chain_str,
-                        ],
-                    queue=self.queue,
-                    )
-            s_balance_outgoing = celery.signature(
-                    'cic_eth.queue.balance.balance_outgoing',
-                    [
-                        address,
-                        self.chain_str,
-                        ],
-                    queue=self.queue,
-                    )
-            s_balance.link(s_balance_incoming)
-            s_balance_incoming.link(s_balance_outgoing)
-            last_in_chain = s_balance_outgoing
-
-        one = celery.chain(s_tokens, s_balance)
-        two = celery.chain(s_tokens, s_balance_incoming)
-        three = celery.chain(s_tokens, s_balance_outgoing)
-
-        t = None
+        
         if self.callback_param != None:
-            s_result.link(self.callback_success).on_error(self.callback_error)
-            t = celery.chord([one, two, three])(s_result)
+            s_balance.link(self.callback_success)
+            s_tokens.link(s_balance).on_error(self.callback_error)
         else:
-            t = celery.chord([one, two, three])(s_result)
-    
+            s_tokens.link(s_balance)
+
+        t = s_tokens.apply_async(queue=self.queue)
         return t
 
 
@@ -450,73 +417,6 @@ class Api:
         return t
 
 
-    def list(self, address, limit=10, external_task=None, external_queue=None):
-        """Retrieve an aggregate list of latest transactions of internal and (optionally) external origin in reverse chronological order.
-
-        The array of transactions returned have the same dict layout as those passed by the callback filter in cic_eth/runnable/manager
-
-        If the external task is defined, this task will be used to query external transactions. If this is not defined, no external transactions will be included. The task must accept (offset, limit, address) as input parameters, and return a bloom filter that will be used to retrieve transaction data for the matching transactions. See cic_eth.ext.tx.list_tx_by_bloom for details on the bloom filter dat format.
-
-        :param address: Ethereum address to list transactions for
-        :type address: str, 0x-hex
-        :param limit: Amount of results to return
-        :type limit: number
-        :param external_task: Celery task providing external transactions
-        :type external_task: str
-        :param external_queue: Celery task queue providing exernal transactions task
-        :type external_queue: str
-        :returns: List of transactions
-        :rtype: list of dict
-        """
-        offset = 0
-        s_local = celery.signature(
-            'cic_eth.queue.tx.get_account_tx',
-            [
-                address,
-                ],
-            queue=self.queue,
-            )
-
-        s_brief = celery.signature(
-            'cic_eth.ext.tx.tx_collate',
-            [
-                self.chain_str,
-                offset,
-                limit
-                ],
-            queue=self.queue,
-            )
-        if self.callback_param != None:
-            s_assemble.link(self.callback_success).on_error(self.callback_error)
-
-        t = None
-        if external_task != None:
-            s_external_get = celery.signature(
-                external_task,
-                [
-                    address,
-                    offset,
-                    limit,
-                    ],
-                queue=external_queue,
-                )
-
-            s_external_process = celery.signature(
-                'cic_eth.ext.tx.list_tx_by_bloom',
-                [
-                    address,
-                    self.chain_str,
-                    ],
-                queue=self.queue,
-                    )
-            c = celery.chain(s_external_get, s_external_process)
-            t = celery.chord([s_local, c])(s_brief)
-        else:
-            t = s_local.apply_sync()
-
-        return t
-
-
     def ping(self, r):
         """A noop callback ping for testing purposes.
 

apps/cic-eth/cic_eth/callbacks/noop.py

@@ -18,4 +18,4 @@ def noop(self, result, param, status_code):
     :rtype: bool
     """
     logg.info('noop callback {} {} {}'.format(result, param, status_code))
-    return result
+    return True

apps/cic-eth/cic_eth/db/enum.py

@@ -4,23 +4,20 @@ import enum
 
 @enum.unique
 class StatusBits(enum.IntEnum):
-    """Individual bit flags that are combined to define the state and legacy of a queued transaction
-
-    """
-    QUEUED = 0x01 # transaction should be sent to network
-    IN_NETWORK = 0x08 # transaction is in network
+    QUEUED = 0x01
+    IN_NETWORK = 0x08
    
-    DEFERRED = 0x10 # an attempt to send the transaction to network has failed
-    GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
+    DEFERRED = 0x10
+    GAS_ISSUES = 0x20
 
-    LOCAL_ERROR = 0x100 # errors that originate internally from the component
-    NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
-    NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
-    UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
+    LOCAL_ERROR = 0x100
+    NODE_ERROR = 0x200
+    NETWORK_ERROR = 0x400
+    UNKNOWN_ERROR = 0x800
 
-    FINAL = 0x1000 # transaction processing has completed
-    OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
-    MANUAL = 0x8000 # transaction processing has been manually overridden
+    FINAL = 0x1000
+    OBSOLETE = 0x2000
+    MANUAL = 0x8000
 
 
 @enum.unique
@@ -82,19 +79,6 @@ class LockEnum(enum.IntEnum):
 
 
 def status_str(v, bits_only=False):
-    """Render a human-readable string describing the status
-
-    If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
-
-    If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
-
-    :param v: Status bit field
-    :type v: number
-    :param bits_only: Only render individual bit labels.
-    :type bits_only: bool
-    :returns: Status string
-    :rtype: str
-    """
     s = ''
     if not bits_only:
         try:
@@ -116,39 +100,14 @@ def status_str(v, bits_only=False):
 
 
 def all_errors():
-    """Bit mask of all error states
-
-    :returns: Error flags
-    :rtype: number
-    """
     return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
 
 
 def is_error_status(v):
-    """Check if value is an error state
-
-    :param v: Status bit field
-    :type v: number
-    :returns: True if error
-    :rtype: bool
-    """
     return bool(v & all_errors())
 
 
-def dead():
-    """Bit mask defining whether a transaction is still likely to be processed on the network.
-
-    :returns: Bit mask
-    :rtype: number
-    """
-    return StatusBits.FINAL | StatusBits.OBSOLETE
-
-
 def is_alive(v):
-    """Check if transaction is still likely to be processed on the network.
+    return bool(v & (StatusBits.FINAL | StatusBits.OBSOLETE) == 0)
 
-    The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
 
-    :returns: 
-    """
-    return bool(v & dead() == 0)

apps/cic-eth/cic_eth/db/migrations/default/versions/e3b5330ee71c_.py

@@ -27,8 +27,8 @@ def upgrade():
             sa.Column('destination_token_address', sa.String(42), nullable=False),
             sa.Column('sender', sa.String(42), nullable=False),
             sa.Column('recipient', sa.String(42), nullable=False),
-            sa.Column('from_value', sa.NUMERIC(), nullable=False),
-            sa.Column('to_value', sa.NUMERIC(), nullable=True),
+            sa.Column('from_value', sa.String(), nullable=False),
+            sa.Column('to_value', sa.String(), nullable=True),
             sa.Column('block_number', sa.BIGINT(), nullable=True),
             sa.Column('tx_index', sa.Integer, nullable=True),
             )

apps/cic-eth/cic_eth/db/migrations/postgresql/versions/e3b5330ee71c_.py

@@ -19,6 +19,7 @@ def upgrade():
     op.create_table(
             'tx_cache',
             sa.Column('id', sa.Integer, primary_key=True),
+#            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=True),
             sa.Column('otx_id', sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
             sa.Column('date_created', sa.DateTime, nullable=False),
             sa.Column('date_updated', sa.DateTime, nullable=False),
@@ -26,8 +27,8 @@ def upgrade():
             sa.Column('destination_token_address', sa.String(42), nullable=False),
             sa.Column('sender', sa.String(42), nullable=False),
             sa.Column('recipient', sa.String(42), nullable=False),
-            sa.Column('from_value', sa.NUMERIC(), nullable=False),
-            sa.Column('to_value', sa.NUMERIC(), nullable=True),
+            sa.Column('from_value', sa.BIGINT(), nullable=False),
+            sa.Column('to_value', sa.BIGINT(), nullable=True),
             sa.Column('block_number', sa.BIGINT(), nullable=True),
             sa.Column('tx_index', sa.Integer, nullable=True),
             )

apps/cic-eth/cic_eth/db/models/base.py

@@ -6,11 +6,6 @@ from sqlalchemy import Column, Integer
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import (
-        StaticPool,
-        QueuePool,
-        AssertionPool,
-        )
 
 logg = logging.getLogger()
 
@@ -54,7 +49,7 @@ class SessionBase(Model):
 
 
     @staticmethod
-    def connect(dsn, pool_size=8, debug=False):
+    def connect(dsn, debug=False):
         """Create new database connection engine and connect to database backend.
 
         :param dsn: DSN string defining connection.
@@ -62,28 +57,14 @@ class SessionBase(Model):
         """
         e = None
         if SessionBase.poolable:
-            poolclass = QueuePool
-            if pool_size > 1:
-                e = create_engine(
-                        dsn,
-                        max_overflow=pool_size*3,
-                        pool_pre_ping=True,
-                        pool_size=pool_size,
-                        pool_recycle=60,
-                        poolclass=poolclass,
-                        echo=debug,
-                    )
-            else:
-                if debug:
-                    poolclass = AssertionPool
-                else:
-                    poolclass = StaticPool
-
-                e = create_engine(
-                        dsn,
-                        poolclass=poolclass,
-                        echo=debug,
-                    )
+            e = create_engine(
+                    dsn,
+                    max_overflow=50,
+                    pool_pre_ping=True,
+                    pool_size=20,
+                    pool_recycle=10,
+                    echo=debug,
+                )
         else:
             e = create_engine(
                     dsn,

apps/cic-eth/cic_eth/db/models/otx.py

@@ -287,6 +287,7 @@ class Otx(SessionBase):
 
         self.__set_status(StatusBits.IN_NETWORK, session)
         self.__reset_status(StatusBits.DEFERRED | StatusBits.QUEUED | StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR, session)
+        logg.debug('<<< status {}'.format(status_str(self.status)))
 
         if self.tracing:
             self.__state_log(session=session)
@@ -421,7 +422,7 @@ class Otx(SessionBase):
 
 
     @staticmethod
-    def get(status=0, limit=4096, status_exact=True, session=None):
+    def get(status=0, limit=4096, status_exact=True):
         """Returns outgoing transaction lists by status.
 
         Status may either be matched exactly, or be an upper bound of the integer value of the status enum.
@@ -436,32 +437,26 @@ class Otx(SessionBase):
         :rtype: tuple, where first element is transaction hash
         """
         e = None
-
-        session = SessionBase.bind_session(session)
-
+        session = Otx.create_session()
         if status_exact:
             e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all()
         else:
             e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all()
-        
-        SessionBase.release_session(session)
+        session.close()
         return e
 
 
     @staticmethod
-    def load(tx_hash, session=None):
+    def load(tx_hash):
         """Retrieves the outgoing transaction record by transaction hash.
 
         :param tx_hash: Transaction hash
         :type tx_hash: str, 0x-hex
         """
-        session = SessionBase.bind_session(session)
-        
+        session = Otx.create_session()
         q = session.query(Otx)
         q = q.filter(Otx.tx_hash==tx_hash)
-
-        SessionBase.release_session(session)
-
+        session.close()
         return q.first()
 
 

apps/cic-eth/cic_eth/db/models/tx.py

@@ -2,7 +2,7 @@
 import datetime
 
 # third-party imports
-from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean, NUMERIC
+from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean
 from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
 #from sqlalchemy.orm import relationship, backref
 #from sqlalchemy.ext.declarative import declarative_base
@@ -55,8 +55,8 @@ class TxCache(SessionBase):
     destination_token_address = Column(String(42))
     sender = Column(String(42))
     recipient = Column(String(42))
-    from_value = Column(NUMERIC())
-    to_value = Column(NUMERIC())
+    from_value = Column(String())
+    to_value = Column(String())
     block_number = Column(Integer())
     tx_index = Column(Integer())
     date_created = Column(DateTime, default=datetime.datetime.utcnow)
@@ -64,6 +64,16 @@ class TxCache(SessionBase):
     date_checked = Column(DateTime, default=datetime.datetime.utcnow)
 
 
+    def values(self):
+        from_value_hex = bytes.fromhex(self.from_value)
+        from_value = int.from_bytes(from_value_hex, 'big')
+
+        to_value_hex = bytes.fromhex(self.to_value)
+        to_value = int.from_bytes(to_value_hex, 'big')
+
+        return (from_value, to_value)
+
+
     def check(self):
         """Update the "checked" timestamp to current time.
 
@@ -85,18 +95,18 @@ class TxCache(SessionBase):
         :param tx_hash_new: tx hash to associate the copied entry with
         :type tx_hash_new: str, 0x-hex
         """
-        localsession = SessionBase.bind_session(session)
-        
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
         q = localsession.query(TxCache)
         q = q.join(Otx)
         q = q.filter(Otx.tx_hash==tx_hash_original)
         txc = q.first()
 
         if txc == None:
-            SessionBase.release_session(localsession)
             raise NotLocalTxError('original {}'.format(tx_hash_original))
         if txc.block_number != None:
-            SessionBase.release_session(localsession)
             raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
 
         q = localsession.query(Otx)
@@ -104,38 +114,44 @@ class TxCache(SessionBase):
         otx = q.first()
 
         if otx == None:
-            SessionBase.release_session(localsession)
             raise NotLocalTxError('new {}'.format(tx_hash_new))
 
+        values = txc.values()
         txc_new = TxCache(
                 otx.tx_hash,
                 txc.sender,
                 txc.recipient,
                 txc.source_token_address,
                 txc.destination_token_address,
-                int(txc.from_value),
-                int(txc.to_value),
+                values[0],
+                values[1],
                 )
         localsession.add(txc_new)
         localsession.commit()
 
-        SessionBase.release_session(localsession)
+        if session == None:
+            localsession.close()
 
 
-    def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None, session=None):
-        localsession = SessionBase.bind_session(session)
-        tx = localsession.query(Otx).filter(Otx.tx_hash==tx_hash).first()
+    def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None):
+        session = SessionBase.create_session()
+        tx = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
         if tx == None:
-            SessionBase.release_session(localsession)
+            session.close()
             raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
         self.otx_id = tx.id
 
+#        if tx == None:
+#            session.close()
+#            raise ValueError('tx hash {} (outgoing: {}) not found'.format(tx_hash, outgoing))
+#        session.close()
+
         self.sender = sender
         self.recipient = recipient
         self.source_token_address = source_token_address
         self.destination_token_address = destination_token_address
-        self.from_value = from_value
-        self.to_value = to_value
+        self.from_value = num_serialize(from_value).hex()
+        self.to_value = num_serialize(to_value).hex()
         self.block_number = block_number
         self.tx_index = tx_index
         # not automatically set in sqlite, it seems:
@@ -143,5 +159,4 @@ class TxCache(SessionBase):
         self.date_updated = self.date_created
         self.date_checked = self.date_created
 
-        SessionBase.release_session(localsession)
 

apps/cic-eth/cic_eth/eth/account.py

@@ -304,8 +304,6 @@ def cache_gift_data(
     tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
     tx_data = unpack_gift(tx['data'])
 
-    session = SessionBase.create_session()
-
     tx_cache = TxCache(
         tx_hash_hex,
         tx['from'],
@@ -314,9 +312,9 @@ def cache_gift_data(
         zero_address,
         0,
         0,
-        session=session,
             )
 
+    session = SessionBase.create_session()
     session.add(tx_cache)
     session.commit()
     cache_id = tx_cache.id
@@ -349,7 +347,6 @@ def cache_account_data(
     tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
     tx_data = unpack_register(tx['data'])
 
-    session = SessionBase.create_session()
     tx_cache = TxCache(
         tx_hash_hex,
         tx['from'],
@@ -358,8 +355,9 @@ def cache_account_data(
         zero_address,
         0,
         0,
-        session=session,
             )
+
+    session = SessionBase.create_session()
     session.add(tx_cache)
     session.commit()
     cache_id = tx_cache.id

apps/cic-eth/cic_eth/eth/request.py

@@ -1,194 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import web3
-import celery
-from erc20_approval_escrow import TransferApproval
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.models.base import SessionBase
-from cic_eth.eth import RpcClient
-from cic_eth.eth.factory import TxFactory
-from cic_eth.eth.task import sign_and_register_tx
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.error import TokenCountError
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-contract_function_signatures = {
-        'request': 'b0addede',
-        }
-
-
-class TransferRequestTxFactory(TxFactory):
-    """Factory for creating Transfer request transactions using the TransferApproval contract backend
-    """
-    def request(
-            self,
-            token_address,
-            beneficiary_address,
-            amount,
-            chain_spec,
-            ):
-        """Create a new TransferApproval.request transaction
-
-        :param token_address: Token to create transfer request for
-        :type token_address: str, 0x-hex
-        :param beneficiary_address: Beneficiary of token transfer
-        :type beneficiary_address: str, 0x-hex
-        :param amount: Amount of tokens to transfer
-        :type amount: number
-        :param chain_spec: Chain spec
-        :type chain_spec: cic_registry.chain.ChainSpec
-        :returns: Transaction in standard Ethereum format
-        :rtype: dict
-        """
-        transfer_approval = CICRegistry.get_contract(chain_spec, 'TransferApproval', 'TransferAuthorization')
-        fn = transfer_approval.function('createRequest')
-        tx_approval_buildable = fn(beneficiary_address, token_address, amount)
-        transfer_approval_gas = transfer_approval.gas('createRequest')
-
-        tx_approval = tx_approval_buildable.buildTransaction({
-            'from': self.address,
-            'gas': transfer_approval_gas,
-            'gasPrice': self.gas_price,
-            'chainId': chain_spec.chain_id(),
-            'nonce': self.next_nonce(),
-            })
-        return tx_approval
-
-
-def unpack_transfer_approval_request(data):
-    """Verifies that a transaction is an "TransferApproval.request" transaction, and extracts call parameters from it.
-
-    :param data: Raw input data from Ethereum transaction.
-    :type data: str, 0x-hex
-    :raises ValueError: Function signature does not match AccountRegister.add
-    :returns: Parsed parameters
-    :rtype: dict
-    """
-    f = data[2:10]
-    if f != contract_function_signatures['request']:
-        raise ValueError('Invalid transfer request data ({})'.format(f))
-
-    d = data[10:]
-    return {
-        'to': web3.Web3.toChecksumAddress('0x' + d[64-40:64]),
-        'token': web3.Web3.toChecksumAddress('0x' + d[128-40:128]),
-        'amount': int(d[128:], 16)
-        }
-
-
-@celery_app.task(bind=True)
-def transfer_approval_request(self, tokens, holder_address, receiver_address, value, chain_str):
-    """Creates a new transfer approval
-
-    :param tokens: Token to generate transfer request for
-    :type tokens: list with single token spec as dict
-    :param holder_address: Address to generate transfer on behalf of
-    :type holder_address: str, 0x-hex
-    :param receiver_address: Address to transfser tokens to
-    :type receiver_address: str, 0x-hex
-    :param value: Amount of tokens to transfer
-    :type value: number
-    :param chain_spec: Chain spec string representation
-    :type chain_spec: str
-    :raises cic_eth.error.TokenCountError: More than one token in tokens argument
-    :returns: Raw signed transaction
-    :rtype: list with transaction as only element
-    """
-
-    if len(tokens) != 1:
-        raise TokenCountError
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    queue = self.request.delivery_info['routing_key']
-
-    t = tokens[0]
-
-    c = RpcClient(holder_address)
-
-    txf = TransferRequestTxFactory(holder_address, c)
-
-    tx_transfer = txf.request(t['address'], receiver_address, value, chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx_transfer, chain_str, queue, 'cic_eth.eth.request.otx_cache_transfer_approval_request')
-
-    gas_budget = tx_transfer['gas'] * tx_transfer['gasPrice']
-
-    s = create_check_gas_and_send_task(
-             [tx_signed_raw_hex],
-             chain_str,
-             holder_address,
-             gas_budget,
-             [tx_hash_hex],
-             queue,
-            )
-    s.apply_async()
-    return [tx_signed_raw_hex]
-
-
-@celery_app.task()
-def otx_cache_transfer_approval_request(
-        tx_hash_hex,
-        tx_signed_raw_hex,
-        chain_str,
-        ):
-    """Generates and commits transaction cache metadata for an TransferApproval.request transaction
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx_signed_raw_hex: Raw signed transaction
-    :type tx_signed_raw_hex: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
-    tx = unpack_signed_raw_tx(tx_signed_raw_bytes, chain_spec.chain_id())
-    logg.debug('in otx acche transfer approval request')
-    (txc, cache_id) = cache_transfer_approval_request_data(tx_hash_hex, tx)
-    return txc
-
-
-@celery_app.task()
-def cache_transfer_approval_request_data(
-    tx_hash_hex,
-    tx,
-        ):
-    """Helper function for otx_cache_transfer_approval_request
-
-    :param tx_hash_hex: Transaction hash
-    :type tx_hash_hex: str, 0x-hex
-    :param tx: Signed raw transaction
-    :type tx: str, 0x-hex
-    :returns: Transaction hash and id of cache element in storage backend, respectively
-    :rtype: tuple
-    """
-    tx_data = unpack_transfer_approval_request(tx['data'])
-    logg.debug('tx approval request data {}'.format(tx_data))
-    logg.debug('tx approval request {}'.format(tx))
-
-    session = SessionBase.create_session()
-    tx_cache = TxCache(
-        tx_hash_hex,
-        tx['from'],
-        tx_data['to'],
-        tx_data['token'],
-        tx_data['token'],
-        tx_data['amount'],
-        tx_data['amount'],
-            )
-    session.add(tx_cache)
-    session.commit()
-    cache_id = tx_cache.id
-    session.close()
-    return (tx_hash_hex, cache_id)

apps/cic-eth/cic_eth/eth/token.py

@@ -6,7 +6,6 @@ import celery
 import requests
 import web3
 from cic_registry import CICRegistry
-from cic_registry import zero_address
 from cic_registry.chain import ChainSpec
 
 # platform imports
@@ -18,7 +17,6 @@ from cic_eth.eth.task import sign_and_register_tx
 from cic_eth.eth.task import create_check_gas_and_send_task
 from cic_eth.eth.factory import TxFactory
 from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.ext.address import translate_address
 
 celery_app = celery.current_app
 logg = logging.getLogger()
@@ -187,6 +185,7 @@ def balance(tokens, holder_address, chain_str):
     """
     #abi = ContractRegistry.abi('ERC20Token')
     chain_spec = ChainSpec.from_chain_str(chain_str)
+    balances = []
     c = RpcClient(chain_spec)
     for t in tokens:
         #token = CICRegistry.get_address(t['address'])
@@ -194,9 +193,9 @@ def balance(tokens, holder_address, chain_str):
         #o = c.w3.eth.contract(abi=abi, address=t['address'])
         o = CICRegistry.get_address(chain_spec, t['address']).contract
         b = o.functions.balanceOf(holder_address).call()
-        t['balance_network'] = b
-
-    return tokens
+        logg.debug('balance {} for {}: {}'.format(t['address'], holder_address, b))
+        balances.append(b)
+    return b
 
 
 @celery_app.task(bind=True)
@@ -325,7 +324,7 @@ def resolve_tokens_by_symbol(token_symbols, chain_str):
         token = CICRegistry.get_token(chain_spec, token_symbol)
         tokens.append({
             'address': token.address(),
-            'converters': [],
+            #'converters': [],
             })
     return tokens
 
@@ -381,7 +380,6 @@ def cache_transfer_data(
         tx['to'],
         tx_data['amount'],
         tx_data['amount'],
-        session=session,
             )
     session.add(tx_cache)
     session.commit()
@@ -441,66 +439,9 @@ def cache_approve_data(
         tx['to'],
         tx_data['amount'],
         tx_data['amount'],
-        session=session,
             )
     session.add(tx_cache)
     session.commit()
     cache_id = tx_cache.id
     session.close()
     return (tx_hash_hex, cache_id)
-
-
-class ExtendedTx:
-
-    _default_decimals = 6
-
-    def __init__(self, tx_hash, chain_spec):
-        self._chain_spec = chain_spec
-        self.chain = str(chain_spec)
-        self.hash = tx_hash
-        self.sender = None
-        self.sender_label = None
-        self.recipient = None
-        self.recipient_label = None
-        self.source_token_value = 0
-        self.destination_token_value = 0
-        self.source_token = zero_address
-        self.destination_token = zero_address
-        self.source_token_symbol = ''
-        self.destination_token_symbol = ''
-        self.source_token_decimals = ExtendedTx._default_decimals
-        self.destination_token_decimals = ExtendedTx._default_decimals
-
-
-    def set_actors(self, sender, recipient, trusted_declarator_addresses=None):
-        self.sender = sender
-        self.recipient = recipient
-        if trusted_declarator_addresses != None:
-            self.sender_label = translate_address(sender, trusted_declarator_addresses, self.chain)
-            self.recipient_label = translate_address(recipient, trusted_declarator_addresses, self.chain)
-
-
-    def set_tokens(self, source, source_value, destination=None, destination_value=None):
-        if destination == None:
-            destination = source
-        if destination_value == None:
-            destination_value = source_value
-        st = CICRegistry.get_address(self._chain_spec, source)
-        dt = CICRegistry.get_address(self._chain_spec, destination)
-        self.source_token = source
-        self.source_token_symbol = st.symbol()
-        self.source_token_decimals = st.decimals()
-        self.source_token_value = source_value
-        self.destination_token = destination
-        self.destination_token_symbol = dt.symbol()
-        self.destination_token_decimals = dt.decimals()
-        self.destination_token_value = destination_value
-
-
-    def to_dict(self):
-        o = {}
-        for attr in dir(self):
-            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'to_dict']:
-                continue
-            o[attr] = getattr(self, attr)
-        return o 

apps/cic-eth/cic_eth/eth/tx.py

@@ -78,6 +78,7 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
 
     # TODO: it should not be necessary to pass address explicitly, if not passed should be derived from the tx
     balance = c.w3.eth.getBalance(address) 
+    logg.debug('check gas txs {}'.format(tx_hashes))
     logg.debug('address {} has gas {} needs {}'.format(address, balance, gas_required))
 
     if gas_required > balance:
@@ -125,6 +126,7 @@ def check_gas(self, tx_hashes, chain_str, txs=[], address=None, gas_required=Non
             queue=queue,
             )
         ready_tasks.append(s)
+    logg.debug('tasks {}'.format(ready_tasks))
     celery.group(ready_tasks)()
 
     return txs
@@ -141,6 +143,7 @@ def hashes_to_txs(self, tx_hashes):
     :returns: Signed raw transactions
     :rtype: list of str, 0x-hex
     """
+    #logg = celery_app.log.get_default_logger()
     if len(tx_hashes) == 0:
         raise ValueError('no transaction to send')
 
@@ -341,6 +344,7 @@ def send(self, txs, chain_str):
 
     chain_spec = ChainSpec.from_chain_str(chain_str)
 
+
     tx_hex = txs[0]
     logg.debug('send transaction {}'.format(tx_hex))
 
@@ -348,12 +352,15 @@ def send(self, txs, chain_str):
     tx_hash_hex = tx_hash.hex()
 
     queue = self.request.delivery_info.get('routing_key', None)
+    if queue == None:
+        logg.debug('send tx {} has no queue', tx_hash)
 
     c = RpcClient(chain_spec)
     r = None
     try:
         r = c.w3.eth.send_raw_transaction(tx_hex)
     except Exception as e:
+        logg.debug('e {}'.format(e))
         raiser = ParityNodeHandler(chain_spec, queue)
         (t, e, m) = raiser.handle(e, tx_hash_hex, tx_hex)
         raise e(m)
@@ -417,7 +424,7 @@ def refill_gas(self, recipient_address, chain_str):
     gas_price = c.gas_price()
     gas_limit = c.default_gas_limit
     refill_amount = c.refill_amount()
-    logg.debug('tx send gas price {} nonce {}'.format(gas_price, nonce))
+    logg.debug('gas price {} nonce {}'.format(gas_price, nonce))
 
     # create and sign transaction
     tx_send_gas = {
@@ -430,6 +437,7 @@ def refill_gas(self, recipient_address, chain_str):
                 'value': refill_amount,
                 'data': '',
             }
+    logg.debug('txsend_gas {}'.format(tx_send_gas))
     tx_send_gas_signed = c.w3.eth.sign_transaction(tx_send_gas)
     tx_hash = web3.Web3.keccak(hexstr=tx_send_gas_signed['raw'])
     tx_hash_hex = tx_hash.hex()
@@ -480,14 +488,11 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
     :rtype: str, 0x-hex
     """
     session = SessionBase.create_session()
-
-    
-    q = session.query(Otx)
-    q = q.filter(Otx.tx_hash==txold_hash_hex)
-    otx = q.first()
-    session.close()
+    otx = session.query(Otx).filter(Otx.tx_hash==txold_hash_hex).first()
     if otx == None:
+        session.close()
         raise NotLocalTxError(txold_hash_hex)
+    session.close()
 
     chain_spec = ChainSpec.from_chain_str(chain_str)
     c = RpcClient(chain_spec)
@@ -504,7 +509,7 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
     else:
         gas_price = c.gas_price()
         if tx['gasPrice'] > gas_price:
-            logg.info('Network gas price {} is lower than overdue tx gas price {}'.format(gas_price, tx['gasPrice']))
+            logg.warning('Network gas price {} is lower than overdue tx gas price {}'.format(gas_price, tx['gasPrice']))
             #tx['gasPrice'] = int(tx['gasPrice'] * default_factor)
             tx['gasPrice'] += 1
         else:
@@ -514,6 +519,9 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
             else:
                 tx['gasPrice'] = new_gas_price
 
+    logg.debug('after {}'.format(tx))
+
+    #(tx_hash_hex, tx_signed_raw_hex) = sign_and_register_tx(tx, chain_str, queue)
     (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, chain_str)
     queue_create(
         tx['nonce'],
@@ -533,7 +541,6 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_str, gas=None, default_fa
             queue=queue,
             )
     s.apply_async()
-
     return tx_hash_hex
 
 
@@ -596,9 +603,7 @@ def resume_tx(self, txpending_hash_hex, chain_str):
     chain_spec = ChainSpec.from_chain_str(chain_str)
 
     session = SessionBase.create_session()
-    q = session.query(Otx.signed_tx)
-    q = q.filter(Otx.tx_hash==txpending_hash_hex)
-    r = q.first()
+    r = session.query(Otx.signed_tx).filter(Otx.tx_hash==txpending_hash_hex).first()
     session.close()
     if r == None:
         raise NotLocalTxError(txpending_hash_hex)

apps/cic-eth/cic_eth/eth/util.py

@@ -104,5 +104,3 @@ def tx_hex_string(tx_hex, chain_id):
 
     tx_raw_bytes = bytes.fromhex(tx_hex)
     return tx_string(tx_raw_bytes, chain_id)
-
-

apps/cic-eth/cic_eth/ext/address.py

@@ -1,43 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from cic_registry.chain import ChainSpec
-from cic_registry import CICRegistry
-
-celery_app = celery.current_app
-
-logg = logging.getLogger()
-
-
-def translate_address(address, trusted_addresses, chain_spec):
-    for trusted_address in trusted_addresses:
-        o = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', 'Declarator')
-        fn = o.function('declaration')
-        declaration_hex = fn(trusted_address, address).call()
-        declaration_bytes = declaration_hex[0].rstrip(b'\x00')
-        declaration = None
-        try:
-            declaration = declaration_bytes.decode('utf-8', errors='strict')
-        except UnicodeDecodeError:
-            continue
-        return declaration
-
-
-@celery_app.task()
-def translate_tx_addresses(tx, trusted_addresses, chain_str):
-
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    declaration = None
-    if tx['sender_label'] == None:
-        declaration = translate_address(tx['sender'], trusted_addresses, chain_spec)
-    tx['sender_label'] = declaration
-
-    declaration = None
-    if tx['recipient_label'] == None:
-        declaration = translate_address(tx['recipient'], trusted_addresses, chain_spec)
-    tx['recipient_label'] = declaration
-
-    return tx

apps/cic-eth/cic_eth/ext/tx.py

@@ -1,176 +0,0 @@
-# standard imports
-import logging
-import math
-
-# third-pary imports
-import web3
-import celery
-import moolb
-from cic_registry.chain import ChainSpec
-from cic_registry.registry import CICRegistry
-from hexathon import strip_0x
-
-# local imports
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.db.models.otx import Otx
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.db.enum import StatusEnum
-from cic_eth.eth.token import unpack_transfer
-from cic_eth.queue.tx import get_tx_cache
-from cic_eth.queue.time import tx_times
-
-celery_app = celery.current_app
-logg = logging.getLogger()
-
-MAX_BLOCK_TX = 250
-
-
-# TODO: Make this method easier to read
-@celery_app.task()
-def list_tx_by_bloom(bloomspec, address, chain_str):
-    """Retrieve external transaction data matching the provided filter
-
-    The bloom filter representation with the following structure (the size of the filter will be inferred from the size of the provided filter data):
-        {
-            'alg': <str; hashing algorithm, currently only "sha256" is understood>,
-            'high': <number; highest block number in matched set>,
-            'low': <number; lowest block number in matched set>,
-            'filter_rounds': <number; hashing rounds used to generate filter entry>,
-            'block_filter': <hex; bloom filter data with block matches>,
-            'blocktx_filter': <hex; bloom filter data with block+tx matches>,
-        }
-
-    :param bloomspec: Bloom filter data
-    :type bloomspec: dict (see description above)
-    :param address: Recipient address to use in matching
-    :type address: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: dict of transaction data as dict, keyed by transaction hash
-    :rtype: dict of dict
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    block_filter_data = bytes.fromhex(bloomspec['block_filter'])
-    tx_filter_data = bytes.fromhex(bloomspec['blocktx_filter'])
-    databitlen = len(block_filter_data)*8
-    block_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=block_filter_data)
-    tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data)
-
-    txs = {}
-    for block_height in range(bloomspec['low'], bloomspec['high']):
-        block_height_bytes = block_height.to_bytes(4, 'big')
-        if block_filter.check(block_height_bytes):
-            logg.debug('filter matched block {}'.format(block_height))
-            block = c.w3.eth.getBlock(block_height, True)
-
-            for tx_index in range(0, len(block.transactions)):
-                composite = tx_index + block_height
-                tx_index_bytes = composite.to_bytes(4, 'big')
-                if tx_filter.check(tx_index_bytes):
-                    logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
-
-                    try:
-                        tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
-                    except web3.exceptions.TransactionNotFound:
-                        logg.debug('false positive on block {} tx {}'.format(block_height, tx_index))
-                        continue
-                    tx_address = None
-                    tx_token_value = 0
-                    try:
-                        transfer_data = unpack_transfer(tx['data'])
-                        tx_address = transfer_data['to']
-                        tx_token_value = transfer_data['amount']
-                    except ValueError:
-                        logg.debug('not a transfer transaction, skipping {}'.format(tx))
-                        continue
-                    if address == tx_address:
-                        status = StatusEnum.SENT
-                        try:
-                            rcpt = c.w3.eth.getTransactionReceipt(tx.hash)
-                            if rcpt['status'] == 0:
-                                pending = StatusEnum.REVERTED
-                            else:
-                                pending = StatusEnum.SUCCESS
-                        except web3.exceptions.TransactionNotFound:
-                            pass
-
-                        tx_hash_hex = tx['hash'].hex()
-
-                        token = CICRegistry.get_address(chain_spec, tx['to'])
-                        token_symbol = token.symbol()
-                        token_decimals = token.decimals()
-                        times = tx_times(tx_hash_hex, chain_str)
-                        tx_r = {
-                            'hash': tx_hash_hex,
-                            'sender': tx['from'],
-                            'recipient': tx_address,
-                            'source_value': tx_token_value,
-                            'destination_value': tx_token_value,
-                            'source_token': tx['to'],
-                            'destination_token': tx['to'],
-                            'source_token_symbol': token_symbol,
-                            'destination_token_symbol': token_symbol,
-                            'source_token_decimals': token_decimals,
-                            'destination_token_decimals': token_decimals,
-                            'source_token_chain': chain_str,
-                            'destination_token_chain': chain_str,
-                            'nonce': tx['nonce'],
-                                }
-                        if times['queue'] != None:
-                            tx_r['date_created'] = times['queue']
-                        else:
-                            tx_r['date_created'] = times['network']
-                        txs[tx_hash_hex] = tx_r
-                        break
-    return txs
-
-
-# TODO: Surely it must be possible to optimize this
-# TODO: DRY this with callback filter in cic_eth/runnable/manager
-# TODO: Remove redundant fields from end representation (timestamp, tx_hash)
-@celery_app.task()
-def tx_collate(tx_batches, chain_str, offset, limit, newest_first=True):
-    """Merges transaction data from multiple sources and sorts them in chronological order.
-
-    :param tx_batches: Transaction data inputs
-    :type tx_batches: lists of lists of transaction data
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :param offset: Number of sorted results to skip (not yet implemented)
-    :type offset: number
-    :param limit: Maximum number of results to return (not yet implemented)
-    :type limit: number
-    :param newest_first: If True, returns results in reverse chronological order
-    :type newest_first: bool
-    :returns: Transactions
-    :rtype: list
-    """
-    txs_by_block = {}
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-
-    for b in tx_batches:
-        for v in b.values():
-            tx = None
-            k = None
-            try:
-                hx = strip_0x(v)
-                tx = unpack_signed_raw_tx(bytes.fromhex(hx), chain_spec.chain_id())
-                txc = get_tx_cache(tx['hash'])
-                txc['timestamp'] = int(txc['date_created'].timestamp())
-                txc['hash'] = txc['tx_hash']
-                tx = txc
-            except TypeError:
-                tx = v
-                tx['timestamp'] = tx['date_created']
-            k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce'])
-            txs_by_block[k] = tx
-
-    txs = []
-    ks = list(txs_by_block.keys())
-    ks.sort()
-    if newest_first:
-        ks.reverse()
-    for k in ks:
-        txs.append(txs_by_block[k])
-    return txs

apps/cic-eth/cic_eth/queue/balance.py

@@ -1,120 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from hexathon import strip_0x
-
-# local imports
-from cic_registry.chain import ChainSpec
-from cic_eth.db import SessionBase
-from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.enum import (
-        StatusBits,
-        dead,
-        )
-
-celery_app = celery.current_app
-
-logg = logging.getLogger()
-
-
-def __balance_outgoing_compatible(token_address, holder_address, chain_str):
-    session = SessionBase.create_session()
-    q = session.query(TxCache.from_value)
-    q = q.join(Otx)
-    q = q.filter(TxCache.sender==holder_address)
-    status_compare = dead()
-    q = q.filter(Otx.status.op('&')(status_compare)==0)
-    q = q.filter(TxCache.source_token_address==token_address)
-    delta = 0
-    for r in q.all():
-        delta += int(r[0])
-    session.close()
-    return delta
-
-
-@celery_app.task()
-def balance_outgoing(tokens, holder_address, chain_str):
-    """Retrieve accumulated value of unprocessed transactions sent from the given address.
-
-    :param tokens: list of token spec dicts with addresses to retrieve balances for
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Sender address
-    :type holder_address: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Tokens dicts with outgoing balance added
-    :rtype: dict
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    for t in tokens: 
-        b = __balance_outgoing_compatible(t['address'], holder_address, chain_str)
-        t['balance_outgoing'] = b
-
-    return tokens
-
-
-def __balance_incoming_compatible(token_address, receiver_address, chain_str):
-    session = SessionBase.create_session()
-    q = session.query(TxCache.to_value)
-    q = q.join(Otx)
-    q = q.filter(TxCache.recipient==receiver_address)
-    status_compare = dead()
-    q = q.filter(Otx.status.op('&')(status_compare)==0)
-    # TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed. 
-    q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
-    q = q.filter(TxCache.destination_token_address==token_address)
-    delta = 0
-    for r in q.all():
-        delta += int(r[0])
-    session.close()
-    return delta
-
-
-@celery_app.task()
-def balance_incoming(tokens, receipient_address, chain_str):
-    """Retrieve accumulated value of unprocessed transactions to be received by the given address.
-
-    :param tokens: list of token spec dicts with addresses to retrieve balances for
-    :type tokens: list of str, 0x-hex
-    :param holder_address: Recipient address
-    :type holder_address: str, 0x-hex
-    :param chain_str: Chain spec string representation
-    :type chain_str: str
-    :returns: Tokens dicts with outgoing balance added
-    :rtype: dict
-    """
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    for t in tokens: 
-        b = __balance_incoming_compatible(t['address'], receipient_address, chain_str)
-        t['balance_incoming'] = b
-
-    return tokens
-
-
-@celery_app.task()
-def assemble_balances(balances_collection):
-    """Combines token spec dicts with individual balances into a single token spec dict.
-
-    A "balance" means any field that is keyed with a string starting with "balance_"
-
-    :param balances_collection: Token spec dicts
-    :type balances_collection: list of lists of dicts
-    :returns: Single token spec dict per token with all balances
-    :rtype: list of dicts
-    """
-    tokens = {}
-    for c in balances_collection:
-        for b in c:
-            address = b['address']
-            if tokens.get(address) == None:
-                tokens[address] = {
-                    'address': address,
-                    'converters': b['converters'],
-                        }
-            for k in b.keys():
-                if k[:8] == 'balance_':
-                    tokens[address][k] = b[k]
-    return list(tokens.values())

apps/cic-eth/cic_eth/queue/time.py

@@ -1,40 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import web3
-import celery
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.db.models.otx import Otx
-from cic_eth.error import NotLocalTxError
-
-celery_app = celery.current_app
-
-logg = logging.getLogger()
-
-
-# TODO: This method does not belong in the _queue_ module, it operates across queue and network
-@celery_app.task()
-def tx_times(tx_hash, chain_str):
-    chain_spec = ChainSpec.from_chain_str(chain_str)
-    c = RpcClient(chain_spec)
-    time_pair = {
-            'network': None,
-            'queue': None,
-            }
-    try:
-        rcpt = c.w3.eth.getTransactionReceipt(tx_hash)
-        block = c.w3.eth.getBlock(rcpt['blockHash'])
-        logg.debug('rcpt {}'.format(block))
-        time_pair['network'] = block['timestamp']
-    except web3.exceptions.TransactionNotFound:
-        pass
-
-    otx = Otx.load(tx_hash)
-    if otx != None:
-        time_pair['queue'] = int(otx['date_created'].timestamp())
-
-    return time_pair

apps/cic-eth/cic_eth/queue/tx.py

@@ -5,7 +5,6 @@ import datetime
 
 # third-party imports
 import celery
-from hexathon import strip_0x
 from sqlalchemy import or_
 from sqlalchemy import not_
 from sqlalchemy import tuple_
@@ -13,7 +12,6 @@ from sqlalchemy import func
 
 # local imports
 from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
 from cic_eth.db.models.otx import Otx
 from cic_eth.db.models.otx import OtxStateLog
 from cic_eth.db.models.tx import TxCache
@@ -24,7 +22,6 @@ from cic_eth.db.enum import (
         LockEnum,
         StatusBits,
         is_alive,
-        dead,
         )
 from cic_eth.eth.util import unpack_signed_raw_tx # TODO: should not be in same sub-path as package that imports queue.tx
 from cic_eth.error import NotLocalTxError
@@ -35,7 +32,8 @@ celery_app = celery.current_app
 logg = logging.getLogger()
 
 
-def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predecessors=True, session=None):
+@celery_app.task()
+def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predecessors=True):
     """Create a new transaction queue record.
 
     :param nonce: Transaction nonce
@@ -51,10 +49,10 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
     :returns: transaction hash
     :rtype: str, 0x-hash
     """
-    session = SessionBase.bind_session(session)
+    session = SessionBase.create_session()
     lock = Lock.check_aggregate(chain_str, LockEnum.QUEUE, holder_address, session=session) 
     if lock > 0:
-        SessionBase.release_session(session)
+        session.close()
         raise LockedError(lock)
 
     o = Otx.add(
@@ -80,7 +78,7 @@ def create(nonce, holder_address, tx_hash, signed_tx, chain_str, obsolete_predec
             otx.cancel(confirmed=False, session=session)
 
     session.commit()
-    SessionBase.release_session(session)
+    session.close()
     logg.debug('queue created nonce {} from {} hash {}'.format(nonce, holder_address, tx_hash))
     return tx_hash
 
@@ -99,9 +97,7 @@ def set_sent_status(tx_hash, fail=False):
     :rtype: boolean
     """
     session = SessionBase.create_session()
-    q = session.query(Otx)
-    q = q.filter(Otx.tx_hash==tx_hash)
-    o = q.first()
+    o = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
     if o == None:
         logg.warning('not local tx, skipping {}'.format(tx_hash))
         session.close()
@@ -382,6 +378,7 @@ def get_tx_cache(tx_hash):
 
     session.close()
 
+    values = txc.values()
     tx = {
         'tx_hash': otx.tx_hash,
         'signed_tx': otx.signed_tx,
@@ -390,12 +387,10 @@ def get_tx_cache(tx_hash):
         'status_code': otx.status,
         'source_token': txc.source_token_address,
         'destination_token': txc.destination_token_address,
-        'block_number': txc.block_number,
-        'tx_index': txc.tx_index,
         'sender': txc.sender,
         'recipient': txc.recipient,
-        'from_value': int(txc.from_value),
-        'to_value': int(txc.to_value),
+        'from_value': values[0],
+        'to_value': values[1],
         'date_created': txc.date_created,
         'date_updated': txc.date_updated,
         'date_checked': txc.date_checked,
@@ -455,7 +450,6 @@ def get_tx(tx_hash):
     session = SessionBase.create_session()
     tx = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
     if tx == None:
-        session.close()
         raise NotLocalTxError('queue does not contain tx hash {}'.format(tx_hash))
 
     o = {
@@ -500,7 +494,7 @@ def get_nonce_tx(nonce, sender, chain_id):
 
 
 # TODO: pass chain spec instead of chain id
-def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
+def get_paused_txs(status=None, sender=None, chain_id=0):
     """Returns not finalized transactions that have been attempted sent without success.
 
     :param status: If set, will return transactions with this local queue status only
@@ -513,13 +507,12 @@ def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
     :returns: Transactions
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
-    session = SessionBase.bind_session(session)
+    session = SessionBase.create_session()
     q = session.query(Otx)
 
     if status != None:
         #if status == StatusEnum.PENDING or status >= StatusEnum.SENT:
         if status == StatusEnum.PENDING or status & StatusBits.IN_NETWORK or not is_alive(status):
-            SessionBase.release_session(session)
             raise ValueError('not a valid paused tx value: {}'.format(status))
         q = q.filter(Otx.status.op('&')(status.value)==status.value)
         q = q.join(TxCache)
@@ -539,12 +532,12 @@ def get_paused_txs(status=None, sender=None, chain_id=0, session=None):
             #gas += tx['gas'] * tx['gasPrice']
             txs[r.tx_hash] = r.signed_tx
 
-    SessionBase.release_session(session)
+    session.close()
 
     return txs
 
 
-def get_status_tx(status, before=None, exact=False, limit=0, session=None):
+def get_status_tx(status, before=None, exact=False, limit=0):
     """Retrieve transaction with a specific queue status.
 
     :param status: Status to match transactions with
@@ -557,7 +550,7 @@ def get_status_tx(status, before=None, exact=False, limit=0, session=None):
     :rtype: list of cic_eth.db.models.otx.Otx
     """
     txs = {}
-    session = SessionBase.bind_session(session)
+    session = SessionBase.create_session()
     q = session.query(Otx)
     q = q.join(TxCache)
     q = q.filter(TxCache.date_updated<before)
@@ -571,12 +564,12 @@ def get_status_tx(status, before=None, exact=False, limit=0, session=None):
             break
         txs[o.tx_hash] = o.signed_tx
         i += 1
-    SessionBase.release_session(session)
+    session.close()
     return txs
 
 
 # TODO: move query to model
-def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, chain_id=0, session=None):
+def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, chain_id=0):
     """Returns the next pending transaction, specifically the transaction with the lowest nonce, for every recipient that has pending transactions.
 
     Will omit addresses that have the LockEnum.SEND bit in Lock set.
@@ -595,7 +588,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
     :returns: Transactions
     :rtype: dict, with transaction hash as key, signed raw transaction as value
     """
-    session = SessionBase.bind_session(session)
+    session = SessionBase.create_session()
     q_outer = session.query(
             TxCache.sender,
             func.min(Otx.nonce).label('nonce'),
@@ -605,7 +598,6 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
     q_outer = q_outer.filter(or_(Lock.flags==None, Lock.flags.op('&')(LockEnum.SEND.value)==0))
 
     if not is_alive(status):
-        SessionBase.release_session(session)
         raise ValueError('not a valid non-final tx value: {}'.format(status))
     if status == StatusEnum.PENDING:
         q_outer = q_outer.filter(Otx.status==status.value)
@@ -647,7 +639,7 @@ def get_upcoming_tx(status=StatusEnum.READYSEND, recipient=None, before=None, ch
         session.add(o)
         session.commit()
 
-    SessionBase.release_session(session)
+    session.close()
 
     return txs
 
@@ -670,7 +662,6 @@ def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None)
     """
     if not as_sender and not as_recipient:
         raise ValueError('at least one of as_sender and as_recipient must be True')
-
     txs = {}
 
     session = SessionBase.create_session()
@@ -686,11 +677,10 @@ def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None)
 
     results = q.all()
     for r in results:
-        if txs.get(r.tx_hash) != None:
-            logg.debug('tx {} already recorded'.format(r.tx_hash))
-            continue
         txs[r.tx_hash] = r.signed_tx
     session.close()
 
     return txs
 
+
+

apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py

@@ -1,4 +0,0 @@
-from .callback import CallbackFilter
-from .tx import TxFilter
-from .gas import GasFilter
-from .register import RegistrationFilter

apps/cic-eth/cic_eth/runnable/daemons/filters/base.py

@@ -1,2 +0,0 @@
-class SyncFilter:
-    pass

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -1,107 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import web3
-import celery
-from cic_registry.error import UnknownContractError
-
-# local imports
-from .base import SyncFilter
-from cic_eth.eth.token import unpack_transfer
-from cic_eth.eth.token import unpack_transferfrom
-from cic_eth.eth.token import ExtendedTx
-from .base import SyncFilter
-
-logg = logging.getLogger()
-
-transfer_method_signature = '0xa9059cbb' # keccak256(transfer(address,uint256))
-transferfrom_method_signature = '0x23b872dd' # keccak256(transferFrom(address,address,uint256))
-giveto_method_signature = '0x63e4bff4' # keccak256(giveTo(address))
-
-
-class CallbackFilter(SyncFilter):
-
-    trusted_addresses = []
-
-    def __init__(self, method, queue):
-        self.queue = queue
-        self.method = method
-
-
-    def call_back(self, transfer_type, result):
-        s = celery.signature(
-            self.method,
-            [
-                result,
-                transfer_type,
-                int(rcpt.status == 0),
-            ],
-            queue=self.queue,
-            )
-#        s_translate = celery.signature(
-#            'cic_eth.ext.address.translate',
-#            [
-#                result,
-#                self.trusted_addresses,
-#                chain_str,
-#                ],
-#            queue=self.queue,
-#            )
-#        s_translate.link(s)
-#        s_translate.apply_async()
-        s.apply_async()
-
-
-    def parse_data(self, tx, rcpt):
-        transfer_type = 'transfer'
-        transfer_data = None
-        method_signature = tx.input[:10]
-
-        if method_signature == transfer_method_signature:
-            transfer_data = unpack_transfer(tx.input)
-            transfer_data['from'] = tx['from']
-            transfer_data['token_address'] = tx['to']
-
-        elif method_signature == transferfrom_method_signature:
-            transfer_type = 'transferfrom'
-            transfer_data = unpack_transferfrom(tx.input)
-            transfer_data['token_address'] = tx['to']
-
-        # TODO: do not rely on logs here
-        elif method_signature == giveto_method_signature:
-            transfer_type = 'tokengift'
-            transfer_data = unpack_gift(tx.input)
-            for l in rcpt.logs:
-                if l.topics[0].hex() == '0x45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
-                    transfer_data['value'] = web3.Web3.toInt(hexstr=l.data)
-                    token_address_bytes = l.topics[2][32-20:]
-                    transfer_data['token_address'] = web3.Web3.toChecksumAddress(token_address_bytes.hex())
-                    transfer_data['from'] = rcpt.to
-
-        return (transfer_type, transfer_data)
-
-
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        logg.debug('applying callback filter "{}:{}"'.format(self.queue, self.method))
-        chain_str = str(chain_spec)
-
-        transfer_data = self.parse_data(tx, rcpt)
-
-        transfer_data = None
-        if len(tx.input) < 10:
-            logg.debug('callbacks filter data length not sufficient for method signature in tx {}, skipping'.format(tx['hash']))
-            return
-
-        logg.debug('checking callbacks filter input {}'.format(tx.input[:10]))
-
-        if transfer_data != None:
-            token_symbol = None
-            result = None
-            try:
-                tokentx = ExtendedTx(self.chain_spec)
-                tokentx.set_actors(transfer_data['from'], transfer_data['to'], self.trusted_addresses)
-                tokentx.set_tokens(transfer_data['token_address'], transfer_data['value'])
-                self.call_back(tokentx.to_dict())
-            except UnknownContractError:
-                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc.queue, tc.method, transfer_data['to'], tx.hash.hex()))

apps/cic-eth/cic_eth/runnable/daemons/filters/convert.py

@@ -1,61 +0,0 @@
-
-#__convert_log_hash = '0x7154b38b5dd31bb3122436a96d4e09aba5b323ae1fd580025fab55074334c095' # keccak256(Conversion(address,address,address,uint256,uint256,address)
-
-#def parse_convert_log(w3, entry):
-#    data = entry.data[2:]
-#    from_amount = int(data[:64], 16)
-#    to_amount = int(data[64:128], 16)
-#    holder_address_hex_raw = '0x' + data[-40:]
-#    holder_address_hex = w3.toChecksumAddress(holder_address_hex_raw)
-#    o = {
-#            'from_amount': from_amount,
-#            'to_amount': to_amount,
-#            'holder_address': holder_address_hex
-#            }
-#    logg.debug('parsed convert log {}'.format(o))
-#    return o
-
-
-#def convert_filter(w3, tx, rcpt, chain_spec):
-#    destination_token_address = None
-#    recipient_address = None
-#    amount = 0
-#    for l in rcpt['logs']:
-#        event_topic_hex = l['topics'][0].hex()
-#        if event_topic_hex == __convert_log_hash:
-#            tx_hash_hex = tx['hash'].hex()
-#            try:
-#                convert_transfer = TxConvertTransfer.get(tx_hash_hex)
-#            except UnknownConvertError:
-#                logg.warning('skipping unknown convert tx {}'.format(tx_hash_hex))
-#                continue
-#            if convert_transfer.transfer_tx_hash != None:
-#                logg.warning('convert tx {} cache record already has transfer hash {}, skipping'.format(tx_hash_hex, convert_transfer.transfer_hash))
-#                continue
-#            recipient_address = convert_transfer.recipient_address
-#            logg.debug('found convert event {} recipient'.format(tx_hash_hex, recipient_address))
-#            r = parse_convert_log(l)
-#            destination_token_address = l['topics'][3][-20:]
-#
-#    if destination_token_address == zero_address or destination_token_address == None:
-#        return None
-#
-#    destination_token_address_hex = destination_token_address.hex()
-#    s = celery.signature(
-#            'cic_eth.eth.bancor.transfer_converted',
-#            [
-#                [{
-#                    'address': w3.toChecksumAddress(destination_token_address_hex),
-#                    }],
-#                r['holder_address'],
-#                recipient_address,
-#                r['to_amount'],
-#                tx_hash_hex,
-#                str(chain_spec),
-#                ],
-#                queue=queue,
-#            )
-#    logg.info('sending tx signature {}'.format(s))
-#    t = s.apply_async()
-#    logg.debug('submitted transfer after convert task uuid {} {}'.format(t, t.successful()))
-#    return t

apps/cic-eth/cic_eth/runnable/daemons/filters/gas.py

@@ -1,56 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-from cic_registry.chain import ChainSpec
-
-# local imports
-from cic_eth.db.models.base import SessionBase
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db import Otx
-from cic_eth.queue.tx import get_paused_txs
-from cic_eth.eth.task import create_check_gas_and_send_task
-from .base import SyncFilter
-
-logg = logging.getLogger()
-
-
-class GasFilter(SyncFilter):
-
-    def __init__(self, gas_provider, queue=None):
-        self.queue = queue
-        self.gas_provider = gas_provider
-
-
-    def filter(self, w3, tx, rcpt, chain_str, session=None):
-        logg.debug('applying gas filter')
-        tx_hash_hex = tx.hash.hex()
-        if tx['value'] > 0:
-            logg.debug('gas refill tx {}'.format(tx_hash_hex))
-            session = SessionBase.bind_session(session)
-            q = session.query(TxCache.recipient)
-            q = q.join(Otx)
-            q = q.filter(Otx.tx_hash==tx_hash_hex)
-            r = q.first()
-
-            if r == None:
-                logg.warning('unsolicited gas refill tx {}'.format(tx_hash_hex))
-                SessionBase.release_session(session)
-                return
-
-            chain_spec = ChainSpec.from_chain_str(chain_str)
-            txs = get_paused_txs(StatusEnum.WAITFORGAS, r[0], chain_spec.chain_id(), session=session)
-
-            SessionBase.release_session(session)
-
-            if len(txs) > 0:
-                logg.info('resuming gas-in-waiting txs for {}: {}'.format(r[0], txs.keys()))
-                s = create_check_gas_and_send_task(
-                        list(txs.values()),
-                        str(chain_str),
-                        r[0],
-                        0,
-                        tx_hashes_hex=list(txs.keys()),
-                        queue=self.queue,
-                )
-                s.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/filters/register.py

@@ -1,35 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-from chainlib.eth.address import to_checksum
-
-# local imports
-from .base import SyncFilter
-
-logg = logging.getLogger()
-
-account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd' # keccak256(AccountAdded(address,uint256))
-
-
-class RegistrationFilter(SyncFilter):
-
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        logg.debug('applying registration filter')
-        registered_address = None
-        for l in rcpt['logs']:
-            event_topic_hex = l['topics'][0].hex()
-            if event_topic_hex == account_registry_add_log_hash:
-                address_bytes = l.topics[1][32-20:]
-                address = to_checksum(address_bytes.hex())
-                logg.debug('request token gift to {}'.format(address))
-                s = celery.signature(
-                    'cic_eth.eth.account.gift',
-                    [
-                        address,
-                        str(chain_spec),
-                        ],
-                    queue=queue,
-                    )
-                s.apply_async()

apps/cic-eth/cic_eth/runnable/daemons/filters/tx.py

@@ -1,41 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-
-# local imports
-from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.base import SessionBase
-from .base import SyncFilter
-
-logg = logging.getLogger()
-
-
-class TxFilter(SyncFilter):
-
-    def __init__(self, queue):
-        self.queue = queue
-
-
-    def filter(self, w3, tx, rcpt, chain_spec, session=None):
-        session = SessionBase.bind_session(session)
-        logg.debug('applying tx filter')
-        tx_hash_hex = tx.hash.hex()
-        otx = Otx.load(tx_hash_hex, session=session)
-        SessionBase.release_session(session)
-        if otx == None:
-            logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
-            return None
-        logg.info('otx found {}'.format(otx.tx_hash))
-        s = celery.signature(
-                'cic_eth.queue.tx.set_final_status',
-                [
-                    tx_hash_hex,
-                    rcpt.blockNumber,
-                    rcpt.status == 0,
-                    ],
-                queue=self.queue,
-                )
-        t = s.apply_async()
-        return t

apps/cic-eth/cic_eth/runnable/daemons/manager.py

@@ -1,207 +0,0 @@
-# standard imports
-import os
-import sys
-import logging
-import time
-import argparse
-import sys
-import re
-
-# third-party imports
-import confini
-import celery
-import rlp
-import web3
-from web3 import HTTPProvider, WebsocketProvider
-from cic_registry import CICRegistry
-from cic_registry.chain import ChainSpec
-from cic_registry import zero_address
-from cic_registry.chain import ChainRegistry
-from cic_registry.error import UnknownContractError
-from cic_bancor.bancor import BancorRegistryClient
-
-# local imports
-import cic_eth
-from cic_eth.eth import RpcClient
-from cic_eth.db import SessionBase
-from cic_eth.db import Otx
-from cic_eth.db import TxConvertTransfer
-from cic_eth.db.models.tx import TxCache
-from cic_eth.db.enum import StatusEnum
-from cic_eth.db import dsn_from_config
-from cic_eth.queue.tx import get_paused_txs
-from cic_eth.sync import Syncer
-from cic_eth.sync.error import LoopDone
-from cic_eth.db.error import UnknownConvertError
-from cic_eth.eth.util import unpack_signed_raw_tx
-from cic_eth.eth.task import create_check_gas_and_send_task
-from cic_eth.sync.backend import SyncerBackend
-from cic_eth.eth.token import unpack_transfer
-from cic_eth.eth.token import unpack_transferfrom
-from cic_eth.eth.account import unpack_gift
-from cic_eth.runnable.daemons.filters import (
-        CallbackFilter,
-        GasFilter,
-        TxFilter,
-        RegistrationFilter,
-        )
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
-logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
-logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
-
-
-config_dir = os.path.join('/usr/local/etc/cic-eth')
-
-argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
-argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
-argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
-argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
-argparser.add_argument('-v', help='be verbose', action='store_true')
-argparser.add_argument('-vv', help='be more verbose', action='store_true')
-argparser.add_argument('mode', type=str, help='sync mode: (head|history)', default='head')
-args = argparser.parse_args(sys.argv[1:])
-
-if args.v == True:
-    logging.getLogger().setLevel(logging.INFO)
-elif args.vv == True:
-    logging.getLogger().setLevel(logging.DEBUG)
-
-config_dir = os.path.join(args.c)
-os.makedirs(config_dir, 0o777, True)
-config = confini.Config(config_dir, args.env_prefix)
-config.process()
-# override args
-args_override = {
-        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
-        'CIC_CHAIN_SPEC': getattr(args, 'i'),
-        }
-config.dict_override(args_override, 'cli flag')
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
-
-app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
-
-queue = args.q
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-
-re_websocket = re.compile('^wss?://')
-re_http = re.compile('^https?://')
-blockchain_provider = config.get('ETH_PROVIDER')
-if re.match(re_websocket, blockchain_provider) != None:
-    blockchain_provider = WebsocketProvider(blockchain_provider)
-elif re.match(re_http, blockchain_provider) != None:
-    blockchain_provider = HTTPProvider(blockchain_provider)
-else:
-    raise ValueError('unknown provider url {}'.format(blockchain_provider))
-
-def web3_constructor():
-    w3 = web3.Web3(blockchain_provider)
-    return (blockchain_provider, w3)
-RpcClient.set_constructor(web3_constructor)
-
-c = RpcClient(chain_spec)
-CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
-CICRegistry.add_path(config.get('ETH_ABI_DIR'))
-chain_registry = ChainRegistry(chain_spec)
-CICRegistry.add_chain_registry(chain_registry, True)
-
-declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
-
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=1, debug=config.true('DATABASE_DEBUG'))
-
-
-def main():
-    global chain_spec, c, queue
-
-    if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
-        CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
-
-    syncers = []
-    block_offset = c.w3.eth.blockNumber
-    chain = str(chain_spec)
-
-    if SyncerBackend.first(chain):
-        from cic_eth.sync.history import HistorySyncer
-        backend = SyncerBackend.initial(chain, block_offset)
-        syncer = HistorySyncer(backend) 
-        syncers.append(syncer)
-
-    if args.mode == 'head':
-        from cic_eth.sync.head import HeadSyncer
-        block_sync = SyncerBackend.live(chain, block_offset+1)
-        syncers.append(HeadSyncer(block_sync))
-    elif args.mode == 'history':
-        from cic_eth.sync.history import HistorySyncer
-        backends = SyncerBackend.resume(chain, block_offset+1)
-        for backend in backends:
-            syncers.append(HistorySyncer(backend))
-        if len(syncers) == 0:
-            logg.info('found no unsynced history. terminating')
-            sys.exit(0)
-    else:
-        sys.stderr.write("unknown mode '{}'\n".format(args.mode))
-        sys.exit(1)
-
-#    bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
-#    bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
-#    bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
-#    bancor_registry.load() 
- 
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-    CallbackFilter.trusted_addresses = trusted_addresses
-
-    callback_filters = []
-    for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
-        task_split = cb.split(':')
-        task_queue = queue
-        if len(task_split) > 1:
-            task_queue = task_split[0]
-        callback_filter = CallbackFilter(task_split[1], task_queue)
-        callback_filters.append(callback_filter)
-
-    tx_filter = TxFilter(queue)
-
-    registration_filter = RegistrationFilter()
-
-    gas_filter = GasFilter(c.gas_provider(), queue)
-
-    i = 0
-    for syncer in syncers:
-        logg.debug('running syncer index {}'.format(i))
-        syncer.filter.append(gas_filter.filter)
-        syncer.filter.append(registration_filter.filter)
-        # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
-        syncer.filter.append(tx_filter.filter)
-        #syncer.filter.append(convert_filter)
-        for cf in callback_filters:
-            syncer.filter.append(cf.filter)
-
-        try:
-            syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')))
-        except LoopDone as e:
-            sys.stderr.write("sync '{}' done at block {}\n".format(args.mode, e))
-
-        i += 1
-
-    sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()

apps/cic-eth/cic_eth/runnable/manager.py

@@ -0,0 +1,410 @@
+# standard imports
+import os
+import sys
+import logging
+import time
+import argparse
+import sys
+import re
+
+# third-party imports
+import confini
+import celery
+import rlp
+import web3
+from web3 import HTTPProvider, WebsocketProvider
+from cic_registry import CICRegistry
+from cic_registry.chain import ChainSpec
+from cic_registry import zero_address
+from cic_registry.chain import ChainRegistry
+from cic_registry.error import UnknownContractError
+from cic_bancor.bancor import BancorRegistryClient
+
+# local imports
+import cic_eth
+from cic_eth.eth import RpcClient
+from cic_eth.db import SessionBase
+from cic_eth.db import Otx
+from cic_eth.db import TxConvertTransfer
+from cic_eth.db.models.tx import TxCache
+from cic_eth.db.enum import StatusEnum
+from cic_eth.db import dsn_from_config
+from cic_eth.queue.tx import get_paused_txs
+from cic_eth.sync import Syncer
+from cic_eth.sync.error import LoopDone
+from cic_eth.db.error import UnknownConvertError
+from cic_eth.eth.util import unpack_signed_raw_tx
+from cic_eth.eth.task import create_check_gas_and_send_task
+from cic_eth.sync.backend import SyncerBackend
+from cic_eth.eth.token import unpack_transfer
+from cic_eth.eth.token import unpack_transferfrom
+from cic_eth.eth.account import unpack_gift
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+logging.getLogger('websockets.protocol').setLevel(logging.CRITICAL)
+logging.getLogger('web3.RequestManager').setLevel(logging.CRITICAL)
+logging.getLogger('web3.providers.WebsocketProvider').setLevel(logging.CRITICAL)
+logging.getLogger('web3.providers.HTTPProvider').setLevel(logging.CRITICAL)
+
+
+config_dir = os.path.join('/usr/local/etc/cic-eth')
+
+argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks')
+argparser.add_argument('-c', type=str, default=config_dir, help='config root to use')
+argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec')
+argparser.add_argument('--abi-dir', dest='abi_dir', type=str, help='Directory containing bytecode and abi')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
+argparser.add_argument('-v', help='be verbose', action='store_true')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('mode', type=str, help='sync mode: (head|history)', default='head')
+args = argparser.parse_args(sys.argv[1:])
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+os.makedirs(config_dir, 0o777, True)
+config = confini.Config(config_dir, args.env_prefix)
+config.process()
+# override args
+args_override = {
+        'ETH_ABI_DIR': getattr(args, 'abi_dir'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+        }
+config.dict_override(args_override, 'cli flag')
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
+
+queue = args.q
+
+dsn = dsn_from_config(config)
+SessionBase.connect(dsn)
+
+# TODO: There is too much code in this file, split it up
+
+transfer_callbacks = []
+for cb in config.get('TASKS_TRANSFER_CALLBACKS', '').split(','):
+    task_split = cb.split(':')
+    task_queue = queue
+    if len(task_split) > 1:
+        task_queue = task_split[0]
+    task_pair = (task_split[1], task_queue)
+    transfer_callbacks.append(task_pair)
+
+
+# TODO: move to contract registry
+__convert_log_hash = '0x7154b38b5dd31bb3122436a96d4e09aba5b323ae1fd580025fab55074334c095' # keccak256(Conversion(address,address,address,uint256,uint256,address)
+__account_registry_add_log_hash = '0x5ed3bdd47b9af629827a8d129aa39c870b10c03f0153fe9ddb8e84b665061acd' # keccak256(AccountAdded(address,uint256))
+
+__transfer_method_signature = '0xa9059cbb' # keccak256(transfer(address,uint256))
+__transferfrom_method_signature = '0x23b872dd' # keccak256(transferFrom(address,address,uint256))
+__giveto_method_signature = '0x63e4bff4' # keccak256(giveTo(address))
+
+# TODO: move to bancor package
+def parse_convert_log(w3, entry):
+    data = entry.data[2:]
+    from_amount = int(data[:64], 16)
+    to_amount = int(data[64:128], 16)
+    holder_address_hex_raw = '0x' + data[-40:]
+    holder_address_hex = w3.toChecksumAddress(holder_address_hex_raw)
+    o = {
+            'from_amount': from_amount,
+            'to_amount': to_amount,
+            'holder_address': holder_address_hex
+            }
+    logg.debug('parsed convert log {}'.format(o))
+    return o
+
+
+def registration_filter(w3, tx, rcpt, chain_spec):
+    registered_address = None
+    for l in rcpt['logs']:
+        event_topic_hex = l['topics'][0].hex()
+        if event_topic_hex == __account_registry_add_log_hash:
+            address_bytes = l.topics[1][32-20:]
+            address = web3.Web3.toChecksumAddress(address_bytes.hex())
+            logg.debug('request token gift to {}'.format(address))
+            s = celery.signature(
+                'cic_eth.eth.account.gift',
+                [
+                    address,
+                    str(chain_spec),
+                    ],
+                queue=queue,
+                )
+            s.apply_async()
+
+
+def convert_filter(w3, tx, rcpt, chain_spec):
+    destination_token_address = None
+    recipient_address = None
+    amount = 0
+    for l in rcpt['logs']:
+        event_topic_hex = l['topics'][0].hex()
+        if event_topic_hex == __convert_log_hash:
+            tx_hash_hex = tx['hash'].hex()
+            try:
+                convert_transfer = TxConvertTransfer.get(tx_hash_hex)
+            except UnknownConvertError:
+                logg.warning('skipping unknown convert tx {}'.format(tx_hash_hex))
+                continue
+            if convert_transfer.transfer_tx_hash != None:
+                logg.warning('convert tx {} cache record already has transfer hash {}, skipping'.format(tx_hash_hex, convert_transfer.transfer_hash))
+                continue
+            recipient_address = convert_transfer.recipient_address
+            logg.debug('found convert event {} recipient'.format(tx_hash_hex, recipient_address))
+            r = parse_convert_log(l)
+            destination_token_address = l['topics'][3][-20:]
+
+    if destination_token_address == zero_address or destination_token_address == None:
+        return None
+
+    destination_token_address_hex = destination_token_address.hex()
+    s = celery.signature(
+            'cic_eth.eth.bancor.transfer_converted',
+            [
+                [{
+                    'address': w3.toChecksumAddress(destination_token_address_hex),
+                    }],
+                r['holder_address'],
+                recipient_address,
+                r['to_amount'],
+                tx_hash_hex,
+                str(chain_spec),
+                ],
+                queue=queue,
+            )
+    logg.info('sending tx signature {}'.format(s))
+    t = s.apply_async()
+    logg.debug('submitted transfer after convert task uuid {} {}'.format(t, t.successful()))
+    return t
+
+
+def tx_filter(w3, tx, rcpt, chain_spec):
+    tx_hash_hex = tx.hash.hex()
+    otx = Otx.load(tx_hash_hex)
+    if otx == None:
+        logg.debug('tx {} not found locally, skipping'.format(tx_hash_hex))
+        return None
+    logg.info('otx found {}'.format(otx.tx_hash))
+    s = celery.signature(
+            'cic_eth.queue.tx.set_final_status',
+            [
+                tx_hash_hex,
+                rcpt.blockNumber,
+                rcpt.status == 0,
+                ],
+            queue=queue,
+            )
+    t = s.apply_async()
+    return t
+
+
+# TODO: replace with registry call instead
+def get_token_symbol(w3, address):
+    #token = CICRegistry.get_address(CICRegistry.chain_spec, tx['to'])
+    logg.warning('token verification missing')
+    c = w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=address)
+    return c.functions.symbol().call()
+
+
+# TODO: replace with registry call instead
+def get_token_decimals(w3, address):
+    #token = CICRegistry.get_address(CICRegistry.chain_spec, tx['to'])
+    logg.warning('token verification missing')
+    c = w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=address)
+    return c.functions.decimals().call()
+
+
+def callbacks_filter(w3, tx, rcpt, chain_spec):
+    transfer_data = None
+    if len(tx.input) < 10:
+        logg.debug('callbacks filter data length not sufficient for method signature in tx {}, skipping'.format(tx['hash']))
+        return
+
+    logg.debug('checking callbacks filter input {}'.format(tx.input[:10]))
+
+    transfer_type = 'transfer'
+    method_signature = tx.input[:10]
+    if method_signature == __transfer_method_signature:
+        transfer_data = unpack_transfer(tx.input)
+        transfer_data['from'] = tx['from']
+        transfer_data['token_address'] = tx['to']
+    elif method_signature == __transferfrom_method_signature:
+        transfer_type = 'transferfrom'
+        transfer_data = unpack_transferfrom(tx.input)
+        transfer_data['token_address'] = tx['to']
+    elif method_signature == __giveto_method_signature:
+        transfer_type = 'tokengift'
+        transfer_data = unpack_gift(tx.input)
+        for l in rcpt.logs:
+            if l.topics[0].hex() == '0x45c201a59ac545000ead84f30b2db67da23353aa1d58ac522c48505412143ffa':
+                transfer_data['amount'] = web3.Web3.toInt(hexstr=l.data)
+                token_address_bytes = l.topics[2][32-20:]
+                transfer_data['token_address'] = web3.Web3.toChecksumAddress(token_address_bytes.hex())
+                transfer_data['from'] = rcpt.to
+
+    if transfer_data != None:
+        for tc in transfer_callbacks:
+            token_symbol = None
+            try:
+                logg.debug('checking token {}'.format(transfer_data['token_address']))
+                token_symbol = get_token_symbol(w3, transfer_data['token_address'])
+                token_decimals = get_token_decimals(w3, transfer_data['token_address'])
+                logg.debug('calling transfer callback {}:{} for tx {}'.format(tc[1], tc[0], tx['hash']))
+            except UnknownContractError:
+                logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tc[1], tc[0], transfer_data['to'], tx.hash.hex()))
+                continue
+            result = {
+                'hash': tx.hash.hex(),
+                'sender': transfer_data['from'],
+                'recipient': transfer_data['to'],
+                'source_value': transfer_data['amount'],
+                'destination_value': transfer_data['amount'],
+                'source_token': transfer_data['token_address'],
+                'destination_token': transfer_data['token_address'],
+                'source_token_symbol': token_symbol,
+                'destination_token_symbol': token_symbol,
+                'source_token_decimals': token_decimals,
+                'destination_token_decimals': token_decimals,
+                'chain': str(chain_spec),
+                    }
+            s = celery.signature(
+                tc[0],
+                [
+                    result,
+                    transfer_type,
+                    int(rcpt.status == 0),
+                ],
+                queue=tc[1],
+                )
+            s.apply_async()
+
+
+class GasFilter:
+
+    def __init__(self, gas_provider):
+        self.gas_provider = gas_provider
+
+    def filter(self, w3, tx, rcpt, chain_str):
+        tx_hash_hex = tx.hash.hex()
+        if tx['value'] > 0:
+            logg.debug('gas refill tx {}'.format(tx_hash_hex))
+            session = SessionBase.create_session()
+            q = session.query(TxCache.recipient)
+            q = q.join(Otx)
+            q = q.filter(Otx.tx_hash==tx_hash_hex)
+            r = q.first()
+
+            session.close()
+
+            if r == None:
+                logg.warning('unsolicited gas refill tx {}'.format(tx_hash_hex))
+                return
+
+            chain_spec = ChainSpec.from_chain_str(chain_str)
+            txs = get_paused_txs(StatusEnum.WAITFORGAS, r[0], chain_spec.chain_id())
+
+            if len(txs) > 0:
+                logg.info('resuming gas-in-waiting txs for {}: {}'.format(r[0], txs.keys()))
+                s = create_check_gas_and_send_task(
+                        list(txs.values()),
+                        str(chain_str),
+                        r[0],
+                        0,
+                        tx_hashes_hex=list(txs.keys()),
+                        queue=queue,
+                )
+                s.apply_async()
+
+
+re_websocket = re.compile('^wss?://')
+re_http = re.compile('^https?://')
+blockchain_provider = config.get('ETH_PROVIDER')
+if re.match(re_websocket, blockchain_provider) != None:
+    blockchain_provider = WebsocketProvider(blockchain_provider)
+elif re.match(re_http, blockchain_provider) != None:
+    blockchain_provider = HTTPProvider(blockchain_provider)
+else:
+    raise ValueError('unknown provider url {}'.format(blockchain_provider))
+
+def web3_constructor():
+    w3 = web3.Web3(blockchain_provider)
+    return (blockchain_provider, w3)
+RpcClient.set_constructor(web3_constructor)
+
+
+def main(): 
+
+    chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
+    c = RpcClient(chain_spec)
+
+    CICRegistry.init(c.w3, config.get('CIC_REGISTRY_ADDRESS'), chain_spec)
+    CICRegistry.add_path(config.get('ETH_ABI_DIR'))
+    chain_registry = ChainRegistry(chain_spec)
+    CICRegistry.add_chain_registry(chain_registry)
+
+    if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
+        CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
+
+    syncers = []
+    block_offset = c.w3.eth.blockNumber
+    chain = str(chain_spec)
+
+    if SyncerBackend.first(chain):
+        from cic_eth.sync.history import HistorySyncer
+        backend = SyncerBackend.initial(chain, block_offset)
+        syncer = HistorySyncer(backend) 
+        syncers.append(syncer)
+
+    if args.mode == 'head':
+        from cic_eth.sync.head import HeadSyncer
+        block_sync = SyncerBackend.live(chain, block_offset+1)
+        syncers.append(HeadSyncer(block_sync))
+    elif args.mode == 'history':
+        from cic_eth.sync.history import HistorySyncer
+        backends = SyncerBackend.resume(chain, block_offset+1)
+        for backend in backends:
+            syncers.append(HistorySyncer(backend))
+        if len(syncers) == 0:
+            logg.info('found no unsynced history. terminating')
+            sys.exit(0)
+    else:
+        sys.stderr.write("unknown mode '{}'\n".format(args.mode))
+        sys.exit(1)
+
+#    bancor_registry_contract = CICRegistry.get_contract(chain_spec, 'BancorRegistry', interface='Registry')
+#    bancor_chain_registry = CICRegistry.get_chain_registry(chain_spec)
+#    bancor_registry = BancorRegistryClient(c.w3, bancor_chain_registry, config.get('ETH_ABI_DIR'))
+#    bancor_registry.load() 
+ 
+    i = 0
+    for syncer in syncers:
+        logg.debug('running syncer index {}'.format(i))
+        gas_filter = GasFilter(c.gas_provider()).filter
+        syncer.filter.append(gas_filter)
+        syncer.filter.append(registration_filter)
+        syncer.filter.append(callbacks_filter)
+        # TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
+        syncer.filter.append(tx_filter)
+        syncer.filter.append(convert_filter)
+
+        try:
+            syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')))
+        except LoopDone as e:
+            sys.stderr.write("sync '{}' done at block {}\n".format(args.mode, e))
+
+        i += 1
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

apps/cic-eth/cic_eth/runnable/server_agent.py

@@ -55,62 +55,25 @@ SessionBase.connect(dsn)
 celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'),  broker=config.get('CELERY_BROKER_URL'))
 queue = args.q
 
-re_transfer_approval_request = r'^/transferrequest/?'
+re_something = r'^/something/?'
 
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
 
-def process_transfer_approval_request(session, env):
-    r = re.match(re_transfer_approval_request, env.get('PATH_INFO'))
+def process_something(session, env):
+    r = re.match(re_something, env.get('PATH_INFO'))
     if not r:
         return None
 
-    if env.get('CONTENT_TYPE') != 'application/json':
-        raise AttributeError('content type')
+    #if env.get('CONTENT_TYPE') != 'application/json':
+    #    raise AttributeError('content type')
 
-    if env.get('REQUEST_METHOD') != 'POST':
-        raise AttributeError('method')
+    #if env.get('REQUEST_METHOD') != 'POST':
+    #    raise AttributeError('method')
 
-    post_data = json.load(env.get('wsgi.input'))
-    token_address = web3.Web3.toChecksumAddress(post_data['token_address'])
-    holder_address = web3.Web3.toChecksumAddress(post_data['holder_address'])
-    beneficiary_address = web3.Web3.toChecksumAddress(post_data['beneficiary_address'])
-    value = int(post_data['value'])
-
-    logg.debug('transfer approval request token {} to {} from {} value {}'.format(
-        token_address,
-        beneficiary_address,
-        holder_address,
-        value,
-        )
-        )
-
-    s = celery.signature(
-        'cic_eth.eth.request.transfer_approval_request',
-        [
-            [
-                {
-                    'address': token_address,
-                    },
-                ],
-            holder_address,
-            beneficiary_address,
-            value,
-            config.get('CIC_CHAIN_SPEC'),
-            ],
-        queue=queue,
-     )
-    t = s.apply_async()
-    r = t.get()
-    tx_raw_bytes = bytes.fromhex(r[0][2:])
-    tx = unpack_signed_raw_tx(tx_raw_bytes, chain_spec.chain_id())
-    for r in t.collect():
-        logg.debug('result {}'.format(r))
-
-    if not t.successful():
-        raise RuntimeError(tx['hash'])
-
-    return ('text/plain', tx['hash'].encode('utf-8'),)
+    #post_data = json.load(env.get('wsgi.input'))
+    
+    #return ('text/plain', 'foo'.encode('utf-8'),)
 
 
 # uwsgi application
@@ -125,7 +88,7 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
-            process_transfer_approval_request,
+            process_something,
             ]:
         try:
             r = handler(session, env)

apps/cic-eth/cic_eth/runnable/tasker.py

@@ -78,7 +78,7 @@ logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)
-SessionBase.connect(dsn, pool_size=8, debug=config.true('DATABASE_DEBUG'))
+SessionBase.connect(dsn)
 
 # verify database connection with minimal sanity query
 session = SessionBase.create_session()
@@ -179,6 +179,7 @@ def web3ext_constructor():
     return (blockchain_provider, w3)
 RpcClient.set_constructor(web3ext_constructor) 
 
+logg.info('ccc {}'.format(config.store['TASKS_TRACE_QUEUE_STATUS']))
 Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
 
 
@@ -219,16 +220,15 @@ def main():
     if config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER') != None:
         CICRegistry.add_role(chain_spec, config.get('ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER'), 'AccountRegistry', True)
 
-    declarator = CICRegistry.get_contract(chain_spec, 'AddressDeclarator', interface='Declarator')
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-    oracle = DeclaratorOracleAdapter(declarator.contract, trusted_addresses)
-    chain_registry.add_oracle('naive_erc20_oracle', oracle)
+    if config.get('CIC_DECLARATOR_ADDRESS') != None:
+        abi_path = os.path.join(config.get('ETH_ABI_DIR'), '{}.json'.format(interface))
+        f = open(abi_path)
+        abi = json.load(abi_path)
+        f.close()
+        c = w3.eth.contract(abi=abi, address=address)
+        trusted_addresses = config.get('CIC_TRUSTED_ADDRESSES', []).split(',')
+        oracle = DeclaratorOracleAdapter(contract, trusted_addresses)
+        chain_registry.add_oracle(oracle)
 
 
     #chain_spec = CICRegistry.default_chain_spec

apps/cic-eth/cic_eth/sync/backend.py

@@ -28,25 +28,20 @@ class SyncerBackend:
     def connect(self):
         """Loads the state of the syncer session with the given id.
         """
-        if self.db_session == None:
-            self.db_session = SessionBase.create_session()
+        self.db_session = SessionBase.create_session()
         q = self.db_session.query(BlockchainSync)
         q = q.filter(BlockchainSync.id==self.object_id)
         self.db_object = q.first()
         if self.db_object == None:
-            self.disconnect()
             raise ValueError('sync entry with id {} not found'.format(self.object_id))
-        return self.db_session
 
 
     def disconnect(self):
         """Commits state of sync to backend.
         """
-        if self.db_session != None:
-            self.db_session.add(self.db_object)
-            self.db_session.commit()
-            self.db_session.close()
-            self.db_session = None
+        self.db_session.add(self.db_object)
+        self.db_session.commit()
+        self.db_session.close()
        
 
     def chain(self):

apps/cic-eth/cic_eth/sync/mined.py

@@ -56,8 +56,7 @@ class MinedSyncer(Syncer):
             # TODO: ensure filter loop can complete on graceful shutdown
             for f in self.filter:
                 #try:
-                session = self.bc_cache.connect()
-                task_uuid = f(w3, tx, rcpt, self.chain(), session)
+                task_uuid = f(w3, tx, rcpt, self.chain())
                 #except Exception as e:
                 #    logg.error('error in filter {} tx {}: {}'.format(f, tx_hash_hex, e))
                 #    continue

apps/cic-eth/cic_eth/version.py

@@ -10,7 +10,7 @@ version = (
         0,
         10,
         0,
-        'alpha.27',
+        'alpha.26',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/cic.ini

@@ -1,5 +1,8 @@
 [cic]
 registry_address = 
+token_index_address =
+accounts_index_address = 
+declarator_address = 
+approval_escrow_address =
 chain_spec = 
 tx_retry_delay = 
-trust_address = 

apps/cic-eth/config/test/cic.ini

@@ -1,4 +1,8 @@
 [cic]
 registry_address = 
+token_index_address =
+accounts_index_address = 
+declarator_address = 
+approval_escrow_address =
 chain_spec = 
-trust_address =
+trusted_addresses =

apps/cic-eth/docker/start_dispatcher.sh

@@ -2,4 +2,4 @@
 
 . ./db.sh
 
-/usr/local/bin/cic-eth-dispatcherd $@
+/usr/local/bin/cic-eth-dispatcher $@

apps/cic-eth/docker/start_manager.sh

@@ -2,4 +2,4 @@
 
 . ./db.sh
 
-/usr/local/bin/cic-eth-managerd $@
+/usr/local/bin/cic-eth-manager $@

apps/cic-eth/docker/start_retry.sh

@@ -2,4 +2,4 @@
 
 . ./db.sh
 
-/usr/local/bin/cic-eth-retrierd $@
+/usr/local/bin/cic-eth-retrier $@

apps/cic-eth/docker/start_tasker.sh

@@ -9,7 +9,7 @@ echo "!!! starting signer"
 python /usr/local/bin/crypto-dev-daemon -vv -c /usr/local/etc/crypto-dev-signer &
 
 echo "!!! starting tracker"
-/usr/local/bin/cic-eth-taskerd $@
+/usr/local/bin/cic-eth-tasker $@
 
 # thanks! https://docs.docker.com/config/containers/multi-service_container/
 sleep 1;

apps/cic-eth/requirements.txt

@@ -2,21 +2,18 @@ web3==5.12.2
 celery==4.4.7
 crypto-dev-signer~=0.4.13rc2
 confini~=0.3.6b1
-cic-registry~=0.5.3a18
+cic-registry~=0.5.3a12
 cic-bancor~=0.0.6
 redis==3.5.3
 alembic==1.4.2
 websockets==8.1
 requests~=2.24.0
 eth_accounts_index~=0.0.10a7
-erc20-approval-escrow~=0.3.0a5
+erc20-transfer-authorization~=0.3.0a7
 erc20-single-shot-faucet~=0.2.0a6
+eth-address-index~=0.1.0a8
 rlp==2.0.1
 uWSGI==2.0.19.1
 semver==2.13.0
 eth-gas-proxy==0.0.1a4
 websocket-client==0.57.0
-moolb~=0.1.1b2
-eth-address-index~=0.1.0a8
-chainlib~=0.0.1a12
-hexathon~=0.0.1a3

apps/cic-eth/setup.cfg

@@ -33,10 +33,7 @@ packages =
 	cic_eth.db.models
 	cic_eth.queue
 	cic_eth.sync
-	cic_eth.ext
 	cic_eth.runnable
-	cic_eth.runnable.daemons
-	cic_eth.runnable.daemons.filters
 	cic_eth.callbacks
 scripts =
 	./scripts/migrate.py
@@ -44,10 +41,10 @@ scripts =
 [options.entry_points]
 console_scripts =
 	# daemons
-	cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main
-	cic-eth-managerd = cic_eth.runnable.daemons.manager:main
-	cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main
-	cic-eth-retrierd = cic_eth.runnable.daemons.retry:main
+	cic-eth-tasker = cic_eth.runnable.tasker:main
+	cic-eth-manager = cic_eth.runnable.manager:main
+	cic-eth-dispatcher = cic_eth.runnable.dispatcher:main
+	cic-eth-retrier = cic_eth.runnable.retry:main
 	# tools	
 	cic-eth-create = cic_eth.runnable.create:main 
 	cic-eth-inspect = cic_eth.runnable.view:main

apps/cic-eth/tests/fixtures_celery.py

@@ -15,15 +15,12 @@ def celery_includes():
         'cic_eth.eth.token',
         'cic_eth.eth.request',
         'cic_eth.eth.tx',
-        'cic_eth.ext.tx',
         'cic_eth.queue.tx',
-        'cic_eth.queue.balance',
         'cic_eth.admin.ctrl',
         'cic_eth.admin.nonce',
         'cic_eth.eth.account',
         'cic_eth.callbacks.noop',
         'cic_eth.callbacks.http',
-        'tests.mock.filter',
     ]
 
 

apps/cic-eth/tests/fixtures_registry.py

@@ -1,50 +1,10 @@
-# standard imports
-import os
-import json
-import logging
-
 # third-party imports
 import pytest
-from eth_address_declarator import AddressDeclarator
+import os
+import json
 
 # local imports
 from cic_registry import CICRegistry
-from cic_registry import to_identifier
 from cic_registry.contract import Contract
 from cic_registry.error import ChainExistsError
 
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(__file__)
-
-
-@pytest.fixture(scope='session')
-def local_cic_registry(
-        cic_registry,
-        ):
-    path = os.path.realpath(os.path.join(script_dir, 'testdata', 'abi'))
-    CICRegistry.add_path(path)
-    return cic_registry
-
-
-@pytest.fixture(scope='function')
-def address_declarator(
-        bloxberg_config,
-        default_chain_spec,
-        default_chain_registry,
-        local_cic_registry,
-        init_rpc,
-        init_w3,
-        ):
-
-    c = init_rpc.w3.eth.contract(abi=AddressDeclarator.abi(), bytecode=AddressDeclarator.bytecode())
-    default_description = '0x{:<064s}'.format(b'test'.hex()) 
-    logg.debug('default_ {}'.format(default_description))
-    tx_hash = c.constructor(default_description).transact()
-    rcpt = init_rpc.w3.eth.getTransactionReceipt(tx_hash)
-
-    registry = init_rpc.w3.eth.contract(abi=CICRegistry.abi(), address=local_cic_registry)
-    chain_identifier = to_identifier(default_chain_registry.chain())
-    registry.functions.set(to_identifier('AddressDeclarator'), rcpt.contractAddress, chain_identifier, bloxberg_config['digest']).transact()
-
-    return rcpt.contractAddress

apps/cic-eth/tests/functional/test_app.py

@@ -29,6 +29,27 @@ def test_account_api(
     assert t.successful()
 
 
+def test_balance_api(
+        default_chain_spec,
+        default_chain_registry,
+        init_w3,
+        cic_registry,
+        init_database,
+        bancor_tokens,
+        bancor_registry,
+        celery_session_worker,
+        ):
+
+    token = CICRegistry.get_address(default_chain_spec, bancor_tokens[0])
+
+    api = Api(str(default_chain_spec), callback_param='balance', callback_task='cic_eth.callbacks.noop.noop', queue=None)
+    t = api.balance(init_w3.eth.accounts[2], token.symbol())
+    t.get()
+    for r in t.collect():
+        print(r)
+    assert t.successful()
+
+
 def test_transfer_api(
         default_chain_spec,
         init_w3,

apps/cic-eth/tests/functional/test_balance.py

@@ -1,40 +0,0 @@
-# standard imports
-import os
-import logging
-
-# local imports
-import web3
-from cic_eth.api.api_task import Api
-
-logg = logging.getLogger()
-
-
-def test_balance_complex_api(
-        default_chain_spec,
-        init_database,
-        init_w3,
-        cic_registry,
-        dummy_token,
-        dummy_token_registered,
-        celery_session_worker,
-        init_eth_tester,
-    ):
-
-    chain_str = str(default_chain_spec)
-
-    api = Api(chain_str, queue=None, callback_param='foo')
-
-    a = web3.Web3.toChecksumAddress('0x' + os.urandom(20).hex())
-    t = api.balance(a, 'DUM')
-    t.get()
-    r = None
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-    logg.debug(r)
-
-    assert r[0].get('balance_incoming') != None
-    assert r[0].get('balance_outgoing') != None
-    assert r[0].get('balance_network') != None
-
-    logg.debug('r {}'.format(r))

apps/cic-eth/tests/functional/test_list.py

@@ -1,92 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from cic_eth.api.api_task import Api
-from cic_eth.eth.token import TokenTxFactory
-from cic_eth.eth.task import sign_tx
-from tests.mock.filter import (
-        block_filter,
-        tx_filter,
-        )
-
-
-logg = logging.getLogger()
-
-
-def test_list_tx(
-        default_chain_spec,
-        default_chain_registry,
-        init_database,
-        init_rpc,
-        init_w3,
-        init_eth_tester,
-        dummy_token_gifted,
-        cic_registry,
-        celery_session_worker,
-        ):
-
-    tx_hashes = []
-    # external tx
-    init_eth_tester.mine_blocks(13)
-    txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc)
-    tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 3000, default_chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec))
-    tx_hashes.append(tx_hash_hex)
-    init_w3.eth.sendRawTransaction(tx_signed_raw_hex)
-    # add to filter
-    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex)
-    a = rcpt['blockNumber']
-    block_filter.add(a.to_bytes(4, 'big'))
-    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-    tx_filter.add(a.to_bytes(4, 'big'))
-
-    # external tx
-    init_eth_tester.mine_blocks(28)
-    txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc)
-    tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 4000, default_chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec))
-    tx_hashes.append(tx_hash_hex)
-    init_w3.eth.sendRawTransaction(tx_signed_raw_hex)
-    # add to filter
-    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex)
-    a = rcpt['blockNumber']
-    block_filter.add(a.to_bytes(4, 'big'))
-    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-    tx_filter.add(a.to_bytes(4, 'big'))
-
-    # custodial tx
-    init_eth_tester.mine_blocks(3)
-    txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc)
-    api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(init_w3.eth.accounts[0], init_w3.eth.accounts[1], 1000, 'DUM')
-    t.get()
-    tx_hash_hex = None
-    for c in t.collect():
-        tx_hash_hex = c[1]
-    assert t.successful()
-    tx_hashes.append(tx_hash_hex)
-
-    # custodial tx
-    init_eth_tester.mine_blocks(6)
-    api = Api(str(default_chain_spec), queue=None)
-    t = api.transfer(init_w3.eth.accounts[0], init_w3.eth.accounts[1], 2000, 'DUM')
-    t.get()
-    tx_hash_hex = None
-    for c in t.collect():
-        tx_hash_hex = c[1]
-    assert t.successful()
-    tx_hashes.append(tx_hash_hex)
-
-    # test the api
-    t = api.list(init_w3.eth.accounts[1], external_task='tests.mock.filter.filter')
-    r = t.get()
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-
-    assert len(r) == 4
-    for tx in r:
-        logg.debug('have tx {}'.format(r))
-        tx_hashes.remove(tx['hash'])
-    assert len(tx_hashes) == 0

apps/cic-eth/tests/mock/__init__.py

@@ -1 +0,0 @@
-from .filter import *

apps/cic-eth/tests/mock/filter.py

@@ -1,22 +0,0 @@
-# third-party imports
-import celery
-import moolb
-
-celery_app = celery.current_app
-
-block_filter = moolb.Bloom(1024, 3)
-tx_filter = moolb.Bloom(1024, 3)
-lo = 0
-hi = 100
-
-
-@celery_app.task()
-def filter(address, offset, limit):
-    return {
-        'alg': 'sha256',
-        'high': hi,
-        'low': lo,
-        'block_filter': block_filter.to_bytes().hex(),
-        'blocktx_filter': tx_filter.to_bytes().hex(),
-        'filter_rounds': 3,
-            }

apps/cic-eth/tests/tasks/test_balance_complex.py

@@ -1,232 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-from cic_registry import CICRegistry
-import celery
-
-# local imports
-from cic_eth.eth.rpc import RpcClient
-from cic_eth.db.models.otx import Otx
-from cic_eth.eth.util import unpack_signed_raw_tx
-
-#logg = logging.getLogger(__name__)
-logg = logging.getLogger()
-
-
-def test_balance_complex(
-        default_chain_spec,
-        init_database,
-        init_w3,
-        cic_registry,
-        dummy_token_gifted,
-        celery_session_worker,
-        init_eth_tester,
-        ):
-
-    chain_str = str(default_chain_spec)
-    token_data = {
-            'address': dummy_token_gifted,
-            'converters': [],
-            }
-
-    tx_hashes = []
-    for i in range(3):
-        s = celery.signature(
-                'cic_eth.eth.token.transfer',
-                [
-                    [token_data],
-                    init_w3.eth.accounts[0],
-                    init_w3.eth.accounts[1],
-                    1000*(i+1),
-                    chain_str,
-                    ],
-        )
-        t = s.apply_async()
-        t.get()
-        r = None
-        for c in t.collect():
-            r = c[1]
-        assert t.successful()
-        tx_hashes.append(r)
-
-        otx = Otx.load(r)
-
-        s_send = celery.signature(
-                'cic_eth.eth.tx.send',
-                [
-                    [otx.signed_tx],
-                    chain_str,
-                    ],
-                )
-        t = s_send.apply_async()
-        t.get()
-        for r in t.collect():
-            pass
-        assert t.successful()
-        init_eth_tester.mine_block() 
-
-
-    # here insert block sync to get state of balance
-
-    s_balance_base = celery.signature(
-            'cic_eth.eth.token.balance',
-            [
-                [token_data],
-                init_w3.eth.accounts[0],
-                chain_str,
-                ],
-            )
-
-    s_balance_out = celery.signature(
-            'cic_eth.queue.balance.balance_outgoing',
-            [
-                init_w3.eth.accounts[0],
-                chain_str,
-                ]
-        )
-
-    s_balance_in = celery.signature(
-            'cic_eth.queue.balance.balance_incoming',
-            [
-                init_w3.eth.accounts[0],
-                chain_str,
-                ]
-        )
-    s_balance_out.link(s_balance_in)
-    s_balance_base.link(s_balance_out)
-    t = s_balance_base.apply_async()
-    t.get()
-    r = None
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-
-    assert r[0]['balance_network'] > 0
-    assert r[0]['balance_incoming'] == 0
-    assert r[0]['balance_outgoing'] > 0
-
-    s_balance_base = celery.signature(
-            'cic_eth.eth.token.balance',
-            [
-                init_w3.eth.accounts[1],
-                chain_str,
-                ],
-            )
-
-    s_balance_out = celery.signature(
-            'cic_eth.queue.balance.balance_outgoing',
-            [
-                [token_data],
-                init_w3.eth.accounts[1],
-                chain_str,
-                ]
-        )
-
-    s_balance_in = celery.signature(
-            'cic_eth.queue.balance.balance_incoming',
-            [
-                init_w3.eth.accounts[1],
-                chain_str,
-                ]
-        )
-
-    s_balance_base.link(s_balance_in)
-    s_balance_out.link(s_balance_base)
-    t = s_balance_out.apply_async()
-    t.get()
-    r = None
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-
-    assert r[0]['balance_network'] > 0
-    assert r[0]['balance_incoming'] > 0
-    assert r[0]['balance_outgoing'] == 0
-
-    # Set confirmed status in backend
-    for tx_hash in tx_hashes:
-        rcpt = init_w3.eth.getTransactionReceipt(tx_hash)
-        assert rcpt['status'] == 1
-        otx = Otx.load(tx_hash, session=init_database)
-        otx.success(block=rcpt['blockNumber'], session=init_database)
-        init_database.add(otx)
-    init_database.commit()
-
-    
-    s_balance_base = celery.signature(
-            'cic_eth.eth.token.balance',
-            [
-                init_w3.eth.accounts[1],
-                chain_str,
-                ],
-            )
-
-    s_balance_out = celery.signature(
-            'cic_eth.queue.balance.balance_outgoing',
-            [
-                [token_data],
-                init_w3.eth.accounts[1],
-                chain_str,
-                ]
-        )
-
-    s_balance_in = celery.signature(
-            'cic_eth.queue.balance.balance_incoming',
-            [
-                init_w3.eth.accounts[1],
-                chain_str,
-                ]
-        )
-
-    s_balance_base.link(s_balance_in)
-    s_balance_out.link(s_balance_base)
-    t = s_balance_out.apply_async()
-    t.get()
-    r = None
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-    assert r[0]['balance_network'] > 0
-    assert r[0]['balance_incoming'] == 0
-    assert r[0]['balance_outgoing'] == 0
-
-    
-    s_balance_base = celery.signature(
-            'cic_eth.eth.token.balance',
-            [
-                init_w3.eth.accounts[0],
-                chain_str,
-                ],
-            )
-
-    s_balance_out = celery.signature(
-            'cic_eth.queue.balance.balance_outgoing',
-            [
-                [token_data],
-                init_w3.eth.accounts[0],
-                chain_str,
-                ]
-        )
-
-    s_balance_in = celery.signature(
-            'cic_eth.queue.balance.balance_incoming',
-            [
-                init_w3.eth.accounts[0],
-                chain_str,
-                ]
-        )
-
-    s_balance_base.link(s_balance_in)
-    s_balance_out.link(s_balance_base)
-    t = s_balance_out.apply_async()
-    t.get()
-    r = None
-    for c in t.collect():
-        r = c[1]
-    assert t.successful()
-    assert r[0]['balance_network'] > 0
-    assert r[0]['balance_incoming'] == 0
-    assert r[0]['balance_outgoing'] == 0
-
-

apps/cic-eth/tests/testdata/abi/Declarator.json

@@ -1 +0,0 @@
-[{"inputs":[{"internalType":"bytes32","name":"_initialDescription","type":"bytes32"}],"stateMutability":"nonpayable","type":"constructor"},{"inputs":[{"internalType":"address","name":"_subject","type":"address"},{"internalType":"bytes32","name":"_proof","type":"bytes32"}],"name":"addDeclaration","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"},{"internalType":"uint256","name":"","type":"uint256"}],"name":"contents","outputs":[{"internalType":"bytes32","name":"","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_declarator","type":"address"},{"internalType":"address","name":"_subject","type":"address"}],"name":"declaration","outputs":[{"internalType":"bytes32[]","name":"","type":"bytes32[]"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_declarator","type":"address"},{"internalType":"uint256","name":"_idx","type":"uint256"}],"name":"declarationAddressAt","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_declarator","type":"address"}],"name":"declarationCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_subject","type":"address"},{"internalType":"uint256","name":"_idx","type":"uint256"}],"name":"declaratorAddressAt","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_subject","type":"address"}],"name":"declaratorCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"owner","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"bytes4","name":"interfaceID","type":"bytes4"}],"name":"supportsInterface","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"transferOwnership","outputs":[],"stateMutability":"nonpayable","type":"function"}]

apps/cic-eth/tests/unit/db/test_tx.py

@@ -59,8 +59,9 @@ def test_set(
         assert (tx_stored.recipient == tx_def['to'])
         assert (tx_stored.source_token_address == bogus_from_token)
         assert (tx_stored.destination_token_address == zero_address)
-        assert (tx_stored.from_value == tx_def['value'])
-        assert (tx_stored.to_value == to_value)
+        assert (tx_stored.from_value == '1b1ae4d6e2ef500000')
+        assert (tx_stored.to_value == '0d8d726b7177a80000')
+        assert (tx_stored.values() == (tx_def['value'], to_value))
         assert (tx_stored.block_number == 666)
         assert (tx_stored.tx_index == 13)
 

apps/cic-eth/tests/unit/eth/test_bancor.py

@@ -126,5 +126,4 @@ def test_queue_cache_convert(
     assert txc.recipient == init_w3.eth.accounts[0]
     assert txc.source_token_address == bancor_tokens[0]
     assert txc.destination_token_address == bancor_tokens[1]
-    assert txc.from_value == amount
-    assert txc.to_value == amount
+    assert txc.values() == (amount, amount)

apps/cic-eth/tests/unit/eth/test_extended_tx.py

@@ -1,58 +0,0 @@
-# standard imports
-import os
-import logging
-
-# third-party imports
-import web3
-from cic_registry import CICRegistry
-
-# local imports
-from cic_eth.eth.token import ExtendedTx
-
-logg = logging.getLogger()
-
-
-def test_extended_token(
-        default_chain_spec,
-        dummy_token,
-        local_cic_registry,
-        address_declarator,
-        init_w3,
-        ):
-
-    address_foo = web3.Web3.toChecksumAddress('0x' + os.urandom(20).hex())
-    label_foo = '0x{:<064s}'.format(b'foo'.hex())
-    address_bar = web3.Web3.toChecksumAddress('0x' + os.urandom(20).hex())
-    label_bar = '0x{:<064s}'.format(b'bar'.hex())
-    label_token = '0x{:<064s}'.format(b'toktoktok'.hex())
-
-    # TODO: still need to test results with two different tokens
-    token_contract = init_w3.eth.contract(abi=CICRegistry.abi('ERC20'), address=dummy_token)
-    token = CICRegistry.add_token(default_chain_spec, token_contract)
-
-    declarator = CICRegistry.get_contract(default_chain_spec, 'AddressDeclarator', 'Declarator')
-    fn = declarator.function('addDeclaration')
-    fn(address_foo, label_foo).transact({'from': init_w3.eth.accounts[1]})
-    fn(address_bar, label_bar).transact({'from': init_w3.eth.accounts[1]})
-    fn(dummy_token, label_token).transact({'from': init_w3.eth.accounts[1]})
-    tx_hash = '0x' + os.urandom(32).hex()
-    xtx = ExtendedTx(tx_hash, default_chain_spec)
-    xtx.set_actors(address_foo, address_bar, [init_w3.eth.accounts[1]])
-    xtx.set_tokens(dummy_token, 1024)
-    tx = xtx.to_dict()
-
-    logg.debug('tx {}'.format(tx))
-    assert tx['hash'] == tx_hash
-    assert tx['source_token'] == dummy_token
-    assert tx['destination_token'] == dummy_token
-    assert tx['source_token_symbol'] == token.symbol()
-    assert tx['destination_token_symbol'] == token.symbol()
-    assert tx['source_token_value'] == 1024
-    assert tx['destination_token_value'] == 1024
-    assert tx['source_token_decimals'] == token.decimals()
-    assert tx['destination_token_decimals'] == token.decimals()
-    assert tx['sender'] == address_foo
-    assert tx['sender_label'] == 'foo'
-    assert tx['recipient'] == address_bar
-    assert tx['recipient_label'] == 'bar'
-    assert tx['chain'] == str(default_chain_spec)

apps/cic-eth/tests/unit/eth/test_token.py

@@ -85,5 +85,4 @@ def test_queue_cache_transfer(
     assert txc.recipient == init_w3.eth.accounts[1]
     assert txc.source_token_address == bancor_tokens[0]
     assert txc.destination_token_address == bancor_tokens[0]
-    assert txc.from_value == value
-    assert txc.to_value == value
+    assert txc.values() == (value, value)

apps/cic-eth/tests/unit/ext/test_address.py

@@ -1,33 +0,0 @@
-# third-party imports
-from eth_address_declarator import AddressDeclarator
-from cic_registry import CICRegistry
-
-# local imports
-from cic_eth.ext.address import translate_tx_addresses
-
-def test_translate(
-        default_chain_spec,
-        address_declarator,
-        init_rpc,
-        init_w3,
-    ):
-
-    chain_str = str(default_chain_spec)
-
-    c = init_rpc.w3.eth.contract(abi=AddressDeclarator.abi(), address=address_declarator)
-
-    description = '0x{:<064s}'.format(b'foo'.hex()) 
-    c.functions.addDeclaration(init_w3.eth.accounts[2], description).transact({'from': init_w3.eth.accounts[1]})
-    description = '0x{:<064s}'.format(b'bar'.hex()) 
-    c.functions.addDeclaration(init_w3.eth.accounts[3], description).transact({'from': init_w3.eth.accounts[1]})
-
-    tx = {
-        'sender': init_w3.eth.accounts[2],
-        'sender_label': None,
-        'recipient': init_w3.eth.accounts[3],
-        'recipient_label': None,
-
-            }
-    tx = translate_tx_addresses(tx, [init_w3.eth.accounts[1]], chain_str)
-    assert tx['sender_label'] == 'foo'
-    assert tx['recipient_label'] == 'bar'

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -1,109 +0,0 @@
-# standard imports
-import logging
-
-# third-party imports
-import celery
-import moolb
-
-# local imports
-from cic_eth.eth.token import TokenTxFactory
-from cic_eth.eth.task import sign_tx
-
-logg = logging.getLogger()
-
-
-# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state
-def test_filter_process(
-        init_rpc,
-        default_chain_spec,
-        default_chain_registry,
-        celery_session_worker,
-        init_eth_tester,
-        init_w3,
-        dummy_token_gifted,
-        cic_registry,
-        ):
-
-    b = moolb.Bloom(1024, 3)
-    t = moolb.Bloom(1024, 3)
-
-    tx_hashes = []
-    # external tx
-    init_eth_tester.mine_blocks(13)
-    txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc)
-    tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 3000, default_chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec))
-    tx_hashes.append(tx_hash_hex)
-    init_w3.eth.sendRawTransaction(tx_signed_raw_hex)
-    # add to filter
-    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex)
-    a = rcpt['blockNumber']
-    b.add(a.to_bytes(4, 'big'))
-    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-    t.add(a.to_bytes(4, 'big'))
-
-    # external tx
-    init_eth_tester.mine_blocks(28)
-    txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc)
-    tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 4000, default_chain_spec)
-    (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec))
-    tx_hashes.append(tx_hash_hex)
-    init_w3.eth.sendRawTransaction(tx_signed_raw_hex)
-    # add to filter
-    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex)
-    a = rcpt['blockNumber']
-    b.add(a.to_bytes(4, 'big'))
-    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-    t.add(a.to_bytes(4, 'big'))
-
-#    init_eth_tester.mine_blocks(13)
-#    tx_hash_one = init_w3.eth.sendTransaction({
-#        'from': init_w3.eth.accounts[2],
-#        'to': init_w3.eth.accounts[1],
-#        'value': 1024,
-#        })
-#    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_one)
-#    a = rcpt['blockNumber']
-#    b.add(a.to_bytes(4, 'big'))
-#    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-#    t.add(a.to_bytes(4, 'big'))
-#
-#    init_eth_tester.mine_blocks(28)
-#    tx_hash_two = init_w3.eth.sendTransaction({
-#        'from': init_w3.eth.accounts[3],
-#        'to': init_w3.eth.accounts[1],
-#        'value': 2048,
-#        })
-#    rcpt = init_w3.eth.getTransactionReceipt(tx_hash_two)
-#    a = rcpt['blockNumber']
-#    b.add(a.to_bytes(4, 'big'))
-#    a = rcpt['blockNumber'] + rcpt['transactionIndex']
-#    t.add(a.to_bytes(4, 'big'))
-
-    init_eth_tester.mine_blocks(10)
-        
-    o = {
-        'alg': 'sha256',
-        'filter_rounds': 3,
-        'low': 0,
-        'high': 50,
-        'block_filter': b.to_bytes().hex(),
-        'blocktx_filter': t.to_bytes().hex(),
-            }
-
-    s = celery.signature(
-            'cic_eth.ext.tx.list_tx_by_bloom',
-        [
-            o,
-            init_w3.eth.accounts[1],
-            str(default_chain_spec),
-            ],
-        queue=None
-        )
-    t = s.apply_async()
-    r = t.get() 
-
-    assert len(r) == 2
-    for tx_hash in r.keys():
-        tx_hashes.remove(tx_hash)
-    assert len(tx_hashes) == 0

apps/cic-eth/tests/unit/queue/test_balances.py

@@ -1,158 +0,0 @@
-# standard imports
-import os
-import logging
-
-# third-party imports
-import pytest
-
-# local imports
-from cic_eth.db.models.otx import Otx
-from cic_eth.db.models.tx import TxCache
-from cic_eth.queue.balance import (
-        balance_outgoing,
-        balance_incoming,
-        assemble_balances,
-        )
-
-logg = logging.getLogger()
-
-
-def test_assemble():
-   
-    token_foo = '0x' + os.urandom(20).hex()
-    token_bar = '0x' + os.urandom(20).hex()
-    b = [
-            [
-        {
-            'address': token_foo,
-            'converters': [],
-            'balance_foo': 42,
-            },
-                {
-            'address': token_bar,
-            'converters': [],
-            'balance_baz': 666,
-            },
-                ],
-            [
-        {
-            'address': token_foo,
-            'converters': [],
-            'balance_bar': 13,
-            },
-
-        {
-            'address': token_bar,
-            'converters': [],
-            'balance_xyzzy': 1337,
-            }
-            ]
-            ]
-    r = assemble_balances(b)
-    logg.debug('r {}'.format(r))
-
-    assert r[0]['address'] == token_foo
-    assert r[1]['address'] == token_bar
-    assert r[0].get('balance_foo') != None
-    assert r[0].get('balance_bar') != None
-    assert r[1].get('balance_baz') != None
-    assert r[1].get('balance_xyzzy') != None
-
-
-@pytest.mark.skip()
-def test_outgoing_balance(
-        default_chain_spec,
-        init_database,
-        ):
-
-    chain_str = str(default_chain_spec)
-    recipient = '0x' + os.urandom(20).hex()
-    tx_hash = '0x' + os.urandom(32).hex()
-    signed_tx = '0x' + os.urandom(128).hex()
-    otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database)
-    init_database.add(otx)
-    init_database.commit()
-    
-    token_address = '0x' + os.urandom(20).hex()
-    sender = '0x' + os.urandom(20).hex()
-    txc = TxCache(
-            tx_hash,
-            sender,
-            recipient,
-            token_address,
-            token_address,
-            1000,
-            1000,
-            )
-    init_database.add(txc)
-    init_database.commit()
-
-    token_data = {
-            'address': token_address,
-            'converters': [],
-            }
-    b = balance_outgoing([token_data], sender, chain_str)
-    assert b[0]['balance_outgoing'] == 1000
-
-    otx.sent(session=init_database)
-    init_database.commit()
-
-    b = balance_outgoing([token_data], sender, chain_str)
-    assert b[0]['balance_outgoing'] == 1000
-    
-    otx.success(block=1024, session=init_database)
-    init_database.commit()
-
-    b = balance_outgoing([token_data], sender, chain_str)
-    assert b[0]['balance_outgoing'] == 0
-
-
-@pytest.mark.skip()
-def test_incoming_balance(
-        default_chain_spec,
-        init_database,
-        ):
-
-    chain_str = str(default_chain_spec)
-    recipient = '0x' + os.urandom(20).hex()
-    tx_hash = '0x' + os.urandom(32).hex()
-    signed_tx = '0x' + os.urandom(128).hex()
-    otx = Otx.add(0, recipient, tx_hash, signed_tx, session=init_database)
-    init_database.add(otx)
-    init_database.commit()
-    
-    token_address = '0x' + os.urandom(20).hex()
-    sender = '0x' + os.urandom(20).hex()
-    txc = TxCache(
-            tx_hash,
-            sender,
-            recipient,
-            token_address,
-            token_address,
-            1000,
-            1000,
-            )
-    init_database.add(txc)
-    init_database.commit()
-
-    token_data = {
-            'address': token_address,
-            'converters': [],
-            }
-    b = balance_incoming([token_data], recipient, chain_str)
-    assert b[0]['balance_incoming'] == 0
-
-    otx.sent(session=init_database)
-    init_database.commit()
-
-    b = balance_incoming([token_data], recipient, chain_str)
-    assert b[0]['balance_incoming'] == 1000
-   
-    otx.success(block=1024, session=init_database)
-    init_database.commit()
-
-    b = balance_incoming([token_data], recipient, chain_str)
-    assert b[0]['balance_incoming'] == 0
-
-
-

apps/contract-migration/docker/Dockerfile

@@ -22,7 +22,7 @@ RUN echo Install confini schema files && \
 ENV CONFINI_DIR /usr/local/etc/cic
 
 
-ARG cic_contracts_commit=698ef3a30fde8d7f2c498f1208fb0ff45d665501
+ARG cic_contracts_commit=9933979e14484e826523df23a0b11480799f6380
 ARG cic_contracts_url=https://gitlab.com/grassrootseconomics/cic-contracts.git/
 RUN echo Install ABI collection for solidity interfaces used across all components && \
 	git clone --depth 1 $cic_contracts_url cic-contracts && \
@@ -106,7 +106,7 @@ RUN cd cic-bancor/python && \
 	pip install --extra-index-url $pip_extra_index_url .
 
 RUN echo installing common python tooling
-ARG cic_python_commit=beecee783ceac2ea0fa711f888ce4c82f1a81490
+ARG cic_python_commit=4573afb7ad4d635c2f3330b8b9b25806523b7e14
 ARG cic_python_url=https://gitlab.com/grassrootseconomics/cic-python.git/
 RUN echo Install sum of python dependencies across all components && \
 	git clone --depth 1 $cic_python_url cic-python && \
@@ -120,26 +120,23 @@ ARG cryptocurrency_cli_tools_version=0.0.4
 RUN pip install --extra-index-url $pip_extra_index_url cryptocurrency-cli-tools==$cryptocurrency_cli_tools_version
 
 RUN echo Install smart contract interface implementations, least frequently changed first
-ARG giftable_erc20_token_version=0.0.7b10
+ARG giftable_erc20_token_version=0.0.7b7
 RUN pip install --extra-index-url $pip_extra_index_url giftable-erc20-token==$giftable_erc20_token_version 
 
 ARG eth_accounts_index_version=0.0.10a6
 RUN pip install --extra-index-url $pip_extra_index_url eth-accounts-index==$eth_accounts_index_version
 
-ARG erc20_approval_escrow_version=0.3.0a4
-RUN pip install --extra-index-url $pip_extra_index_url erc20-approval-escrow==$erc20_approval_escrow_version 
+ARG erc20_transfer_authorization_version=0.3.0a8
+RUN pip install --extra-index-url $pip_extra_index_url erc20-transfer-authorization==$erc20_transfer_authorization_version 
 
-#ARG erc20_single_shot_faucet_version=0.2.0a5
-#RUN pip install --extra-index-url $pip_extra_index_url erc20-single-shot-faucet==$erc20_single_shot_faucet_version 
-
-ARG sarafu_faucet_version==0.0.1a10
-RUN pip install --extra-index-url $pip_extra_index_url sarafu-faucet==$sarafu_faucet_version
+ARG erc20_single_shot_faucet_version=0.2.0a5
+RUN pip install --extra-index-url $pip_extra_index_url erc20-single-shot-faucet==$erc20_single_shot_faucet_version 
 
 ARG eth_address_index_version==0.1.0a8
 RUN pip install --extra-index-url $pip_extra_index_url eth-address-index==$eth_address_index_version
 
 RUN echo Install cic specific python packages
-ARG cic_registry_version=0.5.3a18
+ARG cic_registry_version=0.5.3a11
 RUN pip install  --extra-index-url  $pip_extra_index_url  cic-registry==$cic_registry_version
 
 RUN echo Install misc helpers
@@ -150,9 +147,6 @@ RUN pip install  --extra-index-url $pip_extra_index_url crypto-dev-signer==$cryp
 ARG eth_gas_proxy_version==0.0.1a4
 RUN pip install  --extra-index-url $pip_extra_index_url eth-gas-proxy==$eth_gas_proxy_version
 
-ARG cic_contracts_version==0.0.2a2
-RUN pip install  --extra-index-url $pip_extra_index_url cic-contracts==$cic_contracts_version
-
 WORKDIR /root
 
 COPY contract-migration/testdata/pgp testdata/pgp

apps/contract-migration/reset.sh

@@ -29,7 +29,7 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	echo "waiting for ${ETH_PROVIDER}..."
   	./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
 
-	DEV_ETH_RESERVE_ADDRESS=`giftable-token-deploy -p $ETH_PROVIDER -y $keystore_file -i $CIC_CHAIN_SPEC --account $DEV_ETH_ACCOUNT_RESERVE_MINTER --minter $DEV_ETH_ACCOUNT_RESERVE_MINTER --minter $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -v -w --name "Sarafu" --symbol "SRF" --decimals 6 $DEV_ETH_RESERVE_AMOUNT`
+	DEV_ETH_RESERVE_ADDRESS=`giftable-token-deploy -p $ETH_PROVIDER -y $keystore_file -i $CIC_CHAIN_SPEC --account $DEV_ETH_ACCOUNT_RESERVE_MINTER --minter $DEV_ETH_ACCOUNT_RESERVE_MINTER -v -w --name "Sarafu" --symbol "SRF" --decimals 6 $DEV_ETH_RESERVE_AMOUNT`
 
 	#BANCOR_REGISTRY_ADDRESS=`cic-bancor-deploy --bancor-dir /usr/local/share/cic/bancor -z $DEV_ETH_RESERVE_ADDRESS -p $ETH_PROVIDER -o $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER`
 
@@ -44,7 +44,6 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 	>&2 echo "deploy address declarator contract"
 	declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
 	CIC_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -v $declarator_description`
-	cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -k AddressDeclarator -p $ETH_PROVIDER $CIC_DECLARATOR_ADDRESS  -vv
 
 else
 	echo "\$ETH_PROVIDER not set!"
@@ -60,7 +59,6 @@ export DEV_ETH_RESERVE_AMOUNT=$DEV_ETH_RESERVE_AMOUNT
 export DEV_ETH_ACCOUNTS_INDEX_ADDRESS=$CIC_ACCOUNTS_INDEX_ADDRESS
 export BANCOR_REGISTRY_ADDRESS=$BANCOR_REGISTRY_ADDRESS
 export CIC_REGISTRY_ADDRESS=$CIC_REGISTRY_ADDRESS
-export CIC_TRUST_ADDRESS=$DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 EOF
 

apps/contract-migration/seed_cic_eth.sh

@@ -29,7 +29,7 @@ truncate $env_out_file -s 0
 set -e
 set -a
 
-pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a25 chainlib==0.0.1a11
+pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL cic-eth==0.10.0a25 chainlib==0.0.1a4
 
 >&2 echo "create account for gas gifter"
 old_gas_provider=$DEV_ETH_ACCOUNT_GAS_PROVIDER
@@ -47,10 +47,10 @@ DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=`cic-eth-create $debug --redis-host
 echo DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER=$DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER >> $env_out_file
 cic-eth-tag TRANSFER_AUTHORIZATION_OWNER $DEV_ETH_ACCOUNT_TRANSFER_AUTHORIZATION_OWNER 
 
-#>&2 echo "create account for faucet owner"
-#DEV_ETH_ACCOUNT_FAUCET_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
-#echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_FAUCET_OWNER >> $env_out_file
-#cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER
+>&2 echo "create account for faucet owner"
+DEV_ETH_ACCOUNT_FAUCET_OWNER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
+echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_FAUCET_OWNER >> $env_out_file
+cic-eth-tag FAUCET_GIFTER $DEV_ETH_ACCOUNT_FAUCET_OWNER
 
 >&2 echo "create account for accounts index writer"
 DEV_ETH_ACCOUNT_ACCOUNTS_INDEX_WRITER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
@@ -98,7 +98,7 @@ export CIC_TRANSFER_AUTHORIZATION_ADDRESS=$CIC_TRANSFER_AUTHORIZATION_ADDRESS
 
 # Deploy one-time token faucet for newly created token
 >&2 echo "deploy faucet"
-DEV_ETH_SARAFU_FAUCET_ADDRESS=`sarafu-faucet-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS --editor $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER --set-amount $faucet_amount --accounts-index-address $DEV_ETH_ACCOUNTS_INDEX_ADDRESS -w $debug`
+DEV_ETH_SARAFU_FAUCET_ADDRESS=`erc20-single-shot-faucet-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS --editor $DEV_ETH_ACCOUNT_FAUCET_OWNER --set-amount $faucet_amount -w $debug`
 echo DEV_ETH_SARAFU_FAUCET_ADDRESS=$DEV_ETH_SARAFU_FAUCET_ADDRESS  >> $env_out_file
 export DEV_ETH_SARAFU_FAUCET_ADDRESS=$DEV_ETH_SARAFU_FAUCET_ADDRESS 
 
@@ -110,8 +110,6 @@ export DEV_ETH_SARAFU_FAUCET_ADDRESS=$DEV_ETH_SARAFU_FAUCET_ADDRESS
 >&2 echo "register faucet contract in registry"
 >&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k Faucet -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug $DEV_ETH_SARAFU_FAUCET_ADDRESS
 
->&2 echo "add faucet contract as token minter"
->&2 giftable-token-add -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug --token-address $DEV_ETH_SARAFU_TOKEN_ADDRESS $DEV_ETH_SARAFU_FAUCET_ADDRESS
 
 >&2 echo "deploy token symbol index contract"
 CIC_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -y $keystore_file -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w $debug`
@@ -131,7 +129,7 @@ token_description_two=0x54686973206973207468652053617261667520746f6b656e00000000
 
 
 # Register address declarator
->&2 echo "add address declarator to registry"
+>&2 echo "registry address declarator to registry"
 >&2 cic-registry-set -y $keystore_file -r $CIC_REGISTRY_ADDRESS -k AddressDeclarator -i $CIC_CHAIN_SPEC -w -p $ETH_PROVIDER $CIC_DECLARATOR_ADDRESS 
 
 # We're done with the registry at this point, seal it off

apps/requirements.txt

@@ -9,7 +9,7 @@ ecuth==0.4.5a1
 eth-accounts-index==0.0.10a7
 eth-address-index==0.1.0a7
 eth-tester==0.5.0b3
-erc20-approval-escrow==0.3.0a5
+erc20-transfer-authorization==0.3.0a7
 erc20-single-shot-faucet==0.2.0a6
 faker==4.17.1
 http-hoba-auth==0.2.0
@@ -41,5 +41,3 @@ yaml-acl==0.0.1
 rlp==2.0.1
 cryptocurrency-cli-tools==0.0.4
 giftable-erc20-token==0.0.7b7
-hexathon==0.0.1a3
-chainlib==0.0.1a12

docker-compose.yml

@@ -117,9 +117,9 @@ services:
       CELERY_RESULT_URL: ${CELERY_RESULT_URL:-redis://redis:6379}
       DEV_PIP_EXTRA_INDEX_URL: ${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
     command: ["./seed_cic_eth.sh"]
-      #  deploy:
-      #restart_policy:
-      #  condition: on-failure
+    deploy:
+      restart_policy:
+        condition: on-failure
     depends_on:
       - eth
       - postgres
@@ -129,71 +129,56 @@ services:
       - contract-config:/tmp/cic/config
 
 
-  cic-cache-tracker:
-    build:
-      context: apps/cic-cache/
-      dockerfile: docker/Dockerfile
-    environment:
-      CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
-      ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
-      DATABASE_USER: ${DATABASE_USER:-grassroots}
-      DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
-      DATABASE_HOST: ${DATABASE_HOST:-postgres}
-      DATABASE_PORT: ${DATABASE_PORT:-5432}
-      DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
-      DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
-      DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: 1
-      ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
-      CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C}
-      CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
-      CELERY_BROKER_URL: redis://redis:6379
-      CELERY_RESULT_URL: redis://redis:6379
-    deploy: 
-      restart_policy:
-        condition: on-failure
-    depends_on:
-      - redis
-      - postgres
-      - eth
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        /usr/local/bin/cic-cache-tracker -vv
-    volumes:
-      - contract-config:/tmp/cic/config/:ro
+  # cic-cache-tracker:
+  #   # image: registry.gitlab.com/grassrootseconomics/cic-cache:master-latest
+  #   build: apps/cic-cache
+  #   environment:
+  #     CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning
+  #     ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545}
+  #     BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor}
+  #     DATABASE_USER: ${DATABASE_USER:-grassroots}
+  #     DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql
+  #     DATABASE_HOST: ${DATABASE_HOST:-postgres}
+  #     DATABASE_PORT: ${DATABASE_PORT:-5432}
+  #     DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache}
+  #     DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
+  #     DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
+  #     ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi}
+  #   deploy: 
+  #     restart_policy:
+  #       condition: on-failure
+  #   depends_on:
+  #     - postgres
+  #     - eth
+  #   command:
+  #      - /bin/sh
+  #      - -c
+  #      - |
+  #       if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi
+  #       /usr/local/bin/cic-cache-tracker -vv
+  #   volumes:
+  #     - contract-config:/tmp/cic/config/:ro
+    # entrypoint: ["/usr/local/bin/cic-cache-tracker", "-vv"]
+    # command: "/usr/local/bin/cic-cache-tracker -vv"
 
-  cic-cache-server:
-    build: 
-        context: apps/cic-cache/
-        dockerfile: docker/Dockerfile
-    environment:
-      DATABASE_USER: $DATABASE_USER
-      DATABASE_HOST: $DATABASE_HOST
-      DATABASE_PORT: $DATABASE_PORT
-      DATABASE_PASSWORD: $DATABASE_PASSWORD
-      DATABASE_NAME: $DATABASE_NAME_CIC_CACHE
-      DATABASE_DEBUG: 1
-      PGPASSWORD: $DATABASE_PASSWORD
-      SERVER_PORT: 8000
-    ports:
-      - ${HTTP_PORT_CIC_CACHE:-63313}:8000
-    depends_on:
-      - postgres
-    deploy:
-      restart_policy:
-        condition: on-failure
-    command:
-      - /bin/bash
-      - -c
-      - |
-        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-        "/usr/local/bin/uwsgi" \
-        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \
-        --http :80 \
-        --pyargv -vv
+#   cic-cache-server:
+#     image: grassrootseconomics:cic-cache-uwsgi
+#     environment:
+#       DATABASE_USER: $DATABASE_USER
+#       DATABASE_HOST: $DATABASE_HOST
+#       DATABASE_PORT: $DATABASE_PORT
+#       DATABASE_PASSWORD: $DATABASE_PASSWORD
+#       DATABASE_NAME: $DATABASE_NAME_CIC_CACHE
+#       PGPASSWORD: $DATABASE_PASSWORD
+#       SERVER_PORT: 80
+#     ports:
+#       - ${HTTP_PORT_CIC_CACHE}:80
+#     depends_on:
+#       - postgres
+#     deploy:
+#       restart_policy:
+#         condition: on-failure
+#     command: "/root/start_uwsgi.sh"
 
   cic-eth-tasker:
     # image: grassrootseconomics:cic-eth-service
@@ -212,7 +197,6 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       PGPASSWORD: ${DATABASE_PASSWORD:-tralala}
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
       BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor}
@@ -253,8 +237,6 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-        #DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
-      DATABASE_DEBUG: 1
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
@@ -291,7 +273,6 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
@@ -328,7 +309,6 @@ services:
       DATABASE_PORT: ${DATABASE_PORT:-5432}
       DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres}
       DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2}
-      DATABASE_DEBUG: ${DATABASE_DEBUG:-0}
       CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996}
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
         #BANCOR_DIR: $BANCOR_DIR
@@ -393,42 +373,42 @@ services:
     # command: "/root/start_retry.sh -q cic-eth -vv"
 
 
-#  cic-eth-server:
-#    build:
-#      context: apps/
-#      dockerfile: cic-eth/docker/Dockerfile
-#    environment:
-#      CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
-#      CELERY_BROKER_URL: $CELERY_BROKER_URL
-#      CELERY_RESULT_URL: $CELERY_RESULT_URL
-#      SERVER_PORT: 8000
-#    depends_on:
-#      - eth
-#      - postgres
-#      - redis
-#    ports:
-#      - ${HTTP_PORT_CIC_ETH:-63314}:8000
-#    deploy:
-#      restart_policy:
-#        condition: on-failure
-#    volumes:
-#        - contract-config:/tmp/cic/config/:ro
-#    command:
-#      - /bin/bash
-#      - -c
-#      - |
-#        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
-#        "/usr/local/bin/uwsgi" \
-#        --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \
-#        --http :80 \
-#        --pyargv -vv
-##     entrypoint:
-##       - "/usr/local/bin/uwsgi"
-##       - "--wsgi-file"
-##       - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py"
-##       - "--http"
-##       - ":80"
-#    # command: "--pyargv -vv"
+  cic-eth-server:
+    build:
+      context: apps/
+      dockerfile: cic-eth/docker/Dockerfile
+    environment:
+      CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC
+      CELERY_BROKER_URL: $CELERY_BROKER_URL
+      CELERY_RESULT_URL: $CELERY_RESULT_URL
+      SERVER_PORT: 8000
+    depends_on:
+      - eth
+      - postgres
+      - redis
+    ports:
+      - ${HTTP_PORT_CIC_ETH:-63314}:8000
+    deploy:
+      restart_policy:
+        condition: on-failure
+    volumes:
+        - contract-config:/tmp/cic/config/:ro
+    command:
+      - /bin/bash
+      - -c
+      - |
+        if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
+        "/usr/local/bin/uwsgi" \
+        --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \
+        --http :80 \
+        --pyargv -vv
+#     entrypoint:
+#       - "/usr/local/bin/uwsgi"
+#       - "--wsgi-file"
+#       - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py"
+#       - "--http"
+#       - ":80"
+    # command: "--pyargv -vv"