adding cic-eth as sub dir

apps/cic-eth/cic_eth/db/models/base.py

@@ -0,0 +1,73 @@
+# third-party imports
+from sqlalchemy import Column, Integer
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+Model = declarative_base(name='Model')
+
+
+class SessionBase(Model):
+    """The base object for all SQLAlchemy enabled models. All other models must extend this.
+    """
+    __abstract__ = True
+ 
+    id = Column(Integer, primary_key=True)
+
+    engine = None
+    """Database connection engine of the running aplication"""
+    sessionmaker = None
+    """Factory object responsible for creating sessions from the  connection pool"""
+    transactional = True
+    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
+    poolable = True
+    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
+
+
+    @staticmethod
+    def create_session():
+        """Creates a new database session.
+        """
+        return SessionBase.sessionmaker()
+
+
+    @staticmethod
+    def _set_engine(engine):
+        """Sets the database engine static property
+        """
+        SessionBase.engine = engine
+        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
+
+
+    @staticmethod
+    def connect(dsn, debug=False):
+        """Create new database connection engine and connect to database backend.
+
+        :param dsn: DSN string defining connection.
+        :type dsn: str
+        """
+        e = None
+        if SessionBase.poolable:
+            e = create_engine(
+                    dsn,
+                    max_overflow=50,
+                    pool_pre_ping=True,
+                    pool_size=20,
+                    pool_recycle=10,
+                    echo=debug,
+                )
+        else:
+            e = create_engine(
+                    dsn,
+                    echo=debug,
+                    )
+
+        SessionBase._set_engine(e)
+
+
+    @staticmethod
+    def disconnect():
+        """Disconnect from database and free resources.
+        """
+        SessionBase.engine.dispose()
+        SessionBase.engine = None

apps/cic-eth/cic_eth/db/models/convert.py

@@ -0,0 +1,55 @@
+# third-party imports
+from sqlalchemy import Column, Enum, String, Integer
+from sqlalchemy.ext.hybrid import hybrid_method
+
+# local imports
+from .base import SessionBase
+from ..error import UnknownConvertError
+
+
+class TxConvertTransfer(SessionBase):
+    """Table describing a transfer of funds after conversion has been successfully performed.
+
+    :param convert_tx_hash: Transaction hash of convert transaction
+    :type convert_tx_hash: str, 0x-hex
+    :param recipient_address: Ethereum address of recipient of resulting token balance of conversion
+    :type recipient_address: str, 0x-hex
+    :param chain_str: Chain spec string representation
+    :type chain_str: str
+    """
+    __tablename__ = 'tx_convert_transfer'
+
+    #approve_tx_hash = Column(String(66))
+    convert_tx_hash = Column(String(66))
+    transfer_tx_hash = Column(String(66))
+    recipient_address = Column(String(42))
+
+    
+    @hybrid_method
+    def transfer(self, transfer_tx_hash):
+        """Persists transaction hash of performed transfer. Setting this ends the lifetime of this record.
+        """
+        self.transfer_tx_hash = transfer_tx_hash
+
+
+    @staticmethod
+    def get(convert_tx_hash):
+        """Retrieves a convert transfer record by conversion transaction hash in a separate session.
+
+        :param convert_tx_hash: Transaction hash of convert transaction
+        :type convert_tx_hash: str, 0x-hex
+        """
+        session = SessionBase.create_session()
+        q = session.query(TxConvertTransfer)
+        q = q.filter(TxConvertTransfer.convert_tx_hash==convert_tx_hash)
+        r = q.first()
+        session.close()
+        if r == None:
+            raise UnknownConvertError(convert_tx_hash)
+        return r
+
+
+    def __init__(self, convert_tx_hash, recipient_address, chain_str):
+        self.convert_tx_hash = convert_tx_hash
+        self.recipient_address = recipient_address
+        self.chain_str = chain_str

apps/cic-eth/cic_eth/db/models/lock.py

@@ -0,0 +1,190 @@
+# standard imports
+import datetime
+import logging
+
+# third-party imports
+from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
+from cic_registry import zero_address
+
+# local imports
+from cic_eth.db.models.base import SessionBase
+from cic_eth.db.models.tx import TxCache
+from cic_eth.db.models.otx import Otx
+
+logg = logging.getLogger()
+
+
+class Lock(SessionBase):
+    """Deactivate functionality on a global or per-account basis
+
+    """
+
+    __tablename__ = "lock"
+
+    blockchain = Column(String)
+    address = Column(String, ForeignKey('tx_cache.sender'))
+    flags = Column(Integer)
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    otx_id = Column(Integer, ForeignKey('otx.id'))
+
+
+    def chain(self):
+        """Get chain the cached instance represents.
+        """
+        return self.blockchain
+
+
+    @staticmethod
+    def set(chain_str, flags, address=zero_address, session=None, tx_hash=None):
+        """Sets flags associated with the given address and chain.
+
+        If a flags entry does not exist it is created.
+
+        Does not validate the address against any other tables or components.
+
+        Valid flags can be found in cic_eth.db.enum.LockEnum.
+
+        :param chain_str: Chain spec string representation
+        :type str: str
+        :param flags: Flags to set
+        :type flags: number
+        :param address: Ethereum address
+        :type address: str, 0x-hex
+        :param session: Database session, if None a separate session will be used.
+        :type session: SQLAlchemy session
+        :returns: New flag state of entry
+        :rtype: number
+        """
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        q = localsession.query(Lock)
+        #q = q.join(TxCache, isouter=True)
+        q = q.filter(Lock.address==address)
+        q = q.filter(Lock.blockchain==chain_str)
+        lock = q.first()
+        
+        if lock == None:
+            lock = Lock()
+            lock.flags = 0
+            lock.address = address
+            lock.blockchain = chain_str
+            if tx_hash != None:
+                q = localsession.query(Otx)
+                q = q.filter(Otx.tx_hash==tx_hash)
+                otx = q.first()
+                if otx != None:
+                    lock.otx_id = otx.id
+
+        lock.flags |= flags
+        r = lock.flags
+
+        localsession.add(lock)
+        localsession.commit()
+
+        if session == None:
+            localsession.close()
+        
+        return r
+
+
+    @staticmethod
+    def reset(chain_str, flags, address=zero_address, session=None):
+        """Resets flags associated with the given address and chain.
+
+        If the resulting flags entry value is 0, the entry will be deleted.
+
+        Does not validate the address against any other tables or components.
+
+        Valid flags can be found in cic_eth.db.enum.LockEnum.
+
+        :param chain_str: Chain spec string representation
+        :type str: str
+        :param flags: Flags to set
+        :type flags: number
+        :param address: Ethereum address
+        :type address: str, 0x-hex
+        :param session: Database session, if None a separate session will be used.
+        :type session: SQLAlchemy session
+        :returns: New flag state of entry
+        :rtype: number
+        """
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        q = localsession.query(Lock)
+        #q = q.join(TxCache, isouter=True)
+        q = q.filter(Lock.address==address)
+        q = q.filter(Lock.blockchain==chain_str)
+        lock = q.first()
+
+        r = 0
+        if lock != None:
+            lock.flags &= ~flags
+            if lock.flags == 0:
+                localsession.delete(lock)
+            else:
+                localsession.add(lock)
+                r = lock.flags
+            localsession.commit()
+
+        if session == None:
+            localsession.close()
+
+        return r
+
+
+    @staticmethod
+    def check(chain_str, flags, address=zero_address, session=None):
+        """Checks whether all given flags are set for given address and chain. 
+
+        Does not validate the address against any other tables or components.
+
+        Valid flags can be found in cic_eth.db.enum.LockEnum.
+
+        :param chain_str: Chain spec string representation
+        :type str: str
+        :param flags: Flags to set
+        :type flags: number
+        :param address: Ethereum address
+        :type address: str, 0x-hex
+        :param session: Database session, if None a separate session will be used.
+        :type session: SQLAlchemy session
+        :returns: Returns the value of all flags matched
+        :rtype: number
+        """
+
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        q = localsession.query(Lock)
+        #q = q.join(TxCache, isouter=True)
+        q = q.filter(Lock.address==address)
+        q = q.filter(Lock.blockchain==chain_str)
+        q = q.filter(Lock.flags.op('&')(flags)==flags)
+        lock = q.first()
+        if session == None:
+            localsession.close()
+       
+        r = 0
+        if lock != None:
+            r = lock.flags & flags
+        return r
+
+
+    @staticmethod
+    def check_aggregate(chain_str, flags, address, session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        r = Lock.check(chain_str, flags, session=localsession) 
+        r |= Lock.check(chain_str, flags, address=address, session=localsession) 
+
+        if session == None:
+            localsession.close()
+
+        return r

apps/cic-eth/cic_eth/db/models/nonce.py

@@ -0,0 +1,86 @@
+# standard imports
+import logging
+
+# third-party imports
+from sqlalchemy import Column, String, Integer
+
+# local imports
+from .base import SessionBase
+
+logg = logging.getLogger()
+
+
+class Nonce(SessionBase):
+    """Provides thread-safe nonce increments.
+    """
+    __tablename__ = 'nonce'
+
+    nonce = Column(Integer)
+    address_hex = Column(String(42))
+
+
+    @staticmethod
+    def get(address, session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+
+        q = localsession.query(Nonce)
+        q = q.filter(Nonce.address_hex==address)
+        nonce = q.first()
+
+        nonce_value = None
+        if nonce != None:
+            nonce_value = nonce.nonce;
+
+        if session == None:
+            localsession.close()
+
+        return nonce_value
+
+
+    @staticmethod
+    def __get(conn, address):
+        r = conn.execute("SELECT nonce FROM nonce WHERE address_hex = '{}'".format(address))
+        nonce = r.fetchone()
+        if nonce == None:
+            return None
+        return nonce[0]
+
+
+    @staticmethod
+    def __set(conn, address, nonce):
+        conn.execute("UPDATE nonce set nonce = {} WHERE address_hex = '{}'".format(nonce, address))
+
+
+    @staticmethod
+    def next(address, initial_if_not_exists=0):
+        """Generate next nonce for the given address.
+
+        If there is no previous nonce record for the address, the nonce may be initialized to a specified value, or 0 if no value has been given.
+
+        :param address: Associate Ethereum address 
+        :type address: str, 0x-hex
+        :param initial_if_not_exists: Initial nonce value to set if no record exists
+        :type initial_if_not_exists: number
+        :returns: Nonce
+        :rtype: number
+        """
+        conn = Nonce.engine.connect()
+        if Nonce.transactional:
+            conn.execute('BEGIN')
+            conn.execute('LOCK TABLE nonce IN SHARE ROW EXCLUSIVE MODE')
+        nonce = Nonce.__get(conn, address)
+        logg.debug('get nonce {} for address {}'.format(nonce, address))
+        if nonce == None:
+            nonce = initial_if_not_exists
+            conn.execute("INSERT INTO nonce (nonce, address_hex) VALUES ({}, '{}')".format(nonce, address))
+            logg.debug('setting default nonce to {} for address {}'.format(nonce, address))
+        Nonce.__set(conn, address, nonce+1)
+        if Nonce.transactional:
+            conn.execute('COMMIT')
+        conn.close()
+        return nonce
+
+

apps/cic-eth/cic_eth/db/models/otx.py

@@ -0,0 +1,455 @@
+# standard imports
+import datetime
+import logging
+
+# third-party imports
+from sqlalchemy import Column, Enum, String, Integer, DateTime, Text, or_, ForeignKey
+from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
+
+# local imports
+from .base import SessionBase
+from cic_eth.db.enum import StatusEnum
+from cic_eth.db.error import TxStateChangeError
+#from cic_eth.eth.util import address_hex_from_signed_tx
+
+logg = logging.getLogger()
+
+
+class OtxStateLog(SessionBase):
+
+    __tablename__ = 'otx_state_log'
+
+    date = Column(DateTime, default=datetime.datetime.utcnow)
+    status = Column(Integer)
+    otx_id = Column(Integer, ForeignKey('otx.id'))
+
+
+    def __init__(self, otx):
+        self.otx_id = otx.id
+        self.status = otx.status
+
+
+class Otx(SessionBase):
+    """Outgoing transactions with local origin.
+
+    :param nonce: Transaction nonce
+    :type nonce: number
+    :param address: Ethereum address of recipient - NOT IN USE, REMOVE
+    :type address: str
+    :param tx_hash: Tranasction hash 
+    :type tx_hash: str, 0x-hex
+    :param signed_tx: Signed raw transaction data
+    :type signed_tx: str, 0x-hex
+    """
+    __tablename__ = 'otx'
+
+    tracing = False
+    """Whether to enable queue state tracing"""
+
+    nonce = Column(Integer)
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    tx_hash = Column(String(66))
+    signed_tx = Column(Text)
+    status = Column(Integer)
+    block = Column(Integer)
+
+
+    def __set_status(self, status, session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        self.status = status
+        localsession.add(self)
+        localsession.flush()
+
+        if self.tracing:
+            self.__state_log(session=localsession)
+
+        if session==None:
+            localsession.commit()
+            localsession.close()
+
+
+    def set_block(self, block, session=None):
+        """Set block number transaction was mined in.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :param block: Block number
+        :type block: number
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        if self.block != None:
+            raise TxStateChangeError('Attempted set block {} when block was already {}'.format(block, self.block))
+        self.block = block
+        localsession.add(self)
+        localsession.flush()
+
+        if session==None:
+            localsession.commit()
+            localsession.close()
+
+
+    def waitforgas(self, session=None):
+        """Marks transaction as suspended pending gas funding.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.status >= StatusEnum.SENT.value:
+            raise TxStateChangeError('WAITFORGAS cannot succeed final state, had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.WAITFORGAS, session)
+
+
+    def fubar(self, session=None):
+        """Marks transaction as "fubar." Any transaction marked this way is an anomaly and may be a symptom of a serious problem.
+
+        Only manipulates object, does not transaction or commit to backend.
+        """
+        self.__set_status(StatusEnum.FUBAR, session)
+       
+
+    def reject(self, session=None):
+        """Marks transaction as "rejected," which means the node rejected sending the transaction to the network. The nonce has not been spent, and the transaction should be replaced.
+
+        Only manipulates object, does not transaction or commit to backend.
+        """
+        if self.status >= StatusEnum.SENT.value:
+            raise TxStateChangeError('REJECTED cannot succeed SENT or final state, had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.REJECTED, session)
+            
+
+    def override(self, session=None):
+        """Marks transaction as manually overridden.
+
+        Only manipulates object, does not transaction or commit to backend.
+        """
+        if self.status >= StatusEnum.SENT.value:
+            raise TxStateChangeError('OVERRIDDEN cannot succeed SENT or final state, had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.OVERRIDDEN, session)
+
+
+    def retry(self, session=None):
+        """Marks transaction as ready to retry after a timeout following a sendfail or a completed gas funding.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.status != StatusEnum.SENT.value and self.status != StatusEnum.SENDFAIL.value:
+            raise TxStateChangeError('RETRY must follow SENT or SENDFAIL, but had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.RETRY, session)
+
+
+    def readysend(self, session=None):
+        """Marks transaction as ready for initial send attempt.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.status != StatusEnum.PENDING.value and self.status != StatusEnum.WAITFORGAS.value:
+            raise TxStateChangeError('READYSEND must follow PENDING or WAITFORGAS, but had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.READYSEND, session)
+
+
+    def sent(self, session=None):
+        """Marks transaction as having been sent to network.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.status > StatusEnum.SENT:
+            raise TxStateChangeError('SENT after {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.SENT, session)
+
+
+    def sendfail(self, session=None):
+        """Marks that an attempt to send the transaction to the network has failed.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if self.status not in [StatusEnum.PENDING, StatusEnum.SENT, StatusEnum.WAITFORGAS]:
+            raise TxStateChangeError('SENDFAIL must follow SENT or PENDING, but had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.SENDFAIL, session)
+
+
+    def minefail(self, block, session=None):
+        """Marks that transaction was mined but code execution did not succeed.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :param block: Block number transaction was mined in.
+        :type block: number
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if block != None:
+            self.block = block
+        if self.status != StatusEnum.SENT:
+            logg.warning('REVERTED should follow SENT, but had {}'.format(StatusEnum(self.status).name))
+        #if self.status != StatusEnum.PENDING and self.status != StatusEnum.OBSOLETED and self.status != StatusEnum.SENT:
+        #if self.status > StatusEnum.SENT:
+        #    raise TxStateChangeError('REVERTED must follow OBSOLETED, PENDING or SENT, but had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.REVERTED, session)
+
+
+    def cancel(self, confirmed=False, session=None):
+        """Marks that the transaction has been succeeded by a new transaction with same nonce.
+
+        If set to confirmed, the previous state must be OBSOLETED, and will transition to CANCELLED - a finalized state. Otherwise, the state must follow a non-finalized state, and will be set to OBSOLETED.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :param confirmed: Whether transition is to a final state.
+        :type confirmed: bool
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+        if confirmed:
+            if self.status != StatusEnum.OBSOLETED:
+                logg.warning('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
+                #raise TxStateChangeError('CANCELLED must follow OBSOLETED, but had {}'.format(StatusEnum(self.status).name))
+            self.__set_status(StatusEnum.CANCELLED, session)
+        elif self.status != StatusEnum.OBSOLETED:
+            if self.status > StatusEnum.SENT:
+                logg.warning('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
+                #raise TxStateChangeError('OBSOLETED must follow PENDING, SENDFAIL or SENT, but had {}'.format(StatusEnum(self.status).name))
+            self.__set_status(StatusEnum.OBSOLETED, session)
+
+
+    def success(self, block, session=None):
+        """Marks that transaction was successfully mined.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :param block: Block number transaction was mined in.
+        :type block: number
+        :raises cic_eth.db.error.TxStateChangeError: State change represents a sequence of events that should not exist.
+        """
+
+        if block != None:
+            self.block = block
+        if self.status != StatusEnum.SENT:
+            logg.error('SUCCESS should follow SENT, but had {}'.format(StatusEnum(self.status).name))
+            #raise TxStateChangeError('SUCCESS must follow SENT, but had {}'.format(StatusEnum(self.status).name))
+        self.__set_status(StatusEnum.SUCCESS, session)
+
+
+    @staticmethod
+    def get(status=0, limit=4096, status_exact=True):
+        """Returns outgoing transaction lists by status.
+
+        Status may either be matched exactly, or be an upper bound of the integer value of the status enum.
+
+        :param status: Status value to use in query
+        :type status: cic_eth.db.enum.StatusEnum
+        :param limit: Max results to return
+        :type limit: number
+        :param status_exact: Whether or not to perform exact status match
+        :type bool:
+        :returns: List of transaction hashes
+        :rtype: tuple, where first element is transaction hash
+        """
+        e = None
+        session = Otx.create_session()
+        if status_exact:
+            e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all()
+        else:
+            e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all()
+        session.close()
+        return e
+
+
+    @staticmethod
+    def load(tx_hash):
+        """Retrieves the outgoing transaction record by transaction hash.
+
+        :param tx_hash: Transaction hash
+        :type tx_hash: str, 0x-hex
+        """
+        session = Otx.create_session()
+        q = session.query(Otx)
+        q = q.filter(Otx.tx_hash==tx_hash)
+        session.close()
+        return q.first()
+
+
+    @staticmethod
+    def account(account_address):
+        """Retrieves all transaction hashes for which the given Ethereum address is sender or recipient.
+
+        :param account_address: Ethereum address to use in query.
+        :type account_address: str, 0x-hex
+        :returns: Outgoing transactions
+        :rtype: tuple, where first element is transaction hash
+        """
+        session = Otx.create_session()
+        q = session.query(Otx.tx_hash)
+        q = q.join(TxCache)
+        q = q.filter(or_(TxCache.sender==account_address, TxCache.recipient==account_address))
+        txs = q.all()
+        session.close()
+        return list(txs)
+
+
+    def __state_log(self, session):
+        l = OtxStateLog(self)
+        session.add(l)
+
+
+    @staticmethod
+    def add(nonce, address, tx_hash, signed_tx, session=None):
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        otx = Otx(nonce, address, tx_hash, signed_tx)
+        localsession.add(otx)
+        localsession.flush()
+        if otx.tracing:
+            otx.__state_log(session=localsession)
+        localsession.flush()
+
+        if session==None:
+            localsession.commit()
+            localsession.close()
+            return None
+
+        return otx
+
+
+    def __init__(self, nonce, address, tx_hash, signed_tx):
+        self.nonce = nonce
+        self.tx_hash = tx_hash
+        self.signed_tx = signed_tx
+        self.status = StatusEnum.PENDING
+        signed_tx_bytes = bytes.fromhex(signed_tx[2:])
+
+       # sender_address = address_hex_from_signed_tx(signed_tx_bytes)
+       # logg.debug('decoded tx {}'.format(sender_address))
+
+    
+
+# TODO: Most of the methods on this object are obsolete, but it contains a static function for retrieving "expired" outgoing transactions that should be moved to Otx instead.
+class OtxSync(SessionBase):
+    """Obsolete
+    """
+    __tablename__ = 'otx_sync'
+
+    blockchain = Column(String)
+    block_height_backlog = Column(Integer)
+    tx_height_backlog = Column(Integer)
+    block_height_session = Column(Integer)
+    tx_height_session = Column(Integer)
+    block_height_head = Column(Integer)
+    tx_height_head = Column(Integer)
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    date_updated = Column(DateTime)
+
+
+    def backlog(self, block_height=None, tx_height=None):
+        #session = OtxSync.create_session()
+        if block_height != None:
+            if tx_height == None:
+                raise ValueError('tx height missing')
+            self.block_height_backlog = block_height
+            self.tx_height_backlog = tx_height
+            #session.add(self)
+            self.date_updated = datetime.datetime.utcnow()
+        #session.commit()
+        block_height = self.block_height_backlog
+        tx_height = self.tx_height_backlog
+        #session.close()
+        return (block_height, tx_height)
+   
+
+    def session(self, block_height=None, tx_height=None):
+        #session = OtxSync.create_session()
+        if block_height != None:
+            if tx_height == None:
+                raise ValueError('tx height missing')
+            self.block_height_session = block_height
+            self.tx_height_session = tx_height
+            #session.add(self)
+            self.date_updated = datetime.datetime.utcnow()
+        #session.commit()
+        block_height = self.block_height_session
+        tx_height = self.tx_height_session
+        #session.close()
+        return (block_height, tx_height)
+
+
+    def head(self, block_height=None, tx_height=None):
+        #session = OtxSync.create_session()
+        if block_height != None:
+            if tx_height == None:
+                raise ValueError('tx height missing')
+            self.block_height_head = block_height
+            self.tx_height_head = tx_height
+            #session.add(self)
+            self.date_updated = datetime.datetime.utcnow()
+        #session.commit()
+        block_height = self.block_height_head
+        tx_height = self.tx_height_head
+        #session.close()
+        return (block_height, tx_height)
+
+
+    @hybrid_property
+    def synced(self):
+        #return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.block_height_backlog
+        return self.block_height_session == self.block_height_backlog and self.tx_height_session == self.tx_height_backlog
+
+
+    @staticmethod
+    def load(blockchain_string, session):
+        q = session.query(OtxSync)
+        q = q.filter(OtxSync.blockchain==blockchain_string)
+        return q.first()
+
+
+    @staticmethod
+    def latest(nonce):
+        session = SessionBase.create_session()
+        otx = session.query(Otx).filter(Otx.nonce==nonce).order_by(Otx.created.desc()).first()
+        session.close()
+        return otx
+
+
+    @staticmethod
+    def get_expired(datetime_threshold):
+        session = SessionBase.create_session()
+        q = session.query(Otx)
+        q = q.filter(Otx.date_created<datetime_threshold)
+        q = q.filter(Otx.status==StatusEnum.SENT)
+        q = q.order_by(Otx.date_created.desc())
+        q = q.group_by(Otx.nonce)
+        q = q.group_by(Otx.id)
+        otxs = q.all()
+        session.close()
+        return otxs
+
+
+    def chain(self):
+        return self.blockchain
+
+
+    def __init__(self, blockchain):
+        self.blockchain = blockchain
+        self.block_height_head = 0
+        self.tx_height_head = 0
+        self.block_height_session = 0
+        self.tx_height_session = 0
+        self.block_height_backlog = 0
+        self.tx_height_backlog = 0
+
+
+

apps/cic-eth/cic_eth/db/models/role.py

@@ -0,0 +1,117 @@
+# standard imports
+import logging
+
+# third-party imports
+from sqlalchemy import Column, String, Text
+from cic_registry import zero_address
+
+# local imports
+from .base import SessionBase
+
+logg = logging.getLogger()
+
+
+class AccountRole(SessionBase):
+    """Key-value store providing plaintext tags for Ethereum addresses.
+
+    Address is initialized to the zero-address
+
+    :param tag: Tag
+    :type tag: str
+    """
+    __tablename__ = 'account_role'
+
+    tag = Column(Text)
+    address_hex = Column(String(42))
+
+
+    @staticmethod
+    def get_address(tag):
+        """Get Ethereum address matching the given tag
+
+        :param tag: Tag
+        :type tag: str
+        :returns: Ethereum address, or zero-address if tag does not exist
+        :rtype: str, 0x-hex
+        """
+        role = AccountRole.get_role(tag)
+        if role == None:
+            return zero_address
+        return role.address_hex
+
+
+    @staticmethod
+    def get_role(tag):
+        """Get AccountRole model object matching the given tag
+
+        :param tag: Tag
+        :type tag: str
+        :returns: Role object, if found
+        :rtype: cic_eth.db.models.role.AccountRole
+        """
+        session = AccountRole.create_session()
+        role = AccountRole.__get_role(session, tag)
+        session.close()
+        #return role.address_hex
+        return role
+
+
+    @staticmethod
+    def __get_role(session, tag):
+        return session.query(AccountRole).filter(AccountRole.tag==tag).first()
+
+
+    @staticmethod    
+    def set(tag, address_hex):
+        """Persist a tag to Ethereum address association. 
+
+        This will silently overwrite the existing value.
+
+        :param tag: Tag
+        :type tag: str
+        :param address_hex: Ethereum address
+        :type address_hex: str, 0x-hex
+        :returns: Role object
+        :rtype: cic_eth.db.models.role.AccountRole
+        """
+        #session = AccountRole.create_session()
+        #role = AccountRole.__get(session, tag)
+        role = AccountRole.get_role(tag) #session, tag)
+        if role == None:
+            role = AccountRole(tag)
+        role.address_hex = address_hex
+        #session.add(role)
+        #session.commit()
+        #session.close()
+        return role #address_hex
+
+
+    @staticmethod
+    def role_for(address, session=None):
+        """Retrieve role for the given address
+
+        :param address: Ethereum address to match role for
+        :type address: str, 0x-hex
+        :returns: Role tag, or None if no match
+        :rtype: str or None
+        """
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        q = localsession.query(AccountRole)
+        q = q.filter(AccountRole.address_hex==address)
+        role = q.first()
+        tag = None
+        if role != None:
+            tag = role.tag
+
+        if session == None:
+            localsession.close()
+       
+        return tag
+
+
+    def __init__(self, tag):
+        self.tag = tag
+        self.address_hex = zero_address

apps/cic-eth/cic_eth/db/models/sync.py

@@ -0,0 +1,168 @@
+# standard imports
+import datetime
+
+# third-party imports
+from sqlalchemy import Column, String, Integer, DateTime, Text, Boolean
+from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
+
+# local imports
+from cic_eth.db.models.base import SessionBase
+
+
+class BlockchainSync(SessionBase):
+    """Syncer control backend.
+
+    :param chain: Chain spec string representation
+    :type chain: str
+    :param block_start: Block number to start sync from
+    :type block_start: number
+    :param tx_start: Block transaction number to start sync from
+    :type tx_start: number
+    :param block_target: Block number to sync until, inclusive
+    :type block_target: number
+    """
+    __tablename__ = 'blockchain_sync'
+
+    blockchain = Column(String)
+    block_start = Column(Integer)
+    tx_start = Column(Integer)
+    block_cursor = Column(Integer)
+    tx_cursor = Column(Integer)
+    block_target = Column(Integer)
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    date_updated = Column(DateTime)
+
+
+    @staticmethod
+    def first(chain, session=None):
+        """Check if a sync session for the specified chain already exists.
+
+        :param chain: Chain spec string representation
+        :type chain: str
+        :param session: Session to use. If not specified, a separate session will be created for this method only.
+        :type session: SqlAlchemy Session
+        :returns: True if sync record found
+        :rtype: bool
+        """
+        local_session = False
+        if session == None:
+            session = SessionBase.create_session()
+            local_session = True
+        q = session.query(BlockchainSync.id)
+        q = q.filter(BlockchainSync.blockchain==chain)
+        o = q.first()
+        if local_session:
+            session.close()
+        return o == None
+
+
+    @staticmethod
+    def get_last_live_height(current, session=None):
+        """Get the most recent open-ended ("live") syncer record.
+
+        :param current: Current block number
+        :type current: number
+        :param session: Session to use. If not specified, a separate session will be created for this method only.
+        :type session: SqlAlchemy Session
+        :returns: Block and transaction number, respectively
+        :rtype: tuple
+        """
+        local_session = False
+        if session == None:
+            session = SessionBase.create_session()
+            local_session = True
+        q = session.query(BlockchainSync)
+        q = q.filter(BlockchainSync.block_target==None)
+        q = q.order_by(BlockchainSync.date_created.desc())
+        o = q.first()
+        if local_session:
+            session.close()
+
+        if o == None:
+            return (0, 0)
+
+        return (o.block_cursor, o.tx_cursor)
+
+
+    @staticmethod
+    def get_unsynced(session=None):
+        """Get previous bounded sync sessions that did not complete.
+
+        :param session: Session to use. If not specified, a separate session will be created for this method only.
+        :type session: SqlAlchemy Session
+        :returns: Syncer database ids
+        :rtype: tuple, where first element is id
+        """
+        unsynced = []
+        local_session = False
+        if session == None:
+            session = SessionBase.create_session()
+            local_session = True
+        q = session.query(BlockchainSync.id)
+        q = q.filter(BlockchainSync.block_target!=None)
+        q = q.filter(BlockchainSync.block_cursor<BlockchainSync.block_target)
+        q = q.order_by(BlockchainSync.date_created.asc())
+        for u in q.all():
+            unsynced.append(u[0])
+        if local_session:
+            session.close()
+
+        return unsynced
+
+
+    def set(self, block_height, tx_height):
+        """Set the height of the syncer instance.
+
+        Only manipulates object, does not transaction or commit to backend.
+
+        :param block_height: Block number
+        :type block_height: number
+        :param tx_height: Block transaction number
+        :type tx_height: number
+        """
+        self.block_cursor = block_height
+        self.tx_cursor = tx_height
+
+
+    def cursor(self):
+        """Get current state of cursor from cached instance.
+
+        :returns: Block and transaction height, respectively
+        :rtype: tuple
+        """
+        return (self.block_cursor, self.tx_cursor)
+
+
+    def start(self):
+        """Get sync block start position from cached instance.
+
+        :returns: Block and transaction height, respectively
+        :rtype: tuple
+        """
+        return (self.block_start, self.tx_start)
+
+
+    def target(self):
+        """Get sync block upper bound from cached instance.
+
+        :returns: Block number
+        :rtype: number, or None if sync is open-ended
+        """
+        return self.block_target
+
+
+    def chain(self):
+        """Get chain the cached instance represents.
+        """
+        return self.blockchain
+
+
+    def __init__(self, chain, block_start, tx_start, block_target=None):
+        self.blockchain = chain
+        self.block_start = block_start
+        self.tx_start = tx_start
+        self.block_cursor = block_start
+        self.tx_cursor = tx_start
+        self.block_target = block_target
+        self.date_created = datetime.datetime.utcnow()
+        self.date_modified = datetime.datetime.utcnow()

apps/cic-eth/cic_eth/db/models/tx.py

@@ -0,0 +1,162 @@
+# standard imports
+import datetime
+
+# third-party imports
+from sqlalchemy import Column, String, Integer, DateTime, Enum, ForeignKey, Boolean
+from sqlalchemy.ext.hybrid import hybrid_method, hybrid_property
+#from sqlalchemy.orm import relationship, backref
+#from sqlalchemy.ext.declarative import declarative_base
+
+# local imports
+from .base import SessionBase
+from .otx import Otx
+from cic_eth.db.util import num_serialize
+from cic_eth.error import NotLocalTxError
+from cic_eth.db.error import TxStateChangeError
+
+
+class TxCache(SessionBase):
+    """Metadata expansions for outgoing transactions.
+
+    These records are not essential for handling of outgoing transaction queues. It is implemented to reduce the amount of computation spent of parsing and analysing raw signed transaction data.
+
+    Instantiation of the object will fail if an outgoing transaction record with the same transaction hash does not exist.
+
+    Typically three types of transactions are recorded:
+
+    - Token transfers; where source and destination token values and addresses are identical, sender and recipient differ.
+    - Token conversions; source and destination token values and addresses differ, sender and recipient are identical.
+    - Any other transaction; source and destination token addresses are zero-address.
+
+    :param tx_hash: Transaction hash
+    :type tx_hash: str, 0x-hex
+    :param sender: Ethereum address of transaction sender
+    :type sender: str, 0x-hex
+    :param recipient: Ethereum address of transaction beneficiary (e.g. token transfer recipient)
+    :type recipient: str, 0x-hex
+    :param source_token_address: Contract address of token that sender spent from
+    :type source_token_address: str, 0x-hex
+    :param destination_token_address: Contract address of token that recipient will receive balance of
+    :type destination_token_address: str, 0x-hex
+    :param from_value: Amount of source tokens spent
+    :type from_value: number
+    :param to_value: Amount of destination tokens received
+    :type to_value: number
+    :param block_number: Block height the transaction was mined at, or None if not yet mined
+    :type block_number: number or None
+    :param tx_number: Block transaction height the transaction was mined at, or None if not yet mined
+    :type tx_number: number or None
+    :raises FileNotFoundError: Outgoing transaction for given transaction hash does not exist
+    """
+    __tablename__ = 'tx_cache'
+
+    otx_id = Column(Integer, ForeignKey('otx.id'))
+    source_token_address = Column(String(42))
+    destination_token_address = Column(String(42))
+    sender = Column(String(42))
+    recipient = Column(String(42))
+    from_value = Column(String())
+    to_value = Column(String())
+    block_number = Column(Integer())
+    tx_index = Column(Integer())
+    date_created = Column(DateTime, default=datetime.datetime.utcnow)
+    date_updated = Column(DateTime, default=datetime.datetime.utcnow)
+    date_checked = Column(DateTime, default=datetime.datetime.utcnow)
+
+
+    def values(self):
+        from_value_hex = bytes.fromhex(self.from_value)
+        from_value = int.from_bytes(from_value_hex, 'big')
+
+        to_value_hex = bytes.fromhex(self.to_value)
+        to_value = int.from_bytes(to_value_hex, 'big')
+
+        return (from_value, to_value)
+
+
+    def check(self):
+        """Update the "checked" timestamp to current time.
+
+        Only manipulates object, does not transaction or commit to backend.
+        """
+        self.date_checked = datetime.datetime.now()
+
+
+    @staticmethod
+    def clone(
+            tx_hash_original,
+            tx_hash_new,
+            session=None,
+            ):
+        """Copy tx cache data and associate it with a new transaction.
+
+        :param tx_hash_original: tx cache data to copy
+        :type tx_hash_original: str, 0x-hex
+        :param tx_hash_new: tx hash to associate the copied entry with
+        :type tx_hash_new: str, 0x-hex
+        """
+        localsession = session
+        if localsession == None:
+            localsession = SessionBase.create_session()
+
+        q = localsession.query(TxCache)
+        q = q.join(Otx)
+        q = q.filter(Otx.tx_hash==tx_hash_original)
+        txc = q.first()
+
+        if txc == None:
+            raise NotLocalTxError('original {}'.format(tx_hash_original))
+        if txc.block_number != None:
+            raise TxStateChangeError('cannot clone tx cache of confirmed tx {}'.format(tx_hash_original))
+
+        q = localsession.query(Otx)
+        q = q.filter(Otx.tx_hash==tx_hash_new)
+        otx = q.first()
+
+        if otx == None:
+            raise NotLocalTxError('new {}'.format(tx_hash_new))
+
+        values = txc.values()
+        txc_new = TxCache(
+                otx.tx_hash,
+                txc.sender,
+                txc.recipient,
+                txc.source_token_address,
+                txc.destination_token_address,
+                values[0],
+                values[1],
+                )
+        localsession.add(txc_new)
+        localsession.commit()
+
+        if session == None:
+            localsession.close()
+
+
+    def __init__(self, tx_hash, sender, recipient, source_token_address, destination_token_address, from_value, to_value, block_number=None, tx_index=None):
+        session = SessionBase.create_session()
+        tx = session.query(Otx).filter(Otx.tx_hash==tx_hash).first()
+        if tx == None:
+            session.close()
+            raise FileNotFoundError('outgoing transaction record unknown {} (add a Tx first)'.format(tx_hash))
+        self.otx_id = tx.id
+
+#        if tx == None:
+#            session.close()
+#            raise ValueError('tx hash {} (outgoing: {}) not found'.format(tx_hash, outgoing))
+#        session.close()
+
+        self.sender = sender
+        self.recipient = recipient
+        self.source_token_address = source_token_address
+        self.destination_token_address = destination_token_address
+        self.from_value = num_serialize(from_value).hex()
+        self.to_value = num_serialize(to_value).hex()
+        self.block_number = block_number
+        self.tx_index = tx_index
+        # not automatically set in sqlite, it seems:
+        self.date_created = datetime.datetime.now()
+        self.date_updated = self.date_created
+        self.date_checked = self.date_created
+
+