From be3c59a78059541c4b7ecaf8951e3040b1bd1604 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Wed, 17 Feb 2021 09:33:18 +0000 Subject: [PATCH] cic-eth: Integrate transaction list queries --- apps/cic-eth/cic_eth/api/api_admin.py | 4 +- apps/cic-eth/cic_eth/api/api_task.py | 67 ++++++++ apps/cic-eth/cic_eth/db/models/otx.py | 18 +- apps/cic-eth/cic_eth/ext/tx.py | 176 ++++++++++++++++++++ apps/cic-eth/cic_eth/queue/time.py | 40 +++++ apps/cic-eth/cic_eth/queue/tx.py | 6 + apps/cic-eth/requirements.txt | 4 +- apps/cic-eth/tests/fixtures_celery.py | 2 + apps/cic-eth/tests/functional/test_list.py | 92 ++++++++++ apps/cic-eth/tests/mock/__init__.py | 1 + apps/cic-eth/tests/mock/filter.py | 22 +++ apps/cic-eth/tests/unit/ext/test_ext_tx.py | 109 ++++++++++++ apps/requirements.txt | 2 + docker-compose.yml | 185 +++++++++++---------- 14 files changed, 635 insertions(+), 93 deletions(-) create mode 100644 apps/cic-eth/cic_eth/ext/tx.py create mode 100644 apps/cic-eth/cic_eth/queue/time.py create mode 100644 apps/cic-eth/tests/functional/test_list.py create mode 100644 apps/cic-eth/tests/mock/__init__.py create mode 100644 apps/cic-eth/tests/mock/filter.py create mode 100644 apps/cic-eth/tests/unit/ext/test_ext_tx.py diff --git a/apps/cic-eth/cic_eth/api/api_admin.py b/apps/cic-eth/cic_eth/api/api_admin.py index 391ad83..75a54bd 100644 --- a/apps/cic-eth/cic_eth/api/api_admin.py +++ b/apps/cic-eth/cic_eth/api/api_admin.py @@ -267,7 +267,9 @@ class AdminApi: """ s = celery.signature( 'cic_eth.queue.tx.get_account_tx', - [address], + [ + address, + ], queue=self.queue, ) txs = s.apply_async().get() diff --git a/apps/cic-eth/cic_eth/api/api_task.py b/apps/cic-eth/cic_eth/api/api_task.py index eca5f91..6764b1c 100644 --- a/apps/cic-eth/cic_eth/api/api_task.py +++ b/apps/cic-eth/cic_eth/api/api_task.py @@ -417,6 +417,73 @@ class Api: return t + def list(self, address, limit=10, external_task=None, external_queue=None): + """Retrieve an aggregate list of latest transactions of internal and (optionally) external origin in reverse chronological order. + + The array of transactions returned have the same dict layout as those passed by the callback filter in cic_eth/runnable/manager + + If the external task is defined, this task will be used to query external transactions. If this is not defined, no external transactions will be included. The task must accept (offset, limit, address) as input parameters, and return a bloom filter that will be used to retrieve transaction data for the matching transactions. See cic_eth.ext.tx.list_tx_by_bloom for details on the bloom filter dat format. + + :param address: Ethereum address to list transactions for + :type address: str, 0x-hex + :param limit: Amount of results to return + :type limit: number + :param external_task: Celery task providing external transactions + :type external_task: str + :param external_queue: Celery task queue providing exernal transactions task + :type external_queue: str + :returns: List of transactions + :rtype: list of dict + """ + offset = 0 + s_local = celery.signature( + 'cic_eth.queue.tx.get_account_tx', + [ + address, + ], + queue=self.queue, + ) + + s_brief = celery.signature( + 'cic_eth.ext.tx.tx_collate', + [ + self.chain_str, + offset, + limit + ], + queue=self.queue, + ) + if self.callback_param != None: + s_assemble.link(self.callback_success).on_error(self.callback_error) + + t = None + if external_task != None: + s_external_get = celery.signature( + external_task, + [ + address, + offset, + limit, + ], + queue=external_queue, + ) + + s_external_process = celery.signature( + 'cic_eth.ext.tx.list_tx_by_bloom', + [ + address, + self.chain_str, + ], + queue=self.queue, + ) + c = celery.chain(s_external_get, s_external_process) + t = celery.chord([s_local, c])(s_brief) + else: + t = s_local.apply_sync() + + return t + + def ping(self, r): """A noop callback ping for testing purposes. diff --git a/apps/cic-eth/cic_eth/db/models/otx.py b/apps/cic-eth/cic_eth/db/models/otx.py index 01cb804..c22895b 100644 --- a/apps/cic-eth/cic_eth/db/models/otx.py +++ b/apps/cic-eth/cic_eth/db/models/otx.py @@ -422,7 +422,7 @@ class Otx(SessionBase): @staticmethod - def get(status=0, limit=4096, status_exact=True): + def get(status=0, limit=4096, status_exact=True, session=None): """Returns outgoing transaction lists by status. Status may either be matched exactly, or be an upper bound of the integer value of the status enum. @@ -437,26 +437,32 @@ class Otx(SessionBase): :rtype: tuple, where first element is transaction hash """ e = None - session = Otx.create_session() + + session = SessionBase.bind_session(session) + if status_exact: e = session.query(Otx.tx_hash).filter(Otx.status==status).order_by(Otx.date_created.asc()).limit(limit).all() else: e = session.query(Otx.tx_hash).filter(Otx.status<=status).order_by(Otx.date_created.asc()).limit(limit).all() - session.close() + + SessionBase.release_session(session) return e @staticmethod - def load(tx_hash): + def load(tx_hash, session=None): """Retrieves the outgoing transaction record by transaction hash. :param tx_hash: Transaction hash :type tx_hash: str, 0x-hex """ - session = Otx.create_session() + session = SessionBase.bind_session(session) + q = session.query(Otx) q = q.filter(Otx.tx_hash==tx_hash) - session.close() + + SessionBase.release_session(session) + return q.first() diff --git a/apps/cic-eth/cic_eth/ext/tx.py b/apps/cic-eth/cic_eth/ext/tx.py new file mode 100644 index 0000000..340f77b --- /dev/null +++ b/apps/cic-eth/cic_eth/ext/tx.py @@ -0,0 +1,176 @@ +# standard imports +import logging +import math + +# third-pary imports +import web3 +import celery +import moolb +from cic_registry.chain import ChainSpec +from cic_registry.registry import CICRegistry +from hexathon import strip_0x + +# local imports +from cic_eth.eth.rpc import RpcClient +from cic_eth.db.models.otx import Otx +from cic_eth.eth.util import unpack_signed_raw_tx +from cic_eth.db.enum import StatusEnum +from cic_eth.eth.token import unpack_transfer +from cic_eth.queue.tx import get_tx_cache +from cic_eth.queue.time import tx_times + +celery_app = celery.current_app +logg = logging.getLogger() + +MAX_BLOCK_TX = 250 + + +# TODO: Make this method easier to read +@celery_app.task() +def list_tx_by_bloom(bloomspec, address, chain_str): + """Retrieve external transaction data matching the provided filter + + The bloom filter representation with the following structure (the size of the filter will be inferred from the size of the provided filter data): + { + 'alg': , + 'high': , + 'low': , + 'filter_rounds': , + 'block_filter': , + 'blocktx_filter': , + } + + :param bloomspec: Bloom filter data + :type bloomspec: dict (see description above) + :param address: Recipient address to use in matching + :type address: str, 0x-hex + :param chain_str: Chain spec string representation + :type chain_str: str + :returns: dict of transaction data as dict, keyed by transaction hash + :rtype: dict of dict + """ + chain_spec = ChainSpec.from_chain_str(chain_str) + c = RpcClient(chain_spec) + block_filter_data = bytes.fromhex(bloomspec['block_filter']) + tx_filter_data = bytes.fromhex(bloomspec['blocktx_filter']) + databitlen = len(block_filter_data)*8 + block_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=block_filter_data) + tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data) + + txs = {} + for block_height in range(bloomspec['low'], bloomspec['high']): + block_height_bytes = block_height.to_bytes(4, 'big') + if block_filter.check(block_height_bytes): + logg.debug('filter matched block {}'.format(block_height)) + block = c.w3.eth.getBlock(block_height, True) + + for tx_index in range(0, len(block.transactions)): + composite = tx_index + block_height + tx_index_bytes = composite.to_bytes(4, 'big') + if tx_filter.check(tx_index_bytes): + logg.debug('filter matched block {} tx {}'.format(block_height, tx_index)) + + try: + tx = c.w3.eth.getTransactionByBlock(block_height, tx_index) + except web3.exceptions.TransactionNotFound: + logg.debug('false positive on block {} tx {}'.format(block_height, tx_index)) + continue + tx_address = None + tx_token_value = 0 + try: + transfer_data = unpack_transfer(tx['data']) + tx_address = transfer_data['to'] + tx_token_value = transfer_data['amount'] + except ValueError: + logg.debug('not a transfer transaction, skipping {}'.format(tx)) + continue + if address == tx_address: + status = StatusEnum.SENT + try: + rcpt = c.w3.eth.getTransactionReceipt(tx.hash) + if rcpt['status'] == 0: + pending = StatusEnum.REVERTED + else: + pending = StatusEnum.SUCCESS + except web3.exceptions.TransactionNotFound: + pass + + tx_hash_hex = tx['hash'].hex() + + token = CICRegistry.get_address(chain_spec, tx['to']) + token_symbol = token.symbol() + token_decimals = token.decimals() + times = tx_times(tx_hash_hex, chain_str) + tx_r = { + 'hash': tx_hash_hex, + 'sender': tx['from'], + 'recipient': tx_address, + 'source_value': tx_token_value, + 'destination_value': tx_token_value, + 'source_token': tx['to'], + 'destination_token': tx['to'], + 'source_token_symbol': token_symbol, + 'destination_token_symbol': token_symbol, + 'source_token_decimals': token_decimals, + 'destination_token_decimals': token_decimals, + 'source_token_chain': chain_str, + 'destination_token_chain': chain_str, + 'nonce': tx['nonce'], + } + if times['queue'] != None: + tx_r['date_created'] = times['queue'] + else: + tx_r['date_created'] = times['network'] + txs[tx_hash_hex] = tx_r + break + return txs + + +# TODO: Surely it must be possible to optimize this +# TODO: DRY this with callback filter in cic_eth/runnable/manager +# TODO: Remove redundant fields from end representation (timestamp, tx_hash) +@celery_app.task() +def tx_collate(tx_batches, chain_str, offset, limit, newest_first=True): + """Merges transaction data from multiple sources and sorts them in chronological order. + + :param tx_batches: Transaction data inputs + :type tx_batches: lists of lists of transaction data + :param chain_str: Chain spec string representation + :type chain_str: str + :param offset: Number of sorted results to skip (not yet implemented) + :type offset: number + :param limit: Maximum number of results to return (not yet implemented) + :type limit: number + :param newest_first: If True, returns results in reverse chronological order + :type newest_first: bool + :returns: Transactions + :rtype: list + """ + txs_by_block = {} + chain_spec = ChainSpec.from_chain_str(chain_str) + + for b in tx_batches: + for v in b.values(): + tx = None + k = None + try: + hx = strip_0x(v) + tx = unpack_signed_raw_tx(bytes.fromhex(hx), chain_spec.chain_id()) + txc = get_tx_cache(tx['hash']) + txc['timestamp'] = int(txc['date_created'].timestamp()) + txc['hash'] = txc['tx_hash'] + tx = txc + except TypeError: + tx = v + tx['timestamp'] = tx['date_created'] + k = '{}.{}.{}'.format(tx['timestamp'], tx['sender'], tx['nonce']) + txs_by_block[k] = tx + + txs = [] + ks = list(txs_by_block.keys()) + ks.sort() + if newest_first: + ks.reverse() + for k in ks: + txs.append(txs_by_block[k]) + return txs diff --git a/apps/cic-eth/cic_eth/queue/time.py b/apps/cic-eth/cic_eth/queue/time.py new file mode 100644 index 0000000..aecb665 --- /dev/null +++ b/apps/cic-eth/cic_eth/queue/time.py @@ -0,0 +1,40 @@ +# standard imports +import logging + +# third-party imports +import web3 +import celery +from cic_registry.chain import ChainSpec + +# local imports +from cic_eth.eth.rpc import RpcClient +from cic_eth.db.models.otx import Otx +from cic_eth.error import NotLocalTxError + +celery_app = celery.current_app + +logg = logging.getLogger() + + +# TODO: This method does not belong in the _queue_ module, it operates across queue and network +@celery_app.task() +def tx_times(tx_hash, chain_str): + chain_spec = ChainSpec.from_chain_str(chain_str) + c = RpcClient(chain_spec) + time_pair = { + 'network': None, + 'queue': None, + } + try: + rcpt = c.w3.eth.getTransactionReceipt(tx_hash) + block = c.w3.eth.getBlock(rcpt['blockHash']) + logg.debug('rcpt {}'.format(block)) + time_pair['network'] = block['timestamp'] + except web3.exceptions.TransactionNotFound: + pass + + otx = Otx.load(tx_hash) + if otx != None: + time_pair['queue'] = int(otx['date_created'].timestamp()) + + return time_pair diff --git a/apps/cic-eth/cic_eth/queue/tx.py b/apps/cic-eth/cic_eth/queue/tx.py index e3e13e4..2159ee8 100644 --- a/apps/cic-eth/cic_eth/queue/tx.py +++ b/apps/cic-eth/cic_eth/queue/tx.py @@ -386,6 +386,8 @@ def get_tx_cache(tx_hash): 'status_code': otx.status, 'source_token': txc.source_token_address, 'destination_token': txc.destination_token_address, + 'block_number': txc.block_number, + 'tx_index': txc.tx_index, 'sender': txc.sender, 'recipient': txc.recipient, 'from_value': int(txc.from_value), @@ -661,6 +663,7 @@ def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None) """ if not as_sender and not as_recipient: raise ValueError('at least one of as_sender and as_recipient must be True') + txs = {} session = SessionBase.create_session() @@ -676,6 +679,9 @@ def get_account_tx(address, as_sender=True, as_recipient=True, counterpart=None) results = q.all() for r in results: + if txs.get(r.tx_hash) != None: + logg.debug('tx {} already recorded'.format(r.tx_hash)) + continue txs[r.tx_hash] = r.signed_tx session.close() diff --git a/apps/cic-eth/requirements.txt b/apps/cic-eth/requirements.txt index 9c5938e..5962543 100644 --- a/apps/cic-eth/requirements.txt +++ b/apps/cic-eth/requirements.txt @@ -16,5 +16,7 @@ uWSGI==2.0.19.1 semver==2.13.0 eth-gas-proxy==0.0.1a4 websocket-client==0.57.0 +moolb~=0.1.1b2 eth-address-index~=0.1.0a8 -chainlib~=0.0.1a11 +chainlib~=0.0.1a12 +hexathon~=0.0.1a3 diff --git a/apps/cic-eth/tests/fixtures_celery.py b/apps/cic-eth/tests/fixtures_celery.py index a57c6b9..8d8a31b 100644 --- a/apps/cic-eth/tests/fixtures_celery.py +++ b/apps/cic-eth/tests/fixtures_celery.py @@ -15,12 +15,14 @@ def celery_includes(): 'cic_eth.eth.token', 'cic_eth.eth.request', 'cic_eth.eth.tx', + 'cic_eth.ext.tx', 'cic_eth.queue.tx', 'cic_eth.admin.ctrl', 'cic_eth.admin.nonce', 'cic_eth.eth.account', 'cic_eth.callbacks.noop', 'cic_eth.callbacks.http', + 'tests.mock.filter', ] diff --git a/apps/cic-eth/tests/functional/test_list.py b/apps/cic-eth/tests/functional/test_list.py new file mode 100644 index 0000000..b104f97 --- /dev/null +++ b/apps/cic-eth/tests/functional/test_list.py @@ -0,0 +1,92 @@ +# standard imports +import logging + +# local imports +from cic_eth.api.api_task import Api +from cic_eth.eth.token import TokenTxFactory +from cic_eth.eth.task import sign_tx +from tests.mock.filter import ( + block_filter, + tx_filter, + ) + + +logg = logging.getLogger() + + +def test_list_tx( + default_chain_spec, + default_chain_registry, + init_database, + init_rpc, + init_w3, + init_eth_tester, + dummy_token_gifted, + cic_registry, + celery_session_worker, + ): + + tx_hashes = [] + # external tx + init_eth_tester.mine_blocks(13) + txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc) + tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 3000, default_chain_spec) + (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec)) + tx_hashes.append(tx_hash_hex) + init_w3.eth.sendRawTransaction(tx_signed_raw_hex) + # add to filter + rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex) + a = rcpt['blockNumber'] + block_filter.add(a.to_bytes(4, 'big')) + a = rcpt['blockNumber'] + rcpt['transactionIndex'] + tx_filter.add(a.to_bytes(4, 'big')) + + # external tx + init_eth_tester.mine_blocks(28) + txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc) + tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 4000, default_chain_spec) + (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec)) + tx_hashes.append(tx_hash_hex) + init_w3.eth.sendRawTransaction(tx_signed_raw_hex) + # add to filter + rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex) + a = rcpt['blockNumber'] + block_filter.add(a.to_bytes(4, 'big')) + a = rcpt['blockNumber'] + rcpt['transactionIndex'] + tx_filter.add(a.to_bytes(4, 'big')) + + # custodial tx + init_eth_tester.mine_blocks(3) + txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc) + api = Api(str(default_chain_spec), queue=None) + t = api.transfer(init_w3.eth.accounts[0], init_w3.eth.accounts[1], 1000, 'DUM') + t.get() + tx_hash_hex = None + for c in t.collect(): + tx_hash_hex = c[1] + assert t.successful() + tx_hashes.append(tx_hash_hex) + + # custodial tx + init_eth_tester.mine_blocks(6) + api = Api(str(default_chain_spec), queue=None) + t = api.transfer(init_w3.eth.accounts[0], init_w3.eth.accounts[1], 2000, 'DUM') + t.get() + tx_hash_hex = None + for c in t.collect(): + tx_hash_hex = c[1] + assert t.successful() + tx_hashes.append(tx_hash_hex) + + # test the api + t = api.list(init_w3.eth.accounts[1], external_task='tests.mock.filter.filter') + r = t.get() + for c in t.collect(): + r = c[1] + assert t.successful() + + assert len(r) == 4 + for tx in r: + logg.debug('have tx {}'.format(r)) + tx_hashes.remove(tx['hash']) + assert len(tx_hashes) == 0 diff --git a/apps/cic-eth/tests/mock/__init__.py b/apps/cic-eth/tests/mock/__init__.py new file mode 100644 index 0000000..1e78c9b --- /dev/null +++ b/apps/cic-eth/tests/mock/__init__.py @@ -0,0 +1 @@ +from .filter import * diff --git a/apps/cic-eth/tests/mock/filter.py b/apps/cic-eth/tests/mock/filter.py new file mode 100644 index 0000000..76f549c --- /dev/null +++ b/apps/cic-eth/tests/mock/filter.py @@ -0,0 +1,22 @@ +# third-party imports +import celery +import moolb + +celery_app = celery.current_app + +block_filter = moolb.Bloom(1024, 3) +tx_filter = moolb.Bloom(1024, 3) +lo = 0 +hi = 100 + + +@celery_app.task() +def filter(address, offset, limit): + return { + 'alg': 'sha256', + 'high': hi, + 'low': lo, + 'block_filter': block_filter.to_bytes().hex(), + 'blocktx_filter': tx_filter.to_bytes().hex(), + 'filter_rounds': 3, + } diff --git a/apps/cic-eth/tests/unit/ext/test_ext_tx.py b/apps/cic-eth/tests/unit/ext/test_ext_tx.py new file mode 100644 index 0000000..b1c50bc --- /dev/null +++ b/apps/cic-eth/tests/unit/ext/test_ext_tx.py @@ -0,0 +1,109 @@ +# standard imports +import logging + +# third-party imports +import celery +import moolb + +# local imports +from cic_eth.eth.token import TokenTxFactory +from cic_eth.eth.task import sign_tx + +logg = logging.getLogger() + + +# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state +def test_filter_process( + init_rpc, + default_chain_spec, + default_chain_registry, + celery_session_worker, + init_eth_tester, + init_w3, + dummy_token_gifted, + cic_registry, + ): + + b = moolb.Bloom(1024, 3) + t = moolb.Bloom(1024, 3) + + tx_hashes = [] + # external tx + init_eth_tester.mine_blocks(13) + txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc) + tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 3000, default_chain_spec) + (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec)) + tx_hashes.append(tx_hash_hex) + init_w3.eth.sendRawTransaction(tx_signed_raw_hex) + # add to filter + rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex) + a = rcpt['blockNumber'] + b.add(a.to_bytes(4, 'big')) + a = rcpt['blockNumber'] + rcpt['transactionIndex'] + t.add(a.to_bytes(4, 'big')) + + # external tx + init_eth_tester.mine_blocks(28) + txf = TokenTxFactory(init_w3.eth.accounts[0], init_rpc) + tx = txf.transfer(dummy_token_gifted, init_w3.eth.accounts[1], 4000, default_chain_spec) + (tx_hash_hex, tx_signed_raw_hex) = sign_tx(tx, str(default_chain_spec)) + tx_hashes.append(tx_hash_hex) + init_w3.eth.sendRawTransaction(tx_signed_raw_hex) + # add to filter + rcpt = init_w3.eth.getTransactionReceipt(tx_hash_hex) + a = rcpt['blockNumber'] + b.add(a.to_bytes(4, 'big')) + a = rcpt['blockNumber'] + rcpt['transactionIndex'] + t.add(a.to_bytes(4, 'big')) + +# init_eth_tester.mine_blocks(13) +# tx_hash_one = init_w3.eth.sendTransaction({ +# 'from': init_w3.eth.accounts[2], +# 'to': init_w3.eth.accounts[1], +# 'value': 1024, +# }) +# rcpt = init_w3.eth.getTransactionReceipt(tx_hash_one) +# a = rcpt['blockNumber'] +# b.add(a.to_bytes(4, 'big')) +# a = rcpt['blockNumber'] + rcpt['transactionIndex'] +# t.add(a.to_bytes(4, 'big')) +# +# init_eth_tester.mine_blocks(28) +# tx_hash_two = init_w3.eth.sendTransaction({ +# 'from': init_w3.eth.accounts[3], +# 'to': init_w3.eth.accounts[1], +# 'value': 2048, +# }) +# rcpt = init_w3.eth.getTransactionReceipt(tx_hash_two) +# a = rcpt['blockNumber'] +# b.add(a.to_bytes(4, 'big')) +# a = rcpt['blockNumber'] + rcpt['transactionIndex'] +# t.add(a.to_bytes(4, 'big')) + + init_eth_tester.mine_blocks(10) + + o = { + 'alg': 'sha256', + 'filter_rounds': 3, + 'low': 0, + 'high': 50, + 'block_filter': b.to_bytes().hex(), + 'blocktx_filter': t.to_bytes().hex(), + } + + s = celery.signature( + 'cic_eth.ext.tx.list_tx_by_bloom', + [ + o, + init_w3.eth.accounts[1], + str(default_chain_spec), + ], + queue=None + ) + t = s.apply_async() + r = t.get() + + assert len(r) == 2 + for tx_hash in r.keys(): + tx_hashes.remove(tx_hash) + assert len(tx_hashes) == 0 diff --git a/apps/requirements.txt b/apps/requirements.txt index 4489caf..9c3626c 100644 --- a/apps/requirements.txt +++ b/apps/requirements.txt @@ -41,3 +41,5 @@ yaml-acl==0.0.1 rlp==2.0.1 cryptocurrency-cli-tools==0.0.4 giftable-erc20-token==0.0.7b7 +hexathon==0.0.1a3 +chainlib==0.0.1a12 diff --git a/docker-compose.yml b/docker-compose.yml index 1aa307c..e46f33b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,56 +126,71 @@ services: - contract-config:/tmp/cic/config - # cic-cache-tracker: - # # image: registry.gitlab.com/grassrootseconomics/cic-cache:master-latest - # build: apps/cic-cache - # environment: - # CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning - # ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} - # BANCOR_DIR: ${BANCOR_DIR:-/usr/local/share/cic/bancor} - # DATABASE_USER: ${DATABASE_USER:-grassroots} - # DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql - # DATABASE_HOST: ${DATABASE_HOST:-postgres} - # DATABASE_PORT: ${DATABASE_PORT:-5432} - # DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} - # DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} - # DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} - # ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} - # deploy: - # restart_policy: - # condition: on-failure - # depends_on: - # - postgres - # - eth - # command: - # - /bin/sh - # - -c - # - | - # if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - # /usr/local/bin/cic-cache-tracker -vv - # volumes: - # - contract-config:/tmp/cic/config/:ro - # entrypoint: ["/usr/local/bin/cic-cache-tracker", "-vv"] - # command: "/usr/local/bin/cic-cache-tracker -vv" + cic-cache-tracker: + build: + context: apps/cic-cache/ + dockerfile: docker/Dockerfile + environment: + CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS # supplied at contract-config after contract provisioning + ETH_PROVIDER: ${ETH_PROVIDER:-http://eth:8545} + DATABASE_USER: ${DATABASE_USER:-grassroots} + DATABASE_PASSWORD: ${DATABASE_PASSWORD:-tralala} # this is is set at initdb see: postgres/initdb/create_db.sql + DATABASE_HOST: ${DATABASE_HOST:-postgres} + DATABASE_PORT: ${DATABASE_PORT:-5432} + DATABASE_NAME: ${DATABASE_NAME_CIC_CACHE:-cic_cache} + DATABASE_ENGINE: ${DATABASE_ENGINE:-postgres} + DATABASE_DRIVER: ${DATABASE_DRIVER:-psycopg2} + DATABASE_DEBUG: 1 + ETH_ABI_DIR: ${ETH_ABI_DIR:-/usr/local/share/cic/solidity/abi} + CIC_TRUST_ADDRESS: ${DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER:-0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C} + CIC_CHAIN_SPEC: ${CIC_CHAIN_SPEC:-Bloxberg:8996} + CELERY_BROKER_URL: redis://redis:6379 + CELERY_RESULT_URL: redis://redis:6379 + deploy: + restart_policy: + condition: on-failure + depends_on: + - redis + - postgres + - eth + command: + - /bin/bash + - -c + - | + if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + /usr/local/bin/cic-cache-tracker -vv + volumes: + - contract-config:/tmp/cic/config/:ro -# cic-cache-server: -# image: grassrootseconomics:cic-cache-uwsgi -# environment: -# DATABASE_USER: $DATABASE_USER -# DATABASE_HOST: $DATABASE_HOST -# DATABASE_PORT: $DATABASE_PORT -# DATABASE_PASSWORD: $DATABASE_PASSWORD -# DATABASE_NAME: $DATABASE_NAME_CIC_CACHE -# PGPASSWORD: $DATABASE_PASSWORD -# SERVER_PORT: 80 -# ports: -# - ${HTTP_PORT_CIC_CACHE}:80 -# depends_on: -# - postgres -# deploy: -# restart_policy: -# condition: on-failure -# command: "/root/start_uwsgi.sh" + cic-cache-server: + build: + context: apps/cic-cache/ + dockerfile: docker/Dockerfile + environment: + DATABASE_USER: $DATABASE_USER + DATABASE_HOST: $DATABASE_HOST + DATABASE_PORT: $DATABASE_PORT + DATABASE_PASSWORD: $DATABASE_PASSWORD + DATABASE_NAME: $DATABASE_NAME_CIC_CACHE + DATABASE_DEBUG: 1 + PGPASSWORD: $DATABASE_PASSWORD + SERVER_PORT: 8000 + ports: + - ${HTTP_PORT_CIC_CACHE:-63313}:8000 + depends_on: + - postgres + deploy: + restart_policy: + condition: on-failure + command: + - /bin/bash + - -c + - | + if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi + "/usr/local/bin/uwsgi" \ + --wsgi-file /usr/src/cic-cache/cic_cache/runnable/server.py \ + --http :80 \ + --pyargv -vv cic-eth-tasker: # image: grassrootseconomics:cic-eth-service @@ -370,42 +385,42 @@ services: # command: "/root/start_retry.sh -q cic-eth -vv" - cic-eth-server: - build: - context: apps/ - dockerfile: cic-eth/docker/Dockerfile - environment: - CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC - CELERY_BROKER_URL: $CELERY_BROKER_URL - CELERY_RESULT_URL: $CELERY_RESULT_URL - SERVER_PORT: 8000 - depends_on: - - eth - - postgres - - redis - ports: - - ${HTTP_PORT_CIC_ETH:-63314}:8000 - deploy: - restart_policy: - condition: on-failure - volumes: - - contract-config:/tmp/cic/config/:ro - command: - - /bin/bash - - -c - - | - if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi - "/usr/local/bin/uwsgi" \ - --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \ - --http :80 \ - --pyargv -vv -# entrypoint: -# - "/usr/local/bin/uwsgi" -# - "--wsgi-file" -# - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py" -# - "--http" -# - ":80" - # command: "--pyargv -vv" +# cic-eth-server: +# build: +# context: apps/ +# dockerfile: cic-eth/docker/Dockerfile +# environment: +# CIC_CHAIN_SPEC: $CIC_CHAIN_SPEC +# CELERY_BROKER_URL: $CELERY_BROKER_URL +# CELERY_RESULT_URL: $CELERY_RESULT_URL +# SERVER_PORT: 8000 +# depends_on: +# - eth +# - postgres +# - redis +# ports: +# - ${HTTP_PORT_CIC_ETH:-63314}:8000 +# deploy: +# restart_policy: +# condition: on-failure +# volumes: +# - contract-config:/tmp/cic/config/:ro +# command: +# - /bin/bash +# - -c +# - | +# if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi +# "/usr/local/bin/uwsgi" \ +# --wsgi-file /usr/src/cic-eth/cic_eth/runnable/server_agent.py \ +# --http :80 \ +# --pyargv -vv +## entrypoint: +## - "/usr/local/bin/uwsgi" +## - "--wsgi-file" +## - "/usr/src/cic-eth/cic_eth/runnable/server_agent.py" +## - "--http" +## - ":80" +# # command: "--pyargv -vv"