grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Finish cic-cache integration for transaction listings

Browse Source
This commit is contained in:
Louis Holbrook
2021-08-17 08:03:14 +00:00
parent defa7797dc
commit 0ec9813e5f
33 changed files with 833 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-eth/cic_eth/api/api_task.py
View File

@@ -520,9 +520,9 @@ class Api(ApiBase):
s_external_get = celery.signature(
external_task,
[
address,
offset,
limit,
address,
],
queue=external_queue,
)

18
apps/cic-eth/cic_eth/cli/rpc.py
View File

@@ -24,20 +24,24 @@ class RPC:
def get_default(self):
return RPCConnection.connect(self.chain_spec, 'default')
return self.get_by_label('default')
def get_by_label(self, label):
return RPCConnection.connect(self.chain_spec, label)
@staticmethod
def from_config(config, use_signer=False):
def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
if use_signer:
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label)
rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
logg.info('set up rpc: {}'.format(rpc))
return rpc

2
apps/cic-eth/cic_eth/data/config/dispatcher.ini Normal file
View File

@@ -0,0 +1,2 @@
[dispatcher]
loop_interval = 1

2
apps/cic-eth/cic_eth/data/config/eth.ini
View File

@@ -1,2 +1,2 @@
[eth]
gas_gifter_minimum_balance = 10000000000000000000
gas_gifter_minimum_balance = 10000000000000000000000

3
apps/cic-eth/cic_eth/data/config/retry.ini Normal file
View File

@@ -0,0 +1,3 @@
[retry]
delay =
batch_size =

58
apps/cic-eth/cic_eth/ext/tx.py
View File

@@ -12,9 +12,11 @@ from chainlib.eth.tx import (
transaction_by_block,
receipt,
)
from chainlib.eth.error import RequestMismatchException
from chainlib.eth.block import block_by_number
from chainlib.eth.contract import abi_decode_single
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.tx import Tx
from hexathon import strip_0x
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
@@ -23,6 +25,8 @@ from chainqueue.db.models.otx import Otx
from chainqueue.db.enum import StatusEnum
from chainqueue.sql.query import get_tx_cache
from eth_erc20 import ERC20
from erc20_faucet import Faucet
from potaahto.symbols import snake_and_camel
# local imports
from cic_eth.queue.time import tx_times
@@ -35,6 +39,32 @@ logg = logging.getLogger()
MAX_BLOCK_TX = 250
def parse_transaction(chain_spec, rpc, tx, sender_address=None):
try:
transfer_data = ERC20.parse_transfer_request(tx['input'])
tx_address = transfer_data[0]
tx_token_value = transfer_data[1]
logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
return (tx_address, tx_token_value)
except RequestMismatchException:
pass
try:
transfer_data = Faucet.parse_give_to_request(tx['input'])
tx_address = transfer_data[0]
c = Faucet(chain_spec)
o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number'])
r = rpc.do(o)
tx_token_value = Faucet.parse_token_amount(r)
logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
return (tx_address, tx_token_value)
except RequestMismatchException:
pass
return None
# TODO: Make this method easier to read
@celery_app.task(bind=True, base=BaseTask)
def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
@@ -71,36 +101,39 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data)
txs = {}
logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high']))
for block_height in range(bloomspec['low'], bloomspec['high']):
block_height_bytes = block_height.to_bytes(4, 'big')
if block_filter.check(block_height_bytes):
logg.debug('filter matched block {}'.format(block_height))
o = block_by_number(block_height)
block = rpc.do(o)
logg.debug('block {}'.format(block))
for tx_index in range(0, len(block['transactions'])):
composite = tx_index + block_height
tx_index_bytes = composite.to_bytes(4, 'big')
if tx_filter.check(tx_index_bytes):
tx_index_bytes = tx_index.to_bytes(4, 'big')
composite = block_height_bytes + tx_index_bytes
if tx_filter.check(composite):
logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
o = transaction_by_block(block['hash'], tx_index)
try:
#tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
o = transaction_by_block(block['hash'], tx_index)
tx = rpc.do(o)
except Exception as e:
logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e))
continue
tx = Tx(tx).src()
logg.debug('got tx {}'.format(tx))
tx_address = None
tx_token_value = 0
try:
transfer_data = ERC20.parse_transfer_request(tx['data'])
tx_address = transfer_data[0]
tx_token_value = transfer_data[1]
except ValueError:
logg.debug('not a transfer transaction, skipping {}'.format(tx))
transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address)
if transfer_data == None:
continue
tx_address = transfer_data[0]
tx_token_value = transfer_data[1]
if address == tx_address:
status = StatusEnum.SENT
try:
@@ -136,6 +169,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
return txs
# TODO: Surely it must be possible to optimize this
# TODO: DRY this with callback filter in cic_eth/runnable/manager
# TODO: Remove redundant fields from end representation (timestamp, tx_hash)

10
apps/cic-eth/config/database.ini
View File

@@ -1,10 +0,0 @@
[database]
NAME=cic_eth
USER=postgres
PASSWORD=tralala
HOST=localhost
PORT=63432
ENGINE=postgresql
DRIVER=psycopg2
POOL_SIZE=50
DEBUG=0

2
apps/cic-eth/config/docker/chain.ini Normal file
View File

@@ -0,0 +1,2 @@
[chain]
spec = evm:bloxberg:8996

10
apps/cic-eth/config/docker/database.ini
View File

@@ -1,10 +0,0 @@
[database]
NAME=cic_eth
USER=postgres
PASSWORD=tralala
HOST=localhost
PORT=63432
ENGINE=postgresql
DRIVER=psycopg2
POOL_SIZE=50
DEBUG=0

2
apps/cic-eth/requirements.txt
View File

@@ -1,3 +1,3 @@
celery==4.4.7
chainlib-eth>=0.0.7a1,<0.1.0
chainlib-eth>=0.0.7a5,<0.1.0
semver==2.13.0

2
apps/cic-eth/services_requirements.txt
View File

@@ -9,7 +9,7 @@ liveness~=0.0.1a7
eth-address-index>=0.1.4a1,<0.2.0
eth-accounts-index>=0.0.14a1,<0.1.0
cic-eth-registry>=0.5.8a1,<0.6.0
erc20-faucet>=0.2.4a1,<0.3.0
erc20-faucet>=0.2.4a2,<0.3.0
erc20-transfer-authorization>=0.3.4a1,<0.4.0
sarafu-faucet>=0.0.5a2,<0.1.0
moolb~=0.1.1b2

17
apps/cic-eth/tests/unit/ext/test_ext_tx.py → apps/cic-eth/tests/task/test_ext_tx.py
View File

@@ -20,7 +20,6 @@ from cic_eth.db.models.nonce import (
logg = logging.getLogger()
# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state
def test_filter_process(
init_database,
default_chain_spec,
@@ -48,10 +47,10 @@ def test_filter_process(
eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
a = r['block_number']
b.add(a.to_bytes(4, 'big'))
a = r['block_number'] + r['transaction_index']
t.add(a.to_bytes(4, 'big'))
block_bytes = r['block_number'].to_bytes(4, 'big')
b.add(block_bytes)
tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
t.add(block_bytes + tx_index_bytes)
tx_hashes.append(tx_hash_hex)
# external tx
@@ -61,10 +60,10 @@ def test_filter_process(
eth_rpc.do(o)
o = receipt(tx_hash_hex)
r = eth_rpc.do(o)
a = r['block_number']
b.add(a.to_bytes(4, 'big'))
a = r['block_number'] + r['transaction_index']
t.add(a.to_bytes(4, 'big'))
block_bytes = r['block_number'].to_bytes(4, 'big')
b.add(block_bytes)
tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
t.add(block_bytes + tx_index_bytes)
tx_hashes.append(tx_hash_hex)
init_eth_tester.mine_blocks(10)

2
apps/cic-eth/tests/testdata/config/test.ini vendored Normal file
View File

@@ -0,0 +1,2 @@
[foo]
bar_baz = xyzzy

56
apps/cic-eth/tests/unit/cli/test_cli_args.py Normal file
View File

@@ -0,0 +1,56 @@
# standard imports
import os
import logging
# external imports
import chainlib.cli
# local imports
import cic_eth.cli
logg = logging.getLogger()
script_dir = os.path.dirname(os.path.realpath(__file__))
#config_dir = os.path.join(script_dir, '..', '..', 'testdata', 'config')
def test_argumentparser_to_config():
argparser = cic_eth.cli.ArgumentParser()
local_flags = 0xffff
argparser.process_local_flags(local_flags)
argparser.add_argument('--foo', type=str)
args = argparser.parse_args([
'--redis-host', 'foo',
'--redis-port', '123',
'--redis-db', '0',
'--redis-host-callback', 'bar',
'--redis-port-callback', '456',
'--redis-timeout', '10.0',
'-q', 'baz',
'--offset', '13',
'--no-history',
'-r','0xdeadbeef',
'-vv',
'--foo', 'bar',
])
extra_args = {
'foo': '_BARBARBAR',
}
#config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args)
assert config.get('_BARBARBAR') == 'bar'
assert config.get('REDIS_HOST') == 'foo'
assert config.get('REDIS_PORT') == 123
assert config.get('REDIS_DB') == 0
assert config.get('_REDIS_HOST_CALLBACK') == 'bar'
assert config.get('_REDIS_PORT_CALLBACK') == 456
assert config.get('REDIS_TIMEOUT') == 10.0
assert config.get('CELERY_QUEUE') == 'baz'
assert config.get('SYNCER_NO_HISTORY') == True
assert config.get('SYNCER_OFFSET') == 13
assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'

17
apps/cic-eth/tests/unit/cli/test_cli_celery.py Normal file
View File

@@ -0,0 +1,17 @@
# standard imports
import tempfile
# local imports
import cic_eth.cli
def test_cli_celery():
cf = tempfile.mkdtemp()
config = {
'CELERY_RESULT_URL': 'filesystem://' + cf,
}
cic_eth.cli.CeleryApp.from_config(config)
config['CELERY_BROKER_URL'] = 'filesystem://' + cf
cic_eth.cli.CeleryApp.from_config(config)

64
apps/cic-eth/tests/unit/cli/test_cli_chain.py Normal file
View File

@@ -0,0 +1,64 @@
# external imports
import pytest
from chainlib.eth.gas import (
Gas,
RPCGasOracle,
)
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.block import (
block_latest,
Block,
)
# local imports
import cic_eth.cli
@pytest.mark.xfail()
def test_cli_rpc(
eth_rpc,
eth_signer,
default_chain_spec,
):
config = {
'CHAIN_SPEC': str(default_chain_spec),
'RPC_HTTP_PROVIDER': 'http://localhost:8545',
}
rpc = cic_eth.cli.RPC.from_config(config, default_label='foo')
conn = rpc.get_by_label('foo')
#o = block_latest()
#conn.do(o)
def test_cli_chain(
default_chain_spec,
eth_rpc,
eth_signer,
contract_roles,
agent_roles,
):
ifc = cic_eth.cli.EthChainInterface()
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
gas_oracle = RPCGasOracle(conn=eth_rpc)
c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
(tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], 1024)
r = eth_rpc.do(o)
o = ifc.tx_receipt(r)
r = eth_rpc.do(o)
assert r['status'] == 1
o = ifc.block_by_number(1)
block_src = eth_rpc.do(o)
block = ifc.block_from_src(block_src)
assert block.number == 1
with pytest.raises(KeyError):
assert block_src['gasUsed'] == 21000
assert block_src['gas_used'] == 21000
block_src = ifc.src_normalize(block_src)
assert block_src['gasUsed'] == 21000
assert block_src['gas_used'] == 21000