grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: Improve cic-cache

Browse Source
This commit is contained in:
Louis Holbrook
2022-01-04 16:01:01 +00:00
committed by Philip Wafula
parent b5653a704c
commit 104ff8a76a
74 changed files with 1057 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
apps/cic-eth/admin_requirements.txt
View File

@@ -1,5 +1,4 @@
SQLAlchemy==1.3.20
cic-eth-registry>=0.6.1a6,<0.7.0
hexathon~=0.0.1a8
chainqueue>=0.0.4a6,<0.1.0
eth-erc20>=0.1.2a2,<0.2.0
hexathon~=0.1.0
chainqueue~=0.0.6a4
eth-erc20~=0.1.5

14
apps/cic-eth/cic_eth/cli/config.py
View File

@@ -63,22 +63,32 @@ class Config(BaseConfig):
config.get('REDIS_HOST'),
config.get('REDIS_PORT'),
)
db = getattr(args, 'redis_db', None)
if db != None:
db = str(db)
redis_url = (
'redis',
hostport,
getattr(args, 'redis_db', None),
db,
)
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
hostport = urlhostmerge(
celery_config_url[1],
getattr(args, 'celery_host', None),
getattr(args, 'celery_port', None),
)
db = getattr(args, 'redis_db', None)
if db != None:
db = str(db)
celery_arg_url = (
getattr(args, 'celery_scheme', None),
hostport,
getattr(args, 'celery_db', None),
db,
)
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
celery_url_string = urllib.parse.urlunsplit(celery_url)
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string

10
apps/cic-eth/cic_eth/eth/erc20.py
View File

@@ -22,7 +22,7 @@ from hexathon import (
from chainqueue.error import NotLocalTxError
from eth_erc20 import ERC20
from chainqueue.sql.tx import cache_tx_dict
from okota.token_index import to_identifier
from okota.token_index.index import to_identifier
# local imports
from cic_eth.db.models.base import SessionBase
@@ -46,13 +46,14 @@ from cic_eth.task import (
from cic_eth.eth.nonce import CustodialTaskNonceOracle
from cic_eth.encode import tx_normalize
from cic_eth.eth.trust import verify_proofs
from cic_eth.error import SignerError
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task(base=CriticalWeb3Task)
def balance(tokens, holder_address, chain_spec_dict):
@celery_app.task(bind=True, base=CriticalWeb3Task)
def balance(self, tokens, holder_address, chain_spec_dict):
"""Return token balances for a list of tokens for given address
:param tokens: Token addresses
@@ -71,8 +72,9 @@ def balance(tokens, holder_address, chain_spec_dict):
for t in tokens:
address = t['address']
logg.debug('address {} {}'.format(address, holder_address))
gas_oracle = self.create_gas_oracle(rpc, min_price=self.min_fee_price)
token = ERC20Token(chain_spec, rpc, add_0x(address))
c = ERC20(chain_spec)
c = ERC20(chain_spec, gas_oracle=gas_oracle)
o = c.balance_of(address, holder_address, sender_address=caller_address)
r = rpc.do(o)
t['balance_network'] = c.parse_balance(r)

2
apps/cic-eth/cic_eth/eth/gas.py
View File

@@ -92,7 +92,7 @@ def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
if o == None:
o = GasCache(address, method, value, tx_hash)
elif tx.gas_used > o.value:
elif value > o.value:
o.value = value
o.tx_hash = strip_0x(tx_hash)

8
apps/cic-eth/cic_eth/task.py
View File

@@ -25,12 +25,14 @@ logg = logging.getLogger()
celery_app = celery.current_app
class BaseTask(celery.Task):
session_func = SessionBase.create_session
call_address = ZERO_ADDRESS
trusted_addresses = []
min_fee_price = 1
min_fee_limit = 30000
default_token_address = None
default_token_symbol = None
default_token_name = None
@@ -42,7 +44,7 @@ class BaseTask(celery.Task):
if address == None:
return RPCGasOracle(
conn,
code_callback=kwargs.get('code_callback'),
code_callback=kwargs.get('code_callback', self.get_min_fee_limit),
min_price=self.min_fee_price,
id_generator=kwargs.get('id_generator'),
)
@@ -56,6 +58,10 @@ class BaseTask(celery.Task):
)
def get_min_fee_limit(self, code):
return self.min_fee_limit
def create_session(self):
return BaseTask.session_func()

16
apps/cic-eth/docker/Dockerfile
View File

@@ -11,13 +11,6 @@ ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \
--pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a7
COPY *requirements.txt ./
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \
@@ -25,7 +18,7 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt \
-r services_requirements.txt \
-r admin_requirements.txt
-r admin_requirements.txt
COPY . .
RUN python setup.py install
@@ -40,8 +33,6 @@ RUN chmod 755 *.sh
# # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
#COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
# TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh
@@ -66,9 +57,8 @@ ENTRYPOINT []
## # they can all be overridden by environment variables
## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
#COPY config/ /usr/local/etc/cic-eth/
#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
#COPY scripts/ scripts/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
#COPY scripts/ scripts/
#
## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
##COPY util/liveness/health.sh /usr/local/bin/health.sh

7
apps/cic-eth/requirements.txt
View File

@@ -1,4 +1,7 @@
celery==4.4.7
chainlib-eth>=0.0.10a20,<0.1.0
semver==2.13.0
urlybird~=0.0.1a2
chainlib-eth~=0.0.15
urlybird~=0.0.1
cic-eth-registry~=0.6.6
cic-types~=0.2.1a8
cic-eth-aux-erc20-demurrage-token~=0.0.3

23
apps/cic-eth/services_requirements.txt
View File

@@ -1,16 +1,15 @@
chainqueue>=0.0.6a1,<0.1.0
chainsyncer[sql]>=0.0.7a3,<0.1.0
chainqueue~=0.0.6a4
chainsyncer[sql]~=0.0.7
alembic==1.4.2
confini>=0.3.6rc4,<0.5.0
confini~=0.5.3
redis==3.5.3
hexathon~=0.0.1a8
hexathon~=0.1.0
pycryptodome==3.10.1
liveness~=0.0.1a7
eth-address-index>=0.2.4a1,<0.3.0
eth-accounts-index>=0.1.2a3,<0.2.0
cic-eth-registry>=0.6.1a6,<0.7.0
erc20-faucet>=0.3.2a2,<0.4.0
erc20-transfer-authorization>=0.3.5a2,<0.4.0
sarafu-faucet>=0.0.7a2,<0.1.0
moolb~=0.1.1b2
okota>=0.2.4a6,<0.3.0
eth-address-index~=0.2.4
eth-accounts-index~=0.1.2
erc20-faucet~=0.3.2
erc20-transfer-authorization~=0.3.6
sarafu-faucet~=0.0.7
moolb~=0.2.0
okota~=0.2.5

2
apps/cic-eth/setup.cfg
View File

@@ -1,7 +1,7 @@
[metadata]
name = cic-eth
#version = attr: cic_eth.version.__version_string__
version = 0.12.5a2
version = 0.12.7
description = CIC Network Ethereum interaction
author = Louis Holbrook
author_email = dev@holbrook.no

3
apps/cic-eth/test_requirements.txt
View File

@@ -6,4 +6,5 @@ pytest-redis==2.0.0
redis==3.5.3
eth-tester==0.5.0b3
py-evm==0.3.0a20
eth-erc20~=0.1.2a2
eth-erc20~=0.1.5
erc20-transfer-authorization~=0.3.6

3
apps/cic-eth/tests/filters/test_token_filter.py
View File

@@ -40,6 +40,7 @@ def test_filter_gas(
foo_token,
token_registry,
register_lookups,
register_tokens,
celery_session_worker,
cic_registry,
):
@@ -69,7 +70,7 @@ def test_filter_gas(
tx = Tx(tx_src, block=block)
tx.apply_receipt(rcpt)
t = fltr.filter(eth_rpc, block, tx, db_session=init_database)
assert t == None
assert t.get() == None
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], eth_rpc)
c = TokenUniqueSymbolIndex(default_chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)

1
apps/cic-eth/tests/task/api/test_admin.py
View File

@@ -288,7 +288,6 @@ def test_fix_nonce(
init_database.commit()
logg.debug('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
txs = get_nonce_tx_local(default_chain_spec, 3, agent_roles['ALICE'], session=init_database)
ks = txs.keys()
assert len(ks) == 2

6
apps/cic-eth/tests/task/api/test_app_noncritical.py
View File

@@ -191,11 +191,17 @@ def test_tokens(
break
api_param = str(uuid.uuid4())
fp = os.path.join(CallbackTask.mmap_path, api_param)
f = open(fp, 'wb+')
f.write(b'\x00')
f.close()
api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
t = api.tokens(['BAR'], proof=[[bar_token_declaration]])
r = t.get()
logg.debug('rr {} {}'.format(r, t.children))
while True:
fp = os.path.join(CallbackTask.mmap_path, api_param)
try:

16
apps/cic-eth/tests/task/test_task_gas.py
View File

@@ -35,10 +35,26 @@ from hexathon import strip_0x
from cic_eth.eth.gas import cache_gas_data
from cic_eth.error import OutOfGasError
from cic_eth.queue.tx import queue_create
from cic_eth.task import BaseTask
logg = logging.getLogger()
def test_task_gas_limit(
eth_rpc,
eth_signer,
default_chain_spec,
agent_roles,
celery_session_worker,
):
rpc = RPCConnection.connect(default_chain_spec, 'default')
gas_oracle = BaseTask().create_gas_oracle(rpc)
c = Gas(default_chain_spec, signer=eth_signer, gas_oracle=gas_oracle)
(tx_hash_hex, o) = c.create(agent_roles['ALICE'], agent_roles['BOB'], 10, tx_format=TxFormat.RLP_SIGNED)
tx = unpack(bytes.fromhex(strip_0x(o)), default_chain_spec)
assert (tx['gas'], BaseTask.min_fee_price)
def test_task_check_gas_ok(
default_chain_spec,
eth_rpc,

6
apps/cic-eth/tests/unit/queue/test_balances.py
View File

@@ -143,7 +143,7 @@ def test_incoming_balance(
'converters': [],
}
b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 0
assert b[0]['balance_incoming'] == 1000
otx.readysend(session=init_database)
init_database.flush()
@@ -152,8 +152,8 @@ def test_incoming_balance(
otx.sent(session=init_database)
init_database.commit()
b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
assert b[0]['balance_incoming'] == 1000
#b = balance_incoming([token_data], recipient, default_chain_spec.asdict())
#assert b[0]['balance_incoming'] == 1000
otx.success(block=1024, session=init_database)
init_database.commit()

6
apps/cic-eth/tools_requirements.txt
View File

@@ -1,7 +1,5 @@
crypto-dev-signer>=0.4.15rc2,<=0.4.15
chainqueue>=0.0.5a3,<0.1.0
cic-eth-registry>=0.6.1a6,<0.7.0
chainqueue~=0.0.6a4
redis==3.5.3
hexathon~=0.0.1a8
hexathon~=0.1.0
pycryptodome==3.10.1
pyxdg==0.27