Compare commits

..

23 Commits

Author SHA1 Message Date
nolash
e52db0de35 Do not callback when none in token proofs check 2021-10-14 15:33:54 +02:00
nolash
cc58735ff1 Do not link null callbacks for token proof check 2021-10-14 15:31:58 +02:00
f42f223ce9 Merge branch 'lash/api-choose-token' into 'master'
feat: Add token metadata API

Closes #118

See merge request grassrootseconomics/cic-internal-integration!289
2021-10-14 13:24:51 +00:00
Louis Holbrook
045a279ac2 feat: Add token metadata API 2021-10-14 13:24:51 +00:00
nolash
4b8eae41f7 Allow empty proofs in token info task 2021-10-14 12:22:41 +02:00
nolash
8d571b51c5 More deps upgrades 2021-10-14 05:28:17 +02:00
nolash
13970ec332 bump version cic-eth 2021-10-13 15:14:55 +02:00
nolash
7e6cc5714d Allow empty proof spec for tokens api 2021-10-13 15:13:43 +02:00
nolash
0f5bceb95b Bump version cic-eth 2021-10-13 14:24:29 +02:00
nolash
159fd07d63 Revert "Another attempt at config files, dockerfile change"
This reverts commit 9d3f8bf1e3.
2021-10-09 17:22:23 +02:00
nolash
9d3f8bf1e3 Another attempt at config files, dockerfile change 2021-10-09 17:12:29 +02:00
nolash
2229154c2b Add documentation 2021-10-09 16:55:05 +02:00
nolash
9637cb61f8 Add magic arg parsing for token lookup api 2021-10-09 16:21:03 +02:00
nolash
66f94ae694 Correct test default token task in unit test 2021-10-09 15:06:21 +02:00
nolash
1e65ea8511 reinstate celery seesion worker in token metadata test 2021-10-09 14:38:16 +02:00
nolash
21972e9df5 Complete token lookups with proofs 2021-10-09 14:18:10 +02:00
nolash
77fe41da4b Revert to multi-token task queue entry point for token api call 2021-10-09 13:39:33 +02:00
nolash
6098374a4e POC single token task chain for token proof lookups 2021-10-09 13:22:14 +02:00
nolash
f648d3ee68 WIP token info proof verify 2021-10-09 00:48:20 +02:00
nolash
7e2857d1e7 Add separate generic task for proof verification 2021-10-08 20:19:15 +02:00
nolash
df21db958b Move token tasks to cic_eth.eth.erc20 2021-10-08 19:21:47 +02:00
nolash
6ccffb15b6 Add trust check to token fetch in cic-eth task 2021-10-08 18:39:22 +02:00
nolash
53e9c63698 Add token api metadata getter (first without okota) 2021-10-08 16:04:13 +02:00
20 changed files with 537 additions and 42 deletions

View File

@@ -1,5 +1,5 @@
SQLAlchemy==1.3.20
cic-eth-registry>=0.6.1a3,<0.7.0
cic-eth-registry>=0.6.1a5,<0.7.0
hexathon~=0.0.1a8
chainqueue>=0.0.4a6,<0.1.0
eth-erc20>=0.1.2a2,<0.2.0

View File

@@ -1,21 +1,2 @@
# standard imports
import logging
# external imports
import celery
# local imports
from cic_eth.task import BaseTask
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task(bind=True, base=BaseTask)
def default_token(self):
return {
'symbol': self.default_token_symbol,
'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
}
from cic_eth.eth.erc20 import default_token

View File

@@ -17,15 +17,50 @@ from cic_eth.enum import LockEnum
app = celery.current_app
logg = logging.getLogger(__name__)
#logg = logging.getLogger(__name__)
logg = logging.getLogger()
class Api(ApiBase):
@staticmethod
def to_v_list(v, n):
"""Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
:param v: Arguments
:type v: str or list
:param n: Number of elements to generate arguments for
:type n: int
:rtype: list
:returns: list of assembled arguments
"""
if isinstance(v, str):
vv = v
v = []
for i in range(n):
v.append([vv])
elif not isinstance(v, list):
raise ValueError('argument must be single string, or list or strings or lists')
else:
if len(v) != n:
raise ValueError('v argument count must match integer n')
for i in range(n):
if isinstance(v[i], str):
v[i] = [v[i]]
elif not isinstance(v, list):
raise ValueError('proof argument must be single string, or list or strings or lists')
return v
def default_token(self):
"""Retrieves the default fallback token of the custodial network.
:returns: uuid of root task
:rtype: celery.Task
"""
s_token = celery.signature(
'cic_eth.admin.token.default_token',
'cic_eth.eth.erc20.default_token',
[],
queue=self.queue,
)
@@ -35,6 +70,97 @@ class Api(ApiBase):
return s_token.apply_async()
def token(self, token_symbol, proof=None):
"""Single-token alias for tokens method.
See tokens method for details.
:param token_symbol: Token symbol to look up
:type token_symbol: str
:param proof: Proofs to add to signature verification for the token
:type proof: str or list
:returns: uuid of root task
:rtype: celery.Task
"""
if not isinstance(token_symbol, str):
raise ValueError('token symbol must be string')
return self.tokens([token_symbol], proof=proof)
def tokens(self, token_symbols, proof=None):
"""Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
Token symbols are always strings, and should be specified using uppercase letters.
If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
The proof argument may be specified in a number of ways:
- as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
- as a single string, where the same proof is used for each token lookup
- as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
- as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
This method is not intended to be used synchronously. Do so at your peril.
:param token_symbols: Token symbol strings to look up
:type token_symbol: list
:param proof: Proof(s) to verify tokens against
:type proof: None, str or list
:returns: uuid of root task
:rtype: celery.Task
"""
if not isinstance(token_symbols, list):
raise ValueError('token symbols argument must be list')
if proof == None:
logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
proof = ''
logg.debug('proof is {}'.format(proof))
l = len(token_symbols)
if len(proof) == 0:
l = 0
proof = Api.to_v_list(proof, l)
chain_spec_dict = self.chain_spec.asdict()
s_token_resolve = celery.signature(
'cic_eth.eth.erc20.resolve_tokens_by_symbol',
[
token_symbols,
chain_spec_dict,
],
queue=self.queue,
)
s_token_info = celery.signature(
'cic_eth.eth.erc20.token_info',
[
chain_spec_dict,
proof,
],
queue=self.queue,
)
s_token_verify = celery.signature(
'cic_eth.eth.erc20.verify_token_info',
[
chain_spec_dict,
self.callback_success,
self.callback_error,
],
queue=self.queue,
)
s_token_info.link(s_token_verify)
s_token_resolve.link(s_token_info)
return s_token_resolve.apply_async()
# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
#

View File

@@ -1,7 +1,10 @@
import logging
import celery
celery_app = celery.current_app
logg = celery_app.log.get_default_logger()
#logg = celery_app.log.get_default_logger()
logg = logging.getLogger()
@celery_app.task(bind=True)

View File

@@ -48,8 +48,6 @@ class RoleMissingError(Exception):
pass
class IntegrityError(Exception):
"""Exception raised to signal irregularities with deduplication and ordering of tasks
@@ -85,3 +83,8 @@ class RoleAgencyError(SeppukuError):
class YouAreBrokeError(Exception):
"""Exception raised when a value transfer is attempted without access to sufficient funds
"""
class TrustError(Exception):
"""Exception raised when required trust proofs are missing for a request
"""

View File

@@ -19,6 +19,7 @@ from hexathon import (
from chainqueue.error import NotLocalTxError
from eth_erc20 import ERC20
from chainqueue.sql.tx import cache_tx_dict
from okota.token_index import to_identifier
# local imports
from cic_eth.db.models.base import SessionBase
@@ -39,9 +40,11 @@ from cic_eth.task import (
CriticalSQLAlchemyTask,
CriticalWeb3Task,
CriticalSQLAlchemyAndSignerTask,
BaseTask,
)
from cic_eth.eth.nonce import CustodialTaskNonceOracle
from cic_eth.encode import tx_normalize
from cic_eth.eth.trust import verify_proofs
celery_app = celery.current_app
logg = logging.getLogger()
@@ -473,3 +476,69 @@ def cache_approve_data(
session.close()
return (tx_hash_hex, cache_id)
@celery_app.task(bind=True, base=BaseTask)
def token_info(self, tokens, chain_spec_dict, proofs=[]):
chain_spec = ChainSpec.from_dict(chain_spec_dict)
rpc = RPCConnection.connect(chain_spec, 'default')
i = 0
for token in tokens:
result_data = []
token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
token_chain_object.load(rpc)
token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
token_proofs = [token_symbol_proof_hex]
if len(proofs) > 0:
token_proofs += proofs[i]
tokens[i] = {
'decimals': token_chain_object.decimals,
'name': token_chain_object.name,
'symbol': token_chain_object.symbol,
'address': tx_normalize.executable_address(token_chain_object.address),
'proofs': token_proofs,
'converters': tokens[i]['converters'],
}
i += 1
return tokens
@celery_app.task(bind=True, base=BaseTask)
def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
queue = self.request.delivery_info.get('routing_key')
for token in tokens:
s = celery.signature(
'cic_eth.eth.trust.verify_proofs',
[
token,
token['address'],
token['proofs'],
chain_spec_dict,
success_callback,
error_callback,
],
queue=queue,
)
if success_callback != None:
s.link(success_callback)
if error_callback != None:
s.on_error(error_callback)
s.apply_async()
return tokens
@celery_app.task(bind=True, base=BaseTask)
def default_token(self):
return {
'symbol': self.default_token_symbol,
'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
}

View File

@@ -0,0 +1,77 @@
# standard imports
import logging
# external imports
import celery
from eth_address_declarator import Declarator
from chainlib.connection import RPCConnection
from chainlib.chain import ChainSpec
from cic_eth.db.models.role import AccountRole
from cic_eth_registry import CICRegistry
from hexathon import strip_0x
# local imports
from cic_eth.task import BaseTask
from cic_eth.error import TrustError
celery_app = celery.current_app
logg = logging.getLogger()
@celery_app.task(bind=True, base=BaseTask)
def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
proof = strip_0x(proof)
proofs = []
logg.debug('proof count {}'.format(len(proofs)))
if len(proofs) == 0:
logg.debug('error {}'.format(len(proofs)))
raise TrustError('foo')
return (chained_input, (proof, proofs))
@celery_app.task(bind=True, base=BaseTask)
def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
queue = self.request.delivery_info.get('routing_key')
chain_spec = ChainSpec.from_dict(chain_spec_dict)
rpc = RPCConnection.connect(chain_spec, 'default')
session = self.create_session()
sender_address = AccountRole.get_address('DEFAULT', session)
registry = CICRegistry(chain_spec, rpc)
declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
declarator = Declarator(chain_spec)
have_proofs = {}
for proof in proofs:
proof = strip_0x(proof)
have_proofs[proof] = []
for trusted_address in self.trusted_addresses:
o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
r = rpc.do(o)
declarations = declarator.parse_declaration(r)
logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
for declaration in declarations:
declaration = strip_0x(declaration)
if declaration == proof:
logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
have_proofs[proof].append(trusted_address)
out_proofs = {}
for proof in have_proofs.keys():
if len(have_proofs[proof]) == 0:
logg.error('missing signer for proof {} subject {}'.format(proof, subject))
raise TrustError((subject, proof,))
out_proofs[proof] = have_proofs[proof]
return (chained_input, out_proofs)

View File

@@ -4,18 +4,21 @@ import tempfile
import logging
import shutil
# local impors
# local imports
from cic_eth.task import BaseTask
#logg = logging.getLogger(__name__)
logg = logging.getLogger()
@pytest.fixture(scope='function')
def init_celery_tasks(
contract_roles,
):
BaseTask.call_address = contract_roles['DEFAULT']
BaseTask.trusted_addresses = [
contract_roles['TRUSTED_DECLARATOR'],
contract_roles['CONTRACT_DEPLOYER'],
]
# celery fixtures
@@ -38,6 +41,7 @@ def celery_includes():
'cic_eth.callbacks.noop',
'cic_eth.callbacks.http',
'cic_eth.pytest.mock.filter',
'cic_eth.pytest.mock.callback',
]

View File

@@ -1 +1,2 @@
from .filter import *
from .callback import *

View File

@@ -0,0 +1,38 @@
# standard imports
import os
import logging
import mmap
# standard imports
import tempfile
# external imports
import celery
#logg = logging.getLogger(__name__)
logg = logging.getLogger()
celery_app = celery.current_app
class CallbackTask(celery.Task):
mmap_path = tempfile.mkdtemp()
@celery_app.task(bind=True, base=CallbackTask)
def test_callback(self, a, b, c):
s = 'ok'
if c > 0:
s = 'err'
fp = os.path.join(self.mmap_path, b)
f = open(fp, 'wb+')
f.write(b'\x00')
f.seek(0)
m = mmap.mmap(f.fileno(), length=1)
m.write(c.to_bytes(1, 'big'))
m.close()
f.close()
logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))

View File

@@ -210,6 +210,7 @@ def main():
default_token.load(conn)
BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name
BaseTask.trusted_addresses = trusted_addresses
BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

View File

@@ -28,6 +28,7 @@ class BaseTask(celery.Task):
session_func = SessionBase.create_session
call_address = ZERO_ADDRESS
trusted_addresses = []
create_nonce_oracle = RPCNonceOracle
create_gas_oracle = RPCGasOracle
default_token_address = None

View File

@@ -10,7 +10,7 @@ version = (
0,
12,
4,
'alpha.8',
'alpha.11',
)
version_object = semver.VersionInfo(

View File

@@ -1,4 +1,4 @@
celery==4.4.7
chainlib-eth>=0.0.9rc2,<0.1.0
chainlib-eth>=0.0.9rc4,<0.1.0
semver==2.13.0
crypto-dev-signer>=0.4.15rc2,<0.5.0

View File

@@ -6,10 +6,11 @@ redis==3.5.3
hexathon~=0.0.1a8
pycryptodome==3.10.1
liveness~=0.0.1a7
eth-address-index>=0.2.3a4,<0.3.0
eth-address-index>=0.2.4a1,<0.3.0
eth-accounts-index>=0.1.2a3,<0.2.0
cic-eth-registry>=0.6.1a3,<0.7.0
cic-eth-registry>=0.6.1a5,<0.7.0
erc20-faucet>=0.3.2a2,<0.4.0
erc20-transfer-authorization>=0.3.5a2,<0.4.0
sarafu-faucet>=0.0.7a2,<0.1.0
moolb~=0.1.1b2
okota>=0.2.4a6,<0.3.0

View File

@@ -1,6 +1,27 @@
# standard imports
import logging
import os
import uuid
import time
import mmap
# external imports
import celery
import pytest
from hexathon import (
strip_0x,
uniform as hex_uniform,
)
# local imports
from cic_eth.api.api_task import Api
from cic_eth.task import BaseTask
from cic_eth.error import TrustError
from cic_eth.encode import tx_normalize
from cic_eth.pytest.mock.callback import CallbackTask
logg = logging.getLogger()
def test_default_token(
default_chain_spec,
@@ -17,3 +38,175 @@ def test_default_token(
t = api.default_token()
r = t.get_leaf()
assert r['address'] == foo_token
def test_to_v_list():
assert Api.to_v_list('', 0) == []
assert Api.to_v_list([], 0) == []
assert Api.to_v_list('foo', 1) == [['foo']]
assert Api.to_v_list(['foo'], 1) == [['foo']]
assert Api.to_v_list(['foo', 'bar'], 2) == [['foo'], ['bar']]
assert Api.to_v_list('foo', 3) == [['foo'], ['foo'], ['foo']]
assert Api.to_v_list([['foo'], ['bar']], 2) == [['foo'], ['bar']]
with pytest.raises(ValueError):
Api.to_v_list([['foo'], ['bar']], 3)
with pytest.raises(ValueError):
Api.to_v_list(['foo', 'bar'], 3)
with pytest.raises(ValueError):
Api.to_v_list([['foo'], ['bar'], ['baz']], 2)
assert Api.to_v_list([
['foo'],
'bar',
['inky', 'pinky', 'blinky', 'clyde'],
], 3) == [
['foo'],
['bar'],
['inky', 'pinky', 'blinky', 'clyde'],
]
def test_token_single(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_session_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.token('FOO', proof=None)
r = t.get()
logg.debug('rr {}'.format(r))
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.token('FOO', proof=foo_token_declaration)
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
def test_tokens_noproof(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.tokens(['FOO'], proof=[])
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.tokens(['BAR'], proof='')
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(bar_token)
t = api.tokens(['FOO'], proof=None)
r = t.get()
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
def test_tokens(
default_chain_spec,
foo_token,
bar_token,
token_registry,
register_tokens,
register_lookups,
cic_registry,
init_database,
init_celery_tasks,
custodial_roles,
foo_token_declaration,
bar_token_declaration,
celery_session_worker,
):
api = Api(str(default_chain_spec), queue=None, callback_param='foo')
t = api.tokens(['FOO'], proof=[[foo_token_declaration]])
r = t.get()
logg.debug('rr {}'.format(r))
assert len(r) == 1
assert r[0]['address'] == strip_0x(foo_token)
t = api.tokens(['BAR', 'FOO'], proof=[[bar_token_declaration], [foo_token_declaration]])
r = t.get()
logg.debug('results {}'.format(r))
assert len(r) == 2
assert r[1]['address'] == strip_0x(foo_token)
assert r[0]['address'] == strip_0x(bar_token)
celery_app = celery.current_app
results = []
targets = []
api_param = str(uuid.uuid4())
api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
bogus_proof = os.urandom(32).hex()
t = api.tokens(['FOO'], proof=[[bogus_proof]])
r = t.get()
logg.debug('r {}'.format(r))
while True:
fp = os.path.join(CallbackTask.mmap_path, api_param)
try:
f = open(fp, 'rb')
except FileNotFoundError:
time.sleep(0.1)
logg.debug('look for {}'.format(fp))
continue
f = open(fp, 'rb')
m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
v = m.read(1)
m.close()
f.close()
assert v == b'\x01'
break
api_param = str(uuid.uuid4())
api = Api(str(default_chain_spec), queue=None, callback_param=api_param, callback_task='cic_eth.pytest.mock.callback.test_callback')
t = api.tokens(['BAR'], proof=[[bar_token_declaration]])
r = t.get()
logg.debug('rr {} {}'.format(r, t.children))
while True:
fp = os.path.join(CallbackTask.mmap_path, api_param)
try:
f = open(fp, 'rb')
except FileNotFoundError:
time.sleep(0.1)
continue
m = mmap.mmap(f.fileno(), access=mmap.ACCESS_READ, length=1)
v = m.read(1)
m.close()
f.close()
assert v == b'\x00'
break

View File

@@ -10,7 +10,7 @@ def test_default_token(
):
s = celery.signature(
'cic_eth.admin.token.default_token',
'cic_eth.eth.erc20.default_token',
[],
queue=None,
)

View File

@@ -1,6 +1,6 @@
crypto-dev-signer>=0.4.15a7,<=0.4.15
crypto-dev-signer>=0.4.15rc2,<=0.4.15
chainqueue>=0.0.5a1,<0.1.0
cic-eth-registry>=0.6.1a3,<0.7.0
cic-eth-registry>=0.6.1a5,<0.7.0
redis==3.5.3
hexathon~=0.0.1a8
pycryptodome==3.10.1

View File

@@ -1,11 +1,11 @@
cic-eth[tools]==0.12.4a8
chainlib-eth>=0.0.9rc4,<0.0.10
cic-eth[tools]==0.12.4a11
chainlib-eth>=0.0.9rc4,<0.1.0
chainlib==0.0.9rc1,<0.1.0
eth-erc20>=0.1.2a3,<0.2.0
erc20-demurrage-token>=0.0.5a2,<0.1.0
#eth-accounts-index>=0.1.2a2,<0.2.0
eth-address-index>=0.2.4a1,<0.3.0
cic-eth-registry>=0.6.1a2,<0.7.0
cic-eth-registry>=0.6.1a5,<0.7.0
erc20-transfer-authorization>=0.3.5a2,<0.4.0
erc20-faucet>=0.3.2a2,<0.4.0
sarafu-faucet>=0.0.7a2,<0.1.0

View File

@@ -121,9 +121,6 @@ export CIC_DEFAULT_TOKEN_SYMBOL=$CIC_DEFAULT_TOKEN_SYMBOL
export TOKEN_NAME=$TOKEN_NAME
" >> "${DEV_DATA_DIR}"/env_reset
confini-dump --schema-dir ./config
eth-contract-registry-list -i $CIC_CHAIN_SPEC -p $RPC_PROVIDER -e $CIC_REGISTRY_ADDRESS -u
set +a
set +e