grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

migrations: Enable deployment and data seeding to Bloxberg

Browse Source
This commit is contained in:
Louis Holbrook
2021-12-22 18:24:05 +00:00
committed by Philip Wafula
parent b15cfee1c9
commit d7c4cb71eb
65 changed files with 940 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
apps/cic-eth/cic_eth/api/admin.py
View File

@@ -123,7 +123,7 @@ class AdminApi:
return s_lock.apply_async()
def tag_account(self, tag, address_hex, chain_spec):
def tag_account(self, chain_spec, tag, address):
"""Persistently associate an address with a plaintext tag.
Some tags are known by the system and is used to resolve addresses to use for certain transactions.
@@ -138,7 +138,7 @@ class AdminApi:
'cic_eth.eth.account.set_role',
[
tag,
address_hex,
address,
chain_spec.asdict(),
],
queue=self.queue,
@@ -146,6 +146,30 @@ class AdminApi:
return s_tag.apply_async()
def get_tag_account(self, chain_spec, tag=None, address=None):
if address != None:
s_tag = celery.signature(
'cic_eth.eth.account.role',
[
address,
chain_spec.asdict(),
],
queue=self.queue,
)
else:
s_tag = celery.signature(
'cic_eth.eth.account.role_account',
[
tag,
chain_spec.asdict(),
],
queue=self.queue,
)
return s_tag.apply_async()
def have_account(self, address_hex, chain_spec):
s_have = celery.signature(
'cic_eth.eth.account.have',
@@ -503,7 +527,7 @@ class AdminApi:
queue=self.queue,
)
t = s.apply_async()
role = t.get()
role = t.get()[0][1]
if role != None:
tx['sender_description'] = role
@@ -556,7 +580,7 @@ class AdminApi:
queue=self.queue,
)
t = s.apply_async()
role = t.get()
role = t.get()[0][1]
if role != None:
tx['recipient_description'] = role

26
apps/cic-eth/cic_eth/check/gas.py
View File

@@ -12,8 +12,9 @@ from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import LockEnum
from cic_eth.error import LockedError
from cic_eth.admin.ctrl import check_lock
from cic_eth.eth.gas import have_gas_minimum
logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger(__name__)
def health(*args, **kwargs):
@@ -31,18 +32,15 @@ def health(*args, **kwargs):
return True
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
if config.get('ETH_MIN_FEE_PRICE'):
min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
session.close()
if not r:
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
rpc = RPCConnection.connect(chain_spec, 'default')
o = balance(gas_provider)
r = rpc.do(o)
try:
r = int(r, 16)
except TypeError:
r = int(r)
gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
if r < gas_min:
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
return False
return True
return r

18
apps/cic-eth/cic_eth/check/start.py Normal file
View File

@@ -0,0 +1,18 @@
# external imports
from chainlib.chain import ChainSpec
# local imports
from cic_eth.admin.ctrl import check_lock
from cic_eth.enum import LockEnum
from cic_eth.error import LockedError
def health(*args, **kwargs):
config = kwargs['config']
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
try:
check_lock(None, chain_spec.asdict(), LockEnum.START)
except LockedError as e:
return False
return True

18
apps/cic-eth/cic_eth/cli/arg.py
View File

@@ -16,16 +16,22 @@ class ArgumentParser(BaseArgumentParser):
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
if local_arg_flags & CICFlag.REDIS_CALLBACK:
self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
if local_arg_flags & CICFlag.CELERY:
self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
if local_arg_flags & CICFlag.SYNCER:
self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
if local_arg_flags & CICFlag.CHAIN:
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

4
apps/cic-eth/cic_eth/cli/base.py
View File

@@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum):
# sync - nibble 4
SYNCER = 4096
argflag_local_task = CICFlag.CELERY
argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
argflag_local_task = CICFlag.CELERY
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
argflag_local_chain = CICFlag.CHAIN
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

65
apps/cic-eth/cic_eth/cli/config.py
View File

@@ -1,12 +1,18 @@
# standard imports
import os
import logging
import urllib.parse
import copy
# external imports
from chainlib.eth.cli import (
Config as BaseConfig,
Flag,
)
from urlybird.merge import (
urlhostmerge,
urlmerge,
)
# local imports
from .base import CICFlag
@@ -40,6 +46,7 @@ class Config(BaseConfig):
if local_arg_flags & CICFlag.CHAIN:
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
if local_arg_flags & CICFlag.CELERY:
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
@@ -49,15 +56,61 @@ class Config(BaseConfig):
config.dict_override(local_args_override, 'local cli args')
if local_arg_flags & CICFlag.REDIS_CALLBACK:
config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
local_celery_args_override = {}
if local_arg_flags & CICFlag.CELERY:
hostport = urlhostmerge(
None,
config.get('REDIS_HOST'),
config.get('REDIS_PORT'),
)
redis_url = (
'redis',
hostport,
getattr(args, 'redis_db', None),
)
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
hostport = urlhostmerge(
celery_config_url[1],
getattr(args, 'celery_host', None),
getattr(args, 'celery_port', None),
)
celery_arg_url = (
getattr(args, 'celery_scheme', None),
hostport,
getattr(args, 'celery_db', None),
)
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
celery_url_string = urllib.parse.urlunsplit(celery_url)
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
if not getattr(args, 'celery_no_result'):
local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
if local_celery_args_override['CELERY_RESULT_URL'] == None:
local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
hostport = urlhostmerge(
celery_config_url[1],
getattr(args, 'celery_result_host', None),
getattr(args, 'celery_result_port', None),
)
celery_arg_url = (
getattr(args, 'celery_result_scheme', None),
hostport,
getattr(args, 'celery_result_db', None),
)
celery_url = urlmerge(celery_config_url, celery_arg_url)
logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
celery_url_string = urllib.parse.urlunsplit(celery_url)
local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
config.dict_override(local_celery_args_override, 'local celery cli args')
if local_arg_flags & CICFlag.REDIS_CALLBACK:
redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
logg.debug('config loaded:\n{}'.format(config))
return config

2
apps/cic-eth/cic_eth/data/config/celery.ini
View File

@@ -1,5 +1,5 @@
[celery]
broker_url = redis://localhost:6379
broker_url =
result_url =
queue = cic-eth
debug = 0

2
apps/cic-eth/cic_eth/data/config/cic.ini
View File

@@ -2,5 +2,5 @@
registry_address =
trust_address =
default_token_symbol =
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
run_dir = /run

6
apps/cic-eth/cic_eth/data/config/eth.ini
View File

@@ -1,2 +1,6 @@
[eth]
gas_gifter_minimum_balance = 10000000000000000000000
gas_holder_minimum_units = 180000
gas_holder_refill_units = 15
gas_holder_refill_threshold = 3
gas_gifter_refill_buffer = 3
min_fee_price = 1

2
apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
View File

@@ -23,7 +23,7 @@ def upgrade():
op.create_table(
'lock',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column("address", sa.String(42), nullable=True),
sa.Column("address", sa.String, nullable=True),
sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),

31
apps/cic-eth/cic_eth/db/migrations/default/versions/c91cafc3e0c1_add_gas_cache.py Normal file
View File

@@ -0,0 +1,31 @@
"""Add gas cache
Revision ID: c91cafc3e0c1
Revises: aee12aeb47ec
Create Date: 2021-10-28 20:45:34.239865
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'c91cafc3e0c1'
down_revision = 'aee12aeb47ec'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'gas_cache',
sa.Column('id', sa.Integer, primary_key=True),
sa.Column("address", sa.String, nullable=False),
sa.Column("tx_hash", sa.String, nullable=True),
sa.Column("method", sa.String, nullable=True),
sa.Column("value", sa.BIGINT(), nullable=False),
)
def downgrade():
op.drop_table('gas_cache')

27
apps/cic-eth/cic_eth/db/models/gas_cache.py Normal file
View File

@@ -0,0 +1,27 @@
# standard imports
import logging
# external imports
from sqlalchemy import Column, String, NUMERIC
# local imports
from .base import SessionBase
logg = logging.getLogger(__name__)
class GasCache(SessionBase):
"""Provides gas budget cache for token operations
"""
__tablename__ = 'gas_cache'
address = Column(String())
tx_hash = Column(String())
method = Column(String())
value = Column(NUMERIC())
def __init__(self, address, method, value, tx_hash):
self.address = address
self.tx_hash = tx_hash
self.method = method
self.value = value

4
apps/cic-eth/cic_eth/db/models/nonce.py
View File

@@ -12,7 +12,7 @@ from cic_eth.error import (
IntegrityError,
)
logg = logging.getLogger()
logg = logging.getLogger(__name__)
class Nonce(SessionBase):
@@ -21,7 +21,7 @@ class Nonce(SessionBase):
__tablename__ = 'nonce'
nonce = Column(Integer)
address_hex = Column(String(42))
address_hex = Column(String(40))
@staticmethod

18
apps/cic-eth/cic_eth/db/models/role.py
View File

@@ -24,8 +24,22 @@ class AccountRole(SessionBase):
tag = Column(Text)
address_hex = Column(String(42))
# TODO:
@staticmethod
def all(session=None):
session = SessionBase.bind_session(session)
pairs = []
q = session.query(AccountRole.tag, AccountRole.address_hex)
for r in q.all():
pairs.append((r[1], r[0]),)
SessionBase.release_session(session)
return pairs
@staticmethod
def get_address(tag, session):
"""Get Ethereum address matching the given tag

6
apps/cic-eth/cic_eth/enum.py
View File

@@ -69,9 +69,12 @@ class StatusEnum(enum.IntEnum):
class LockEnum(enum.IntEnum):
"""
STICKY: When set, reset is not possible
INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
START: When set, startup is not possible, regardless of state
CREATE: Disable creation of accounts
SEND: Disable sending to network
QUEUE: Disable queueing new or modified transactions
QUERY: Disable all queue state and transaction queries
"""
STICKY=1
INIT=2
@@ -79,7 +82,8 @@ class LockEnum(enum.IntEnum):
SEND=8
QUEUE=16
QUERY=32
ALL=int(0xfffffffffffffffe)
START=int(0x80000000)
ALL=int(0x7ffffffe)
def status_str(v, bits_only=False):

4
apps/cic-eth/cic_eth/error.py
View File

@@ -64,8 +64,10 @@ class LockedError(Exception):
class SeppukuError(Exception):
"""Exception base class for all errors that should cause system shutdown
"""
def __init__(self, message, lockdown=False):
self.message = message
self.lockdown = lockdown
class SignerError(SeppukuError):

39
apps/cic-eth/cic_eth/eth/account.py
View File

@@ -136,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
# Generate and sign transaction
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@@ -192,7 +192,7 @@ def gift(self, account_address, chain_spec_dict):
# Generate and sign transaction
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
(tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
rpc_signer.disconnect()
@@ -266,19 +266,46 @@ def set_role(self, tag, address, chain_spec_dict):
@celery_app.task(bind=True, base=BaseTask)
def role(self, address, chain_spec_dict):
"""Return account role for address
"""Return account role for address and/or role
:param account: Account to check
:type account: str, 0x-hex
:param chain_str: Chain spec string representation
:type chain_str: str
:param chain_spec_dict: Chain spec dict representation
:type chain_spec_dict: dict
:returns: Account, or None if not exists
:rtype: Varies
"""
session = self.create_session()
role_tag = AccountRole.role_for(address, session=session)
session.close()
return role_tag
return [(address, role_tag,)]
@celery_app.task(bind=True, base=BaseTask)
def role_account(self, role_tag, chain_spec_dict):
"""Return address for role.
If the role parameter is None, will return addresses for all roles.
:param role_tag: Role to match
:type role_tag: str
:param chain_spec_dict: Chain spec dict representation
:type chain_spec_dict: dict
:returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
:rtype: list
"""
session = self.create_session()
pairs = None
if role_tag != None:
addr = AccountRole.get_address(role_tag, session=session)
pairs = [(addr, role_tag,)]
else:
pairs = AccountRole.all(session=session)
session.close()
return pairs
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)

27
apps/cic-eth/cic_eth/eth/erc20.py
View File

@@ -10,6 +10,9 @@ from chainlib.eth.tx import (
TxFormat,
unpack,
)
from chainlib.eth.contract import (
ABIContractEncoder,
)
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from hexathon import (
@@ -31,10 +34,8 @@ from cic_eth.error import (
YouAreBrokeError,
)
from cic_eth.queue.tx import register_tx
from cic_eth.eth.gas import (
create_check_gas_task,
MaxGasOracle,
)
from cic_eth.eth.gas import create_check_gas_task
from cic_eth.eth.util import CacheGasOracle
from cic_eth.ext.address import translate_address
from cic_eth.task import (
CriticalSQLAlchemyTask,
@@ -154,8 +155,12 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
enc = ABIContractEncoder()
enc.method('transferFrom')
method = enc.get()
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
try:
(tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -225,8 +230,12 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
session = self.create_session()
enc = ABIContractEncoder()
enc.method('transfer')
method = enc.get()
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
try:
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
@@ -294,8 +303,12 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
session = self.create_session()
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
enc = ABIContractEncoder()
enc.method('approve')
method = enc.get()
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
try:
(tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)

55
apps/cic-eth/cic_eth/eth/gas.py
View File

@@ -41,6 +41,7 @@ from chainqueue.db.models.tx import TxCache
from chainqueue.db.models.otx import Otx
# local imports
from cic_eth.db.models.gas_cache import GasCache
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase
from cic_eth.error import (
@@ -65,17 +66,56 @@ from cic_eth.encode import (
ZERO_ADDRESS_NORMAL,
unpack_normal,
)
from cic_eth.error import SeppukuError
from cic_eth.eth.util import MAXIMUM_FEE_UNITS
celery_app = celery.current_app
logg = logging.getLogger()
MAXIMUM_FEE_UNITS = 8000000
class MaxGasOracle:
@celery_app.task(base=CriticalSQLAlchemyTask)
def apply_gas_value_cache(address, method, value, tx_hash):
return apply_gas_value_cache_local(address, method, value, tx_hash)
def gas(code=None):
return MAXIMUM_FEE_UNITS
def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
address = tx_normalize.executable_address(address)
tx_hash = tx_normalize.tx_hash(tx_hash)
value = int(value)
session = SessionBase.bind_session(session)
q = session.query(GasCache)
q = q.filter(GasCache.address==address)
q = q.filter(GasCache.method==method)
o = q.first()
if o == None:
o = GasCache(address, method, value, tx_hash)
elif tx.gas_used > o.value:
o.value = value
o.tx_hash = strip_0x(tx_hash)
session.add(o)
session.commit()
SessionBase.release_session(session)
def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
if rpc == None:
rpc = RPCConnection.connect(chain_spec, 'default')
o = balance(add_0x(address))
r = rpc.do(o)
try:
r = int(r)
except ValueError:
r = strip_0x(r)
r = int(r, 16)
logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
if r < min_gas:
return False
return True
def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
@@ -357,6 +397,13 @@ def refill_gas(self, recipient_address, chain_spec_dict):
# set up evm RPC connection
rpc = RPCConnection.connect(chain_spec, 'default')
# check the gas balance of the gifter
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
# set up transaction builder
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
gas_oracle = self.create_gas_oracle(rpc)

54
apps/cic-eth/cic_eth/eth/util.py Normal file
View File

@@ -0,0 +1,54 @@
# standard imports
import logging
# external imports
from chainlib.eth.gas import RPCGasOracle
from hexathon import strip_0x
# local imports
from cic_eth.db.models.gas_cache import GasCache
from cic_eth.encode import tx_normalize
from cic_eth.db.models.base import SessionBase
MAXIMUM_FEE_UNITS = 8000000
logg = logging.getLogger(__name__)
class MaxGasOracle(RPCGasOracle):
def get_fee_units(self, code=None):
return MAXIMUM_FEE_UNITS
class CacheGasOracle(MaxGasOracle):
"""Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
:todo: instead of max units, connect a pluggable gas heuristics engine.
"""
def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
self.value = None
self.address = address
self.method = method
address = tx_normalize.executable_address(address)
session = SessionBase.bind_session(session)
q = session.query(GasCache)
q = q.filter(GasCache.address==address)
if method != None:
method = strip_0x(method)
q = q.filter(GasCache.method==method)
o = q.first()
if o != None:
self.value = int(o.value)
SessionBase.release_session(session)
def get_fee_units(self, code=None):
if self.value != None:
logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
return self.value
return super(CacheGasOracle, self).get_fee_units(code=code)

7
apps/cic-eth/cic_eth/pytest/fixtures_config.py
View File

@@ -8,15 +8,14 @@ import confini
script_dir = os.path.dirname(os.path.realpath(__file__))
root_dir = os.path.dirname(os.path.dirname(script_dir))
config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
logg = logging.getLogger(__name__)
@pytest.fixture(scope='session')
def load_config():
config_dir = os.environ.get('CONFINI_DIR')
if config_dir == None:
config_dir = os.path.join(root_dir, 'config/test')
conf = confini.Config(config_dir, 'CICTEST')
override_config_dir = os.path.join(root_dir, 'config', 'test')
conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
conf.process()
logg.debug('config {}'.format(conf))
return conf

1
apps/cic-eth/cic_eth/runnable/daemons/filters/__init__.py
View File

@@ -3,3 +3,4 @@ from .tx import TxFilter
from .gas import GasFilter
from .register import RegistrationFilter
from .transferauth import TransferAuthFilter
from .token import TokenFilter

63
apps/cic-eth/cic_eth/runnable/daemons/filters/token.py Normal file
View File

@@ -0,0 +1,63 @@
# standard imports
import logging
# external imports
from eth_erc20 import ERC20
from chainlib.eth.contract import (
ABIContractEncoder,
ABIContractType,
)
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.address import is_same_address
from chainlib.eth.error import RequestMismatchException
from cic_eth_registry import CICRegistry
from cic_eth_registry.erc20 import ERC20Token
from eth_token_index import TokenUniqueSymbolIndex
import celery
# local imports
from .base import SyncFilter
logg = logging.getLogger(__name__)
class TokenFilter(SyncFilter):
def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
self.queue = queue
self.chain_spec = chain_spec
self.caller_address = call_address
def filter(self, conn, block, tx, db_session=None):
if not tx.payload:
return (None, None)
try:
r = ERC20.parse_transfer_request(tx.payload)
except RequestMismatchException:
return (None, None)
token_address = tx.inputs[0]
token = ERC20Token(self.chain_spec, conn, token_address)
registry = CICRegistry(self.chain_spec, conn)
r = registry.by_name(token.symbol, sender_address=self.caller_address)
if is_same_address(r, ZERO_ADDRESS):
return None
enc = ABIContractEncoder()
enc.method('transfer')
method = enc.get()
s = celery.signature(
'cic_eth.eth.gas.apply_gas_value_cache',
[
token_address,
method,
tx.gas_used,
tx.hash,
],
queue=self.queue,
)
return s.apply_async()

18
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@@ -67,7 +67,10 @@ from cic_eth.registry import (
connect_declarator,
connect_token_registry,
)
from cic_eth.task import BaseTask
from cic_eth.task import (
BaseTask,
CriticalWeb3Task,
)
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@@ -76,18 +79,18 @@ arg_flags = cic_eth.cli.argflag_std_read
local_arg_flags = cic_eth.cli.argflag_local_task
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
#argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei')
argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
args = argparser.parse_args()
# process config
extra_args = {
# 'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
'aux_all': None,
'aux': None,
'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
'min_fee_price': 'ETH_MIN_FEE_PRICE',
}
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
@@ -215,6 +218,7 @@ def main():
argv.append('-n')
argv.append(config.get('CELERY_QUEUE'))
# TODO: More elegant way of setting queue-wide settings
BaseTask.default_token_symbol = default_token_symbol
BaseTask.default_token_address = default_token_address
default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
@@ -222,6 +226,14 @@ def main():
BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name
BaseTask.trusted_addresses = trusted_addresses
CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
if config.get('ETH_MIN_FEE_PRICE'):
BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

10
apps/cic-eth/cic_eth/runnable/daemons/tracker.py
View File

@@ -36,6 +36,7 @@ from cic_eth.runnable.daemons.filters import (
TxFilter,
RegistrationFilter,
TransferAuthFilter,
TokenFilter,
)
from cic_eth.stat import init_chain_stat
from cic_eth.registry import (
@@ -99,10 +100,10 @@ def main():
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
if len(syncer_backends) == 0:
initial_block_start = config.get('SYNCER_OFFSET')
initial_block_offset = block_offset
initial_block_start = int(config.get('SYNCER_OFFSET'))
initial_block_offset = int(block_offset)
if config.true('SYNCER_NO_HISTORY'):
initial_block_start = block_offset
initial_block_start = initial_block_offset
initial_block_offset += 1
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
@@ -154,6 +155,8 @@ def main():
gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE'))
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
i = 0
@@ -163,6 +166,7 @@ def main():
syncer.add_filter(registration_filter)
# TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
syncer.add_filter(tx_filter)
syncer.add_filter(token_gas_cache_filter)
#syncer.add_filter(transfer_auth_filter)
for cf in callback_filters:
syncer.add_filter(cf)

25
apps/cic-eth/cic_eth/runnable/tag.py
View File

@@ -8,6 +8,7 @@ import re
# external imports
import cic_eth.cli
from chainlib.chain import ChainSpec
from chainlib.eth.address import is_address
from xdg.BaseDirectory import xdg_config_home
# local imports
@@ -21,12 +22,18 @@ logg = logging.getLogger()
arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_positional('tag', type=str, help='address tag')
argparser.add_positional('address', type=str, help='address')
argparser.add_argument('--set', action='store_true', help='sets the given tag')
argparser.add_argument('--tag', type=str, help='operate on the given tag')
argparser.add_positional('address', required=False, type=str, help='address associated with tag')
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
extra_args = {
'set': None,
'tag': None,
'address': None,
}
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
@@ -39,7 +46,17 @@ api = AdminApi(None)
def main():
admin_api.tag_account(args.tag, args.address, chain_spec)
if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')):
sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS')))
sys.exit(1)
if config.get('_SET'):
admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS'))
else:
t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS'))
r = t.get()
for v in r:
sys.stdout.write('{}\t{}\n'.format(v[1], v[0]))
if __name__ == '__main__':

2
apps/cic-eth/cic_eth/runnable/transfer.py
View File

@@ -18,7 +18,7 @@ from cic_eth.api import Api
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger('create_account_script')
arg_flags = cic_eth.cli.argflag_std_base
arg_flags = cic_eth.cli.argflag_local_base
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')

97
apps/cic-eth/cic_eth/runnable/view.py
View File

@@ -16,9 +16,14 @@ import confini
import celery
from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
from hexathon import add_0x
from hexathon import (
add_0x,
strip_0x,
uniform as hex_uniform,
)
# local imports
import cic_eth.cli
from cic_eth.api.admin import AdminApi
from cic_eth.db.enum import (
StatusEnum,
@@ -31,59 +36,35 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
default_format = 'terminal'
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
argparser = argparse.ArgumentParser()
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
arg_flags = cic_eth.cli.argflag_std_base
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
argparser = cic_eth.cli.ArgumentParser(arg_flags)
argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-v', action='store_true', help='Be verbose')
argparser.add_argument('-vv', help='be more verbose', action='store_true')
argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"')
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
if args.v == True:
logging.getLogger().setLevel(logging.INFO)
elif args.vv == True:
logging.getLogger().setLevel(logging.DEBUG)
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config.process()
args_override = {
'ETH_PROVIDER': getattr(args, 'p'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
extra_args = {
'f': '_FORMAT',
'query': '_QUERY',
}
# override args
config.dict_override(args_override, 'cli args')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
try:
config.add(add_0x(args.query), '_QUERY', True)
except:
config.add(args.query, '_QUERY', True)
celery_app = cic_eth.cli.CeleryApp.from_config(config)
queue = config.get('CELERY_QUEUE')
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
queue = args.q
# connect to celery
celery_app = cic_eth.cli.CeleryApp.from_config(config)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
# set up rpc
rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True)
conn = rpc.get_default()
rpc = EthHTTPConnection(args.p)
#registry_address = config.get('CIC_REGISTRY_ADDRESS')
admin_api = AdminApi(rpc)
admin_api = AdminApi(conn)
t = admin_api.registry()
registry_address = t.get()
@@ -113,7 +94,7 @@ def render_tx(o, **kwargs):
for v in o.get('status_log', []):
d = datetime.datetime.fromisoformat(v[0])
e = status_str(v[1], args.status_raw)
e = status_str(v[1], config.get('_RAW'))
content += '{}: {}\n'.format(d, e)
return content
@@ -154,20 +135,24 @@ def render_lock(o, **kwargs):
def main():
txs = []
renderer = render_tx
if len(config.get('_QUERY')) > 66:
#registry = connect_registry(rpc, chain_spec, registry_address)
#admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
elif len(config.get('_QUERY')) > 42:
#registry = connect_registry(rpc, chain_spec, registry_address)
#admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
elif len(config.get('_QUERY')) == 42:
#registry = connect_registry(rpc, chain_spec, registry_address)
txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
query = config.get('_QUERY')
try:
query = hex_uniform(strip_0x(query))
except TypeError:
pass
except ValueError:
pass
if len(query) > 64:
admin_api.tx(chain_spec, tx_raw=query, renderer=renderer)
elif len(query) > 40:
admin_api.tx(chain_spec, tx_hash=query, renderer=renderer)
elif len(query) == 40:
txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account)
renderer = render_account
elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
elif len(query) >= 4 and query[:4] == 'lock':
t = admin_api.get_lock()
txs = t.get()
renderer = render_lock
@@ -175,7 +160,7 @@ def main():
r = renderer(txs)
sys.stdout.write(r + '\n')
else:
raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
raise ValueError('cannot parse argument {}'.format(query))
if __name__ == '__main__':

37
apps/cic-eth/cic_eth/task.py
View File

@@ -17,6 +17,7 @@ from cic_eth_registry.error import UnknownContractError
# local imports
from cic_eth.error import SeppukuError
from cic_eth.db.models.base import SessionBase
from cic_eth.eth.util import CacheGasOracle
#logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger()
@@ -29,14 +30,32 @@ class BaseTask(celery.Task):
session_func = SessionBase.create_session
call_address = ZERO_ADDRESS
trusted_addresses = []
create_nonce_oracle = RPCNonceOracle
create_gas_oracle = RPCGasOracle
min_fee_price = 1
default_token_address = None
default_token_symbol = None
default_token_name = None
default_token_decimals = None
run_dir = '/run'
def create_gas_oracle(self, conn, address=None, *args, **kwargs):
if address == None:
return RPCGasOracle(
conn,
code_callback=kwargs.get('code_callback'),
min_price=self.min_fee_price,
id_generator=kwargs.get('id_generator'),
)
return CacheGasOracle(
conn,
address,
method=kwargs.get('method'),
min_price=self.min_fee_price,
id_generator=kwargs.get('id_generator'),
)
def create_session(self):
return BaseTask.session_func()
@@ -78,19 +97,18 @@ class CriticalWeb3Task(CriticalTask):
autoretry_for = (
ConnectionError,
)
safe_gas_threshold_amount = 2000000000 * 60000 * 3
safe_gas_threshold_amount = 60000 * 3
safe_gas_refill_amount = safe_gas_threshold_amount * 5
safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100
class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task):
autoretry_for = (
sqlalchemy.exc.DatabaseError,
sqlalchemy.exc.TimeoutError,
ConnectionError,
sqlalchemy.exc.ResourceClosedError,
)
safe_gas_threshold_amount = 2000000000 * 60000 * 3
safe_gas_refill_amount = safe_gas_threshold_amount * 5
class CriticalSQLAlchemyAndSignerTask(CriticalTask):
@@ -100,14 +118,11 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
sqlalchemy.exc.ResourceClosedError,
)
class CriticalWeb3AndSignerTask(CriticalTask):
class CriticalWeb3AndSignerTask(CriticalWeb3Task):
autoretry_for = (
ConnectionError,
)
safe_gas_threshold_amount = 2000000000 * 60000 * 3
safe_gas_refill_amount = safe_gas_threshold_amount * 5
@celery_app.task()
def check_health(self):
pass