Merge remote-tracking branch 'origin/master' into lash/default-token

This commit is contained in:
nolash 2021-04-23 23:05:32 +02:00
commit 489796abd8
22 changed files with 165 additions and 36 deletions

View File

@ -32,7 +32,9 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
:returns: New lock state for address :returns: New lock state for address
:rtype: number :rtype: number
""" """
chain_str = str(ChainSpec.from_dict(chain_spec_dict)) chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash) r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r)) logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
return chained_input return chained_input
@ -51,7 +53,9 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
:returns: New lock state for address :returns: New lock state for address
:rtype: number :rtype: number
""" """
chain_str = str(ChainSpec.from_dict(chain_spec_dict)) chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
r = Lock.reset(chain_str, flags, address=address) r = Lock.reset(chain_str, flags, address=address)
logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r)) logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
return chained_input return chained_input
@ -127,7 +131,9 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
@celery_app.task(base=CriticalSQLAlchemyTask) @celery_app.task(base=CriticalSQLAlchemyTask)
def check_lock(chained_input, chain_spec_dict, lock_flags, address=None): def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
chain_str = str(ChainSpec.from_dict(chain_spec_dict)) chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
session = SessionBase.create_session() session = SessionBase.create_session()
r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session) r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
if address != None: if address != None:

View File

View File

@ -1,5 +1,6 @@
from cic_eth.db.models.base import SessionBase from cic_eth.db.models.base import SessionBase
def health(*args, **kwargs): def health(*args, **kwargs):
session = SessionBase.create_session() session = SessionBase.create_session()
session.execute('SELECT count(*) from alembic_version') session.execute('SELECT count(*) from alembic_version')

View File

@ -0,0 +1,48 @@
# standard imports
import logging
# external imports
from chainlib.connection import RPCConnection
from chainlib.chain import ChainSpec
from chainlib.eth.gas import balance
# local imports
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import LockEnum
from cic_eth.error import LockedError
from cic_eth.admin.ctrl import check_lock
logg = logging.getLogger().getChild(__name__)
def health(*args, **kwargs):
session = SessionBase.create_session()
config = kwargs['config']
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
try:
check_lock(None, None, LockEnum.INIT)
except LockedError:
logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
return True
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.close()
rpc = RPCConnection.connect(chain_spec, 'default')
o = balance(gas_provider)
r = rpc.do(o)
try:
r = int(r, 16)
except TypeError:
r = int(r)
gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
if r < gas_min:
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
return False
return True

View File

@ -0,0 +1,37 @@
# standard imports
import time
import logging
from urllib.error import URLError
# external imports
from chainlib.connection import RPCConnection
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.sign import sign_message
from chainlib.error import JSONRPCException
logg = logging.getLogger().getChild(__name__)
def health(*args, **kwargs):
blocked = True
max_attempts = 5
conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
for i in range(max_attempts):
idx = i + 1
logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
try:
conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
except FileNotFoundError:
pass
except ConnectionError:
pass
except URLError:
pass
except JSONRPCException:
logg.debug('signer connection succeeded')
return True
if idx < max_attempts:
time.sleep(0.5)
return False

View File

@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum):
QUEUE: Disable queueing new or modified transactions QUEUE: Disable queueing new or modified transactions
""" """
STICKY=1 STICKY=1
CREATE=2 INIT=2
SEND=4 CREATE=4
QUEUE=8 SEND=8
QUERY=16 QUEUE=16
QUERY=32
ALL=int(0xfffffffffffffffe) ALL=int(0xfffffffffffffffe)

View File

@ -5,8 +5,11 @@ Revises: 1f1b3b641d08
Create Date: 2021-04-02 18:41:20.864265 Create Date: 2021-04-02 18:41:20.864265
""" """
import datetime
from alembic import op from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from chainlib.eth.constant import ZERO_ADDRESS
from cic_eth.db.enum import LockEnum
# revision identifiers, used by Alembic. # revision identifiers, used by Alembic.
@ -23,10 +26,11 @@ def upgrade():
sa.Column("address", sa.String(42), nullable=True), sa.Column("address", sa.String(42), nullable=True),
sa.Column('blockchain', sa.String), sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0), sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False), sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True), sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
) )
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True) op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
def downgrade(): def downgrade():

View File

@ -59,6 +59,7 @@ args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'), 'CIC_CHAIN_SPEC': getattr(args, 'i'),
} }
# override args # override args
config.dict_override(args_override, 'cli')
config.censor('PASSWORD', 'DATABASE') config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL') config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config)) logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@ -67,7 +68,9 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
queue = args.q queue = args.q
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_spec = None
if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
admin_api = AdminApi(None) admin_api = AdminApi(None)
@ -82,6 +85,9 @@ def lock_names_to_flag(s):
# TODO: move each command to submodule # TODO: move each command to submodule
def main(): def main():
chain_spec_dict = None
if chain_spec != None:
chain_spec_dict = chain_spec.asdict()
if args.command == 'unlock': if args.command == 'unlock':
flags = lock_names_to_flag(args.flags) flags = lock_names_to_flag(args.flags)
if not is_checksum_address(args.address): if not is_checksum_address(args.address):
@ -91,7 +97,7 @@ def main():
'cic_eth.admin.ctrl.unlock', 'cic_eth.admin.ctrl.unlock',
[ [
None, None,
chain_spec.asdict(), chain_spec_dict,
args.address, args.address,
flags, flags,
], ],
@ -110,7 +116,7 @@ def main():
'cic_eth.admin.ctrl.lock', 'cic_eth.admin.ctrl.lock',
[ [
None, None,
chain_spec.asdict(), chain_spec_dict,
args.address, args.address,
flags, flags,
], ],

View File

@ -15,8 +15,10 @@ from chainlib.connection import RPCConnection
from chainlib.eth.connection import EthUnixSignerConnection from chainlib.eth.connection import EthUnixSignerConnection
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainqueue.db.models.otx import Otx from chainqueue.db.models.otx import Otx
from cic_eth_registry.error import UnknownContractError
import liveness.linux import liveness.linux
# local imports # local imports
from cic_eth.eth import ( from cic_eth.eth import (
erc20, erc20,
@ -142,11 +144,15 @@ else:
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC')) chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default') RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection) #RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS') Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
liveness.linux.load(health_modules) #import cic_eth.checks.gas
#if not cic_eth.checks.gas.health(config=config):
# raise RuntimeError()
liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config)
def main(): def main():
argv = ['worker'] argv = ['worker']
@ -170,7 +176,12 @@ def main():
rpc = RPCConnection.connect(chain_spec, 'default') rpc = RPCConnection.connect(chain_spec, 'default')
registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS')) try:
registry = connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
except UnknownContractError as e:
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
sys.exit(1)
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS') trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None: if trusted_addresses_src == None:
@ -179,6 +190,7 @@ def main():
trusted_addresses = trusted_addresses_src.split(',') trusted_addresses = trusted_addresses_src.split(',')
for address in trusted_addresses: for address in trusted_addresses:
logg.info('using trusted address {}'.format(address)) logg.info('using trusted address {}'.format(address))
connect_declarator(rpc, chain_spec, trusted_addresses) connect_declarator(rpc, chain_spec, trusted_addresses)
connect_token_registry(rpc, chain_spec) connect_token_registry(rpc, chain_spec)

View File

@ -96,3 +96,8 @@ class CriticalWeb3AndSignerTask(CriticalTask):
def hello(self): def hello(self):
time.sleep(0.1) time.sleep(0.1)
return id(SessionBase.create_session) return id(SessionBase.create_session)
@celery_app.task()
def check_health(self):
celery.app.control.shutdown()

View File

@ -3,5 +3,6 @@ registry_address =
chain_spec = evm:bloxberg:8996 chain_spec = evm:bloxberg:8996
tx_retry_delay = tx_retry_delay =
trust_address = trust_address =
health_modules = cic_eth.k8s.db
default_token_symbol = GFT default_token_symbol = GFT
health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

View File

@ -4,3 +4,5 @@ chain_spec = evm:bloxberg:8996
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
tx_retry_delay = 20 tx_retry_delay = 20
default_token_symbol = GFT default_token_symbol = GFT
health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

View File

@ -1,8 +1,4 @@
[eth] [eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:63545 provider = http://localhost:63545
gas_provider_address = health_modules = cic_eth.check.db,cic_eth.check.gas
#chain_id = gas_gifter_minimum_balance = 10000000000000000000000
abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
account_accounts_index_writer =

View File

@ -1,5 +1,5 @@
[signer] [signer]
socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
secret = deedbeef secret = deedbeef
database_name = signer_test database_name = signer_test
dev_keys_path = dev_keys_path =

View File

@ -1,8 +1,4 @@
[eth] [eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:8545 provider = http://localhost:8545
gas_provider_address = gas_gifter_minimum_balance = 10000000000000000000000
#chain_id = health_modules = cic_eth.check.db,cic_eth.check.gas
abi_dir = /usr/local/share/cic/solidity/abi
account_accounts_index_writer =

View File

@ -1,6 +1,6 @@
cic-base==0.1.2a79+build.35e442bc cic-base==0.1.2b1
celery==4.4.7 celery==4.4.7
crypto-dev-signer~=0.4.14b2 crypto-dev-signer~=0.4.14b3
confini~=0.3.6rc3 confini~=0.3.6rc3
cic-eth-registry~=0.5.4a16 cic-eth-registry~=0.5.4a16
#cic-bancor~=0.0.6 #cic-bancor~=0.0.6

View File

@ -38,7 +38,7 @@ packages =
cic_eth.runnable.daemons.filters cic_eth.runnable.daemons.filters
cic_eth.callbacks cic_eth.callbacks
cic_eth.sync cic_eth.sync
cic_eth.k8s cic_eth.check
scripts = scripts =
./scripts/migrate.py ./scripts/migrate.py

View File

@ -57,8 +57,8 @@ WORKDIR /home/grassroots
USER grassroots USER grassroots
ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433 ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
ARG cic_base_version=0.1.2a77 ARG cic_base_version=0.1.2a79
ARG cic_eth_version=0.11.0b6 ARG cic_eth_version=0.11.0b8+build.c2286e5c
ARG sarafu_faucet_version=0.0.2a28 ARG sarafu_faucet_version=0.0.2a28
ARG sarafu_token_version==0.0.1a6 ARG sarafu_token_version==0.0.1a6
ARG cic_contracts_version=0.0.2a2 ARG cic_contracts_version=0.0.2a2

View File

@ -13,6 +13,12 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi
DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"} DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic} CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic}
ETH_PASSPHRASE='' ETH_PASSPHRASE=''
DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
if [ $DEV_TOKEN_TYPE = 'giftable' ]; then
token_symbol='GFT'
else
token_symbol='SRF'
fi
# Debug flag # Debug flag
DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@ -46,6 +52,7 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H
echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
>&2 echo "create account for sarafu gifter" >&2 echo "create account for sarafu gifter"
DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register` DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
@ -97,5 +104,10 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
#echo -n 0 > $init_level_file #echo -n 0 > $init_level_file
# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
cic-eth-ctl -i :: unlock INIT
cic-eth-ctl -i :: unlock SEND
cic-eth-ctl -i :: unlock QUEUE
set +a set +a
set +e set +e

View File

@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs
checks.append(module) checks.append(module)
for check in checks: for check in checks:
r = check.health(args, kwargs) r = check.health(*args, **kwargs)
if r == False: if r == False:
raise RuntimeError('liveness check {} failed'.format(str(check))) raise RuntimeError('liveness check {} failed'.format(str(check)))
logg.info('liveness check passed: {}'.format(str(check))) logg.info('liveness check passed: {}'.format(str(check)))

View File

@ -1,7 +1,7 @@
from setuptools import setup from setuptools import setup
setup( setup(
name='liveness', name='liveness',
version='0.0.1a6', version='0.0.1a7',
packages=['liveness'], packages=['liveness'],
include_package_data=True, include_package_data=True,
) )

View File

@ -116,7 +116,9 @@ class TestImports(unittest.TestCase):
def test_args(self): def test_args(self):
checks = ['tests.imports.import_args'] checks = ['tests.imports.import_args']
liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42}) aargs=['foo']
kwaargs={'bar': 42}
liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs)
f = open(self.pid_path, 'r') f = open(self.pid_path, 'r')
r = f.read() r = f.read()
f.close() f.close()