grassrootseconomics/cic-stack
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

cic-eth: Add sanity checks for emergency shutdown / liveness tests

Browse Source
This commit is contained in:
Louis Holbrook 2021-04-23 21:02:51 +00:00
parent 6b6ec8659b
commit 3129a78e06
23 changed files with 165 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/cic-eth/cic_eth/admin/ctrl.py
View File

@ -32,6 +32,8 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.AL
:returns: New lock state for address
:rtype: number
"""
chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
r = Lock.set(chain_str, flags, address=address, tx_hash=tx_hash)
logg.debug('Locked {} for {}, flag now {}'.format(flags, address, r))
@ -51,6 +53,8 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.
:returns: New lock state for address
:rtype: number
"""
chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
r = Lock.reset(chain_str, flags, address=address)
logg.debug('Unlocked {} for {}, flag now {}'.format(flags, address, r))
@ -127,6 +131,8 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
@celery_app.task(base=CriticalSQLAlchemyTask)
def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
chain_str = '::'
if chain_spec_dict != None:
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
session = SessionBase.create_session()
r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)

0
apps/cic-eth/cic_eth/check/__init__.py Normal file
View File

1
apps/cic-eth/cic_eth/k8s/db.py → apps/cic-eth/cic_eth/check/db.py
View File

@ -1,5 +1,6 @@
from cic_eth.db.models.base import SessionBase
def health(*args, **kwargs):
session = SessionBase.create_session()
session.execute('SELECT count(*) from alembic_version')

48
apps/cic-eth/cic_eth/check/gas.py Normal file
View File

@ -0,0 +1,48 @@
# standard imports
import logging
# external imports
from chainlib.connection import RPCConnection
from chainlib.chain import ChainSpec
from chainlib.eth.gas import balance
# local imports
from cic_eth.db.models.role import AccountRole
from cic_eth.db.models.base import SessionBase
from cic_eth.db.enum import LockEnum
from cic_eth.error import LockedError
from cic_eth.admin.ctrl import check_lock
logg = logging.getLogger().getChild(__name__)
def health(*args, **kwargs):
session = SessionBase.create_session()
config = kwargs['config']
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
logg.debug('check gas balance of gas gifter for chain {}'.format(chain_spec))
try:
check_lock(None, None, LockEnum.INIT)
except LockedError:
logg.warning('INIT lock is set, skipping GAS GIFTER balance check.')
return True
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
session.close()
rpc = RPCConnection.connect(chain_spec, 'default')
o = balance(gas_provider)
r = rpc.do(o)
try:
r = int(r, 16)
except TypeError:
r = int(r)
gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
if r < gas_min:
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
return False
return True

37
apps/cic-eth/cic_eth/check/signer.py Normal file
View File

@ -0,0 +1,37 @@
# standard imports
import time
import logging
from urllib.error import URLError
# external imports
from chainlib.connection import RPCConnection
from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.sign import sign_message
from chainlib.error import JSONRPCException
logg = logging.getLogger().getChild(__name__)
def health(*args, **kwargs):
blocked = True
max_attempts = 5
conn = RPCConnection.connect(kwargs['config'].get('CIC_CHAIN_SPEC'), tag='signer')
for i in range(max_attempts):
idx = i + 1
logg.debug('attempt signer connection check {}/{}'.format(idx, max_attempts))
try:
conn.do(sign_message(ZERO_ADDRESS, '0x2a'))
except FileNotFoundError:
pass
except ConnectionError:
pass
except URLError:
pass
except JSONRPCException:
logg.debug('signer connection succeeded')
return True
if idx < max_attempts:
time.sleep(0.5)
return False

9
apps/cic-eth/cic_eth/db/enum.py
View File

@ -74,10 +74,11 @@ class LockEnum(enum.IntEnum):
QUEUE: Disable queueing new or modified transactions
"""
STICKY=1
CREATE=2
SEND=4
QUEUE=8
QUERY=16
INIT=2
CREATE=4
SEND=8
QUEUE=16
QUERY=32
ALL=int(0xfffffffffffffffe)

6
apps/cic-eth/cic_eth/db/migrations/default/versions/75d4767b3031_lock.py
View File

@ -5,8 +5,11 @@ Revises: 1f1b3b641d08
Create Date: 2021-04-02 18:41:20.864265
"""
import datetime
from alembic import op
import sqlalchemy as sa
from chainlib.eth.constant import ZERO_ADDRESS
from cic_eth.db.enum import LockEnum
# revision identifiers, used by Alembic.
@ -23,10 +26,11 @@ def upgrade():
sa.Column("address", sa.String(42), nullable=True),
sa.Column('blockchain', sa.String),
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
sa.Column("date_created", sa.DateTime, nullable=False),
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
)
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
def downgrade():

10
apps/cic-eth/cic_eth/runnable/ctrl.py
View File

@ -59,6 +59,7 @@ args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
}
# override args
config.dict_override(args_override, 'cli')
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
@ -67,6 +68,8 @@ celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=confi
queue = args.q
chain_spec = None
if config.get('CIC_CHAIN_SPEC') != None and config.get('CIC_CHAIN_SPEC') != '::':
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
admin_api = AdminApi(None)
@ -82,6 +85,9 @@ def lock_names_to_flag(s):
# TODO: move each command to submodule
def main():
chain_spec_dict = None
if chain_spec != None:
chain_spec_dict = chain_spec.asdict()
if args.command == 'unlock':
flags = lock_names_to_flag(args.flags)
if not is_checksum_address(args.address):
@ -91,7 +97,7 @@ def main():
'cic_eth.admin.ctrl.unlock',
[
None,
chain_spec.asdict(),
chain_spec_dict,
args.address,
flags,
],
@ -110,7 +116,7 @@ def main():
'cic_eth.admin.ctrl.lock',
[
None,
chain_spec.asdict(),
chain_spec_dict,
args.address,
flags,
],

15
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@ -15,8 +15,10 @@ from chainlib.connection import RPCConnection
from chainlib.eth.connection import EthUnixSignerConnection
from chainlib.chain import ChainSpec
from chainqueue.db.models.otx import Otx
from cic_eth_registry.error import UnknownContractError
import liveness.linux
# local imports
from cic_eth.eth import (
erc20,
@ -138,11 +140,15 @@ else:
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
#RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer', constructor=EthUnixSignerConnection)
RPCConnection.register_location(config.get('SIGNER_SOCKET_PATH'), chain_spec, 'signer')
Otx.tracing = config.true('TASKS_TRACE_QUEUE_STATUS')
liveness.linux.load(health_modules)
#import cic_eth.checks.gas
#if not cic_eth.checks.gas.health(config=config):
# raise RuntimeError()
liveness.linux.load(health_modules, rundir=config.get('CIC_RUN_DIR'), config=config)
def main():
argv = ['worker']
@ -166,7 +172,11 @@ def main():
rpc = RPCConnection.connect(chain_spec, 'default')
try:
connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
except UnknownContractError as e:
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
sys.exit(1)
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None:
@ -175,6 +185,7 @@ def main():
trusted_addresses = trusted_addresses_src.split(',')
for address in trusted_addresses:
logg.info('using trusted address {}'.format(address))
connect_declarator(rpc, chain_spec, trusted_addresses)
connect_token_registry(rpc, chain_spec)

5
apps/cic-eth/cic_eth/task.py
View File

@ -94,3 +94,8 @@ class CriticalWeb3AndSignerTask(CriticalTask):
def hello(self):
time.sleep(0.1)
return id(SessionBase.create_session)
@celery_app.task()
def check_health(self):
celery.app.control.shutdown()

2
apps/cic-eth/cic_eth/version.py
View File

@ -10,7 +10,7 @@ version = (
0,
11,
0,
'beta.7',
'beta.8',
)
version_object = semver.VersionInfo(

3
apps/cic-eth/config/cic.ini
View File

@ -3,4 +3,5 @@ registry_address =
chain_spec = evm:bloxberg:8996
tx_retry_delay =
trust_address =
health_modules = cic_eth.k8s.db
health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

2
apps/cic-eth/config/docker/cic.ini
View File

@ -3,3 +3,5 @@ registry_address =
chain_spec = evm:bloxberg:8996
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
tx_retry_delay = 20
health_modules = cic_eth.check.db,cic_eth.check.signer,cic_eth.check.gas
run_dir = /run

8
apps/cic-eth/config/docker/eth.ini
View File

@ -1,8 +1,4 @@
[eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:63545
gas_provider_address =
#chain_id =
abi_dir = /home/lash/src/ext/cic/grassrootseconomics/cic-contracts/abis
account_accounts_index_writer =
health_modules = cic_eth.check.db,cic_eth.check.gas
gas_gifter_minimum_balance = 10000000000000000000000

2
apps/cic-eth/config/docker/signer.ini
View File

@ -1,5 +1,5 @@
[signer]
socket_path = /tmp/crypto-dev-signer/jsonrpc.ipc
socket_path = ipc:///tmp/crypto-dev-signer/jsonrpc.ipc
secret = deedbeef
database_name = signer_test
dev_keys_path =

8
apps/cic-eth/config/eth.ini
View File

@ -1,8 +1,4 @@
[eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:8545
gas_provider_address =
#chain_id =
abi_dir = /usr/local/share/cic/solidity/abi
account_accounts_index_writer =
gas_gifter_minimum_balance = 10000000000000000000000
health_modules = cic_eth.check.db,cic_eth.check.gas

4
apps/cic-eth/requirements.txt
View File

@ -1,6 +1,6 @@
cic-base==0.1.2a79+build.35e442bc
cic-base==0.1.2b1
celery==4.4.7
crypto-dev-signer~=0.4.14b2
crypto-dev-signer~=0.4.14b3
confini~=0.3.6rc3
cic-eth-registry~=0.5.4a16
#cic-bancor~=0.0.6

2
apps/cic-eth/setup.cfg
View File

@ -38,7 +38,7 @@ packages =
cic_eth.runnable.daemons.filters
cic_eth.callbacks
cic_eth.sync
cic_eth.k8s
cic_eth.check
scripts =
./scripts/migrate.py

4
apps/contract-migration/docker/Dockerfile
View File

@ -57,8 +57,8 @@ WORKDIR /home/grassroots
USER grassroots
ARG pip_extra_index_url=https://pip.grassrootseconomics.net:8433
ARG cic_base_version=0.1.2a77
ARG cic_eth_version=0.11.0b6
ARG cic_base_version=0.1.2a79
ARG cic_eth_version=0.11.0b8+build.c2286e5c
ARG sarafu_faucet_version=0.0.2a28
ARG sarafu_token_version==0.0.1a6
ARG cic_contracts_version=0.0.2a2

12
apps/contract-migration/seed_cic_eth.sh
View File

@ -13,6 +13,12 @@ DEV_PIP_EXTRA_INDEX_URL=${DEV_PIP_EXTRA_INDEX_URL:-https://pip.grassrootseconomi
DEV_DATABASE_NAME_CIC_ETH=${DEV_DATABASE_NAME_CIC_ETH:-"cic-eth"}
CIC_DATA_DIR=${CIC_DATA_DIR:-/tmp/cic}
ETH_PASSPHRASE=''
DEV_TOKEN_TYPE=${DEV_TOKEN_TYPE:-giftable}
if [ $DEV_TOKEN_TYPE = 'giftable' ]; then
token_symbol='GFT'
else
token_symbol='SRF'
fi
# Debug flag
DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER=0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
@ -46,6 +52,7 @@ DEV_ETH_ACCOUNT_GAS_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_H
echo DEV_ETH_ACCOUNT_GAS_GIFTER=$DEV_ETH_ACCOUNT_GAS_GIFTER >> $env_out_file
cic-eth-tag -i $CIC_CHAIN_SPEC GAS_GIFTER $DEV_ETH_ACCOUNT_GAS_GIFTER
>&2 echo "create account for sarafu gifter"
DEV_ETH_ACCOUNT_SARAFU_GIFTER=`cic-eth-create $debug --redis-host-callback=$REDIS_HOST --redis-port-callback=$REDIS_PORT --no-register`
echo DEV_ETH_ACCOUNT_SARAFU_GIFTER=$DEV_ETH_ACCOUNT_SARAFU_GIFTER >> $env_out_file
@ -97,5 +104,10 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
#echo -n 0 > $init_level_file
# Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
cic-eth-ctl -i :: unlock INIT
cic-eth-ctl -i :: unlock SEND
cic-eth-ctl -i :: unlock QUEUE
set +a
set +e

2
apps/util/liveness/liveness/linux.py
View File

@ -29,7 +29,7 @@ def load(check_strs, namespace=default_namespace, rundir='/run', *args, **kwargs
checks.append(module)
for check in checks:
r = check.health(args, kwargs)
r = check.health(*args, **kwargs)
if r == False:
raise RuntimeError('liveness check {} failed'.format(str(check)))
logg.info('liveness check passed: {}'.format(str(check)))

2
apps/util/liveness/setup.py
View File

@ -1,7 +1,7 @@
from setuptools import setup
setup(
name='liveness',
version='0.0.1a6',
version='0.0.1a7',
packages=['liveness'],
include_package_data=True,
)

4
apps/util/liveness/tests/test_imports.py
View File

@ -116,7 +116,9 @@ class TestImports(unittest.TestCase):
def test_args(self):
checks = ['tests.imports.import_args']
liveness.linux.load(checks, namespace=self.unit, rundir=self.run_dir, args=['foo'], kwargs={'bar': 42})
aargs=['foo']
kwaargs={'bar': 42}
liveness.linux.load(checks, self.unit, self.run_dir, *aargs, **kwaargs)
f = open(self.pid_path, 'r')
r = f.read()
f.close()