kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rehabilitate imports

Browse Source
This commit is contained in:
nolash 2021-06-29 22:56:29 +02:00
parent 740a809506
commit e0394ebeba
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
13 changed files with 108 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
apps/cic-cache/cic_cache/runnable/daemons/tracker.py
View File

@ -23,15 +23,20 @@ from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.connection import RPCConnection
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
Tx,
)
from chainlib.interface import ChainInterface
from hexathon import (
strip_0x,
)
from chainsyncer.backend.sql import SQLBackend
from chainsyncer.driver import (
HeadSyncer,
HistorySyncer,
)
from chainsyncer.driver.head import HeadSyncer
from chainsyncer.driver.history import HistorySyncer
from chainsyncer.db.models.base import SessionBase
# local imports
@ -83,6 +88,16 @@ def register_filter_tags(filters, session):
session.rollback()
logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
class EthChainInterface(ChainInterface):
def __init__(self):
self._tx_receipt = receipt
self._block_by_number = block_by_number
self._block_from_src = Block.from_src
self._src_normalize = Tx.src_normalize
chain_interface = EthChainInterface()
def main():
# Connect to blockchain with chainlib
@ -113,10 +128,10 @@ def main():
logg.info('resuming sync session {}'.format(syncer_backend))
for syncer_backend in syncer_backends:
syncers.append(HistorySyncer(syncer_backend))
syncers.append(HistorySyncer(syncer_backend, chain_interface))
syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
syncers.append(HeadSyncer(syncer_backend))
syncers.append(HeadSyncer(syncer_backend, chain_interface))
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
if trusted_addresses_src == None:

4
apps/cic-cache/requirements.txt
View File

@ -1,4 +1,4 @@
cic-base~=0.1.3a1
cic-base~=0.1.3a2
alembic==1.4.2
confini~=0.3.6rc3
uwsgi==2.0.19.1
@ -9,5 +9,5 @@ semver==2.13.0
psycopg2==2.8.6
celery==4.4.7
redis==3.5.3
chainsyncer[sql]~=0.0.3a1
chainsyncer[sql]~=0.0.3a3
erc20-faucet~=0.2.2a1

9
apps/cic-cache/scripts/migrate.py
View File

@ -2,6 +2,7 @@
import os
import argparse
import logging
import re
import alembic
from alembic.config import Config as AlembicConfig
@ -23,6 +24,8 @@ argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
argparser.add_argument('-f', action='store_true', help='force action')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
args = argparser.parse_args()
@ -53,4 +56,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
ac.set_main_option('sqlalchemy.url', dsn)
ac.set_main_option('script_location', migrations_dir)
if args.reset:
if not args.f:
if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
logg.error('user chickened out on requested reset, bailing')
sys.exit(1)
alembic.command.downgrade(ac, 'base')
alembic.command.upgrade(ac, 'head')

22
apps/cic-eth/cic_eth/runnable/daemons/tracker.py
View File

@ -21,7 +21,14 @@ from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.connection import RPCConnection
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
Tx,
)
from chainlib.interface import ChainInterface
from hexathon import (
strip_0x,
)
@ -78,6 +85,17 @@ chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
class EthChainInterface(ChainInterface):
def __init__(self):
self._tx_receipt = receipt
self._block_by_number = block_by_number
self._block_from_src = Block.from_src
self._src_normalize = Tx.src_normalize
chain_interface = EthChainInterface()
def main():
# connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
@ -119,11 +137,11 @@ def main():
for syncer_backend in syncer_backends:
try:
syncers.append(HistorySyncer(syncer_backend))
syncers.append(HistorySyncer(syncer_backend, chain_interface))
logg.info('Initializing HISTORY syncer on backend {}'.format(syncer_backend))
except AttributeError:
logg.info('Initializing HEAD syncer on backend {}'.format(syncer_backend))
syncers.append(HeadSyncer(syncer_backend))
syncers.append(HeadSyncer(syncer_backend, chain_interface))
connect_registry(rpc, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))

4
apps/cic-eth/requirements.txt
View File

@ -1,4 +1,4 @@
cic-base~=0.1.3a1
cic-base~=0.1.3a2
celery==4.4.7
crypto-dev-signer~=0.4.14b6
confini~=0.3.6rc3
@ -16,7 +16,7 @@ moolb~=0.1.1b2
eth-address-index~=0.1.2a1
chainlib-eth~=0.0.5a1
hexathon~=0.0.1a7
chainsyncer[sql]~=0.0.3a1
chainsyncer[sql]~=0.0.3a3
chainqueue~=0.0.2b5
sarafu-faucet~=0.0.4a1
erc20-faucet~=0.2.2a1

10
apps/cic-eth/scripts/migrate.py
View File

@ -2,6 +2,8 @@
import os
import argparse
import logging
import re
import sys
import alembic
from alembic.config import Config as AlembicConfig
@ -23,6 +25,8 @@ argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
argparser.add_argument('-f', action='store_true', help='force action')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
args = argparser.parse_args()
@ -53,4 +57,10 @@ ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
ac.set_main_option('sqlalchemy.url', dsn)
ac.set_main_option('script_location', migrations_dir)
if args.reset:
if not args.f:
if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
logg.error('user chickened out on requested reset, bailing')
sys.exit(1)
alembic.command.downgrade(ac, 'base')
alembic.command.upgrade(ac, 'head')

24
apps/data-seeding/cic_eth/import_balance.py
View File

@ -18,13 +18,18 @@ from hexathon import (
add_0x,
)
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver import HeadSyncer
from chainsyncer.driver.head import HeadSyncer
from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
Tx,
)
from chainlib.interface import ChainInterface
from chainlib.hash import keccak256_string_to_hex
from chainlib.eth.address import to_checksum_address
from chainlib.eth.gas import OverrideGasOracle
@ -70,13 +75,14 @@ elif args.vv == True:
config_dir = os.path.join(args.c)
os.makedirs(config_dir, 0o777, True)
config = confini.Config(config_dir, args.env_prefix)
config.process()
# override args
config.process()
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
'KEYSTORE_FILE_PATH': getattr(args, 'y'),
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
@ -109,6 +115,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py
token_symbol = args.token_symbol
class EthChainInterface(ChainInterface):
def __init__(self):
self._tx_receipt = receipt
self._block_by_number = block_by_number
self._block_from_src = Block.from_src
self._src_normalize = Tx.src_normalize
chain_interface = EthChainInterface()
class Handler:
@ -308,7 +324,7 @@ def main():
f.close()
syncer_backend.set(block_offset, 0)
syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
syncer.add_filter(handler)
syncer.loop(1, conn)

3
apps/data-seeding/config/cic.ini
View File

@ -7,4 +7,5 @@ approval_escrow_address =
chain_spec = evm:bloxberg:8996
tx_retry_delay =
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
user_ussd_svc_service_port=
user_ussd_svc_service_port =

6
apps/data-seeding/config/eth.ini
View File

@ -1,8 +1,2 @@
[eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:63545
gas_provider_address =
#chain_id =
abi_dir = /usr/local/share/cic/solidity/abi
account_accounts_index_writer =

4
apps/data-seeding/create_import_users.py
View File

@ -204,9 +204,9 @@ def gen():
]))
if random.randint(0, 1):
# fake.local_latitude()
p.location['latitude'] = (random.random() + 180) - 90
p.location['latitude'] = (random.random() * 180) - 90
# fake.local_latitude()
p.location['longitude'] = (random.random() + 360) - 180
p.location['longitude'] = (random.random() * 360) - 179
return (old_blockchain_checksum_address, phone, p)

21
apps/data-seeding/eth/import_balance.py
View File

@ -18,13 +18,18 @@ from hexathon import (
add_0x,
)
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver import HeadSyncer
from chainsyncer.driver.head import HeadSyncer
from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.block import (
block_latest,
block_by_number,
Block,
)
from chainlib.eth.tx import (
receipt,
Tx,
)
from chainlib.interface import ChainInterface
from chainlib.hash import keccak256_string_to_hex
from chainlib.eth.address import to_checksum_address
from chainlib.eth.gas import OverrideGasOracle
@ -75,7 +80,7 @@ args_override = {
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'ETH_PROVIDER': getattr(args, 'p'),
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
'KEYSTORE_FILE_PATH': getattr(args, 'y')
}
config.dict_override(args_override, 'cli flag')
config.censor('PASSWORD', 'DATABASE')
@ -108,6 +113,16 @@ user_dir = args.user_dir # user_out_dir from import_users.py
token_symbol = args.token_symbol
class EthChainInterface(ChainInterface):
def __init__(self):
self._tx_receipt = receipt
self._block_by_number = block_by_number
self._block_from_src = Block.from_src
self._src_normalize = Tx.src_normalize
chain_interface = EthChainInterface()
class Handler:
@ -303,7 +318,7 @@ def main():
f.close()
syncer_backend.set(block_offset, 0)
syncer = HeadSyncer(syncer_backend, block_callback=progress_callback)
syncer = HeadSyncer(syncer_backend, chain_interface, block_callback=progress_callback)
handler = Handler(conn, chain_spec, user_dir, balances, sarafu_token_address, signer, gas_oracle, nonce_oracle)
syncer.add_filter(handler)
syncer.loop(1, conn)

2
apps/data-seeding/eth/import_users.py
View File

@ -59,7 +59,7 @@ config.process()
args_override = {
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
'CIC_CHAIN_SPEC': getattr(args, 'i'),
'KEYSTORE_FILE_PATH': getattr(args, 'key-file')
'KEYSTORE_FILE_PATH': getattr(args, 'y')
}
config.dict_override(args_override, 'cli')
config.add(args.user_dir, '_USERDIR', True)

2
apps/data-seeding/requirements.txt
View File

@ -1,4 +1,4 @@
cic-base[full_graph]==0.1.3a1
cic-base[full_graph]==0.1.3a2
sarafu-faucet==0.0.4a1
cic-eth==0.11.1a1
cic-types==0.1.0a13