From c68d9d8404c9ae8c4e5492edf9dcfc7bdbbf1f53 Mon Sep 17 00:00:00 2001 From: Louis Holbrook Date: Mon, 22 Feb 2021 20:00:18 +0000 Subject: [PATCH] Rehabilitate import scripts after leak fixes --- apps/cic-eth/cic_eth/eth/account.py | 4 +++- apps/cic-meta/docker/Dockerfile | 5 ++--- apps/cic-meta/package-lock.json | 2 +- apps/cic-meta/package.json | 2 +- .../scripts/{README => README.md} | 2 +- apps/contract-migration/scripts/import_balance.py | 2 +- apps/contract-migration/scripts/import_users.py | 5 +++-- .../scripts/local/{noop_traffic.py => noop.py} | 0 apps/contract-migration/scripts/requirements.txt | 2 +- apps/contract-migration/scripts/verify.py | 15 ++++++++------- 10 files changed, 21 insertions(+), 18 deletions(-) rename apps/contract-migration/scripts/{README => README.md} (96%) rename apps/contract-migration/scripts/local/{noop_traffic.py => noop.py} (100%) diff --git a/apps/cic-eth/cic_eth/eth/account.py b/apps/cic-eth/cic_eth/eth/account.py index a3b2322c..6729ca45 100644 --- a/apps/cic-eth/cic_eth/eth/account.py +++ b/apps/cic-eth/cic_eth/eth/account.py @@ -178,8 +178,10 @@ def register(self, account_address, chain_str, writer_address=None): """ chain_spec = ChainSpec.from_chain_str(chain_str) + session = SessionBase.create_session() if writer_address == None: - writer_address = AccountRole.get_address('ACCOUNTS_INDEX_WRITER') + writer_address = AccountRole.get_address('ACCOUNTS_INDEX_WRITER', session) + session.close() if writer_address == zero_address: raise RoleMissingError(account_address) diff --git a/apps/cic-meta/docker/Dockerfile b/apps/cic-meta/docker/Dockerfile index d3440175..f81e6720 100644 --- a/apps/cic-meta/docker/Dockerfile +++ b/apps/cic-meta/docker/Dockerfile @@ -3,11 +3,8 @@ FROM node:15.3.0-alpine3.10 WORKDIR /tmp/src/cic-meta COPY cic-meta/package.json \ - cic-meta/package-lock.json \ ./ -RUN npm install - COPY cic-meta/src/ src/ COPY cic-meta/tests/ tests/ COPY cic-meta/scripts/ scripts/ @@ -15,6 +12,8 @@ COPY cic-meta/scripts/ scripts/ RUN alias tsc=node_modules/typescript/bin/tsc +RUN npm install + COPY cic-meta/.config/ /usr/local/etc/cic-meta/ # COPY cic-meta/scripts/server/initdb/server.postgres.sql /usr/local/share/cic-meta/sql/server.sql diff --git a/apps/cic-meta/package-lock.json b/apps/cic-meta/package-lock.json index 414c1138..c0c67d0d 100644 --- a/apps/cic-meta/package-lock.json +++ b/apps/cic-meta/package-lock.json @@ -1,6 +1,6 @@ { "name": "cic-client-meta", - "version": "0.0.6", + "version": "0.0.7-alpha.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/apps/cic-meta/package.json b/apps/cic-meta/package.json index 28475bdb..c3979d8e 100644 --- a/apps/cic-meta/package.json +++ b/apps/cic-meta/package.json @@ -6,7 +6,7 @@ "types": "dist/index.d.ts", "scripts": { "test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts", - "build": "node_modules/typescript/bin/tsc -d --outDir dist && npm run build-server", + "build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts", "build-server": "tsc -d --outDir dist-server scripts/server/*.ts", "pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack", "clean": "rm -rf dist", diff --git a/apps/contract-migration/scripts/README b/apps/contract-migration/scripts/README.md similarity index 96% rename from apps/contract-migration/scripts/README rename to apps/contract-migration/scripts/README.md index 00dbac22..048a1f6f 100644 --- a/apps/contract-migration/scripts/README +++ b/apps/contract-migration/scripts/README.md @@ -49,7 +49,7 @@ This will monitor new mined blocks and send balances to the newly created accoun Without any modifications to the cluster and config files: -`python -c config --redis-host-callback redis ` +`python import_users.py -c config --redis-host-callback redis ` ** A note on the The callback**: The script uses a redis callback to retrieve the newly generated custodial address. This is the redis server _from the perspective of the cic-eth component_. diff --git a/apps/contract-migration/scripts/import_balance.py b/apps/contract-migration/scripts/import_balance.py index 427fc9f9..305d6da9 100644 --- a/apps/contract-migration/scripts/import_balance.py +++ b/apps/contract-migration/scripts/import_balance.py @@ -191,7 +191,7 @@ class BlockGetter: def progress_callback(block_number, tx_index, s): - sys.stdout.write(s.ljust(200) + "\n") + sys.stdout.write(str(s).ljust(200) + "\n") diff --git a/apps/contract-migration/scripts/import_users.py b/apps/contract-migration/scripts/import_users.py index 675d36c7..74d791c2 100644 --- a/apps/contract-migration/scripts/import_users.py +++ b/apps/contract-migration/scripts/import_users.py @@ -20,7 +20,7 @@ from hexathon import ( from chainlib.eth.address import to_checksum from cic_types.models.person import Person from cic_eth.api.api_task import Api -from cic_registry.chain import ChainSpec +from chainlib.chain import ChainSpec from cic_types.processor import generate_metadata_pointer logging.basicConfig(level=logging.WARNING) @@ -142,7 +142,8 @@ if __name__ == '__main__': new_address = register_eth(i, u) if u.identities.get('evm') == None: u.identities['evm'] = {} - u.identities['evm'][chain_str] = [new_address] + sub_chain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) + u.identities['evm'][sub_chain_str] = [new_address] register_ussd(u) diff --git a/apps/contract-migration/scripts/local/noop_traffic.py b/apps/contract-migration/scripts/local/noop.py similarity index 100% rename from apps/contract-migration/scripts/local/noop_traffic.py rename to apps/contract-migration/scripts/local/noop.py diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/contract-migration/scripts/requirements.txt index 3f049744..c9838fa5 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/contract-migration/scripts/requirements.txt @@ -2,7 +2,7 @@ psycopg2==2.8.6 chainlib~=0.0.1a15 chainsyncer~=0.0.1a10 cic-eth==0.10.0a30+build.fdb16130 -cic-registry~=0.5.3a19 +cic-registry~=0.5.3a21 confini~=0.3.6rc3 celery==4.4.7 redis==3.5.3 diff --git a/apps/contract-migration/scripts/verify.py b/apps/contract-migration/scripts/verify.py index 0b9ae20e..ff6a032e 100644 --- a/apps/contract-migration/scripts/verify.py +++ b/apps/contract-migration/scripts/verify.py @@ -11,7 +11,7 @@ import csv import json import urllib -# external impotts +# external imports import celery import eth_abi import confini @@ -19,9 +19,9 @@ from hexathon import ( strip_0x, add_0x, ) -from cic_registry.chain import ChainSpec from chainsyncer.backend import MemBackend from chainsyncer.driver import HeadSyncer +from chainlib.chain import ChainSpec from chainlib.eth.connection import HTTPConnection from chainlib.eth.constant import ZERO_ADDRESS from chainlib.eth.block import ( @@ -53,7 +53,7 @@ config_dir = '/usr/local/etc/cic-syncer' argparser = argparse.ArgumentParser(description='daemon that monitors transactions in new blocks') argparser.add_argument('-p', '--provider', dest='p', type=str, help='chain rpc provider address') argparser.add_argument('-c', type=str, default=config_dir, help='config root to use') -argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='oldchain:1', help='chain spec') +argparser.add_argument('--old-chain-spec', type=str, dest='old_chain_spec', default='evm:oldchain:1', help='chain spec') argparser.add_argument('-i', '--chain-spec', type=str, dest='i', help='chain spec') argparser.add_argument('--meta-provider', type=str, dest='meta_provider', default='http://localhost:63380', help='cic-meta url') argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address') @@ -254,7 +254,6 @@ def main(): balances = {} f = open('{}/balances.csv'.format(user_dir, 'r')) - remove_zeros = 10**12 i = 0 while True: l = f.readline() @@ -266,7 +265,7 @@ def main(): sys.stdout.write('loading balance {} {}'.format(i, address).ljust(200) + "\r") except ValueError: break - balance = int(int(r[1].rstrip()) / remove_zeros) + balance = int(r[1].rstrip()) balances[address] = balance i += 1 @@ -294,8 +293,10 @@ def main(): u = Person.deserialize(o) logg.debug('data {}'.format(u.identities['evm'])) - new_address = u.identities['evm'][chain_str][0] - old_address = u.identities['evm'][old_chain_str][0] + subchain_str = '{}:{}'.format(chain_spec.common_name(), chain_spec.network_id()) + new_address = u.identities['evm'][subchain_str][0] + subchain_str = '{}:{}'.format(old_chain_spec.common_name(), old_chain_spec.network_id()) + old_address = u.identities['evm'][subchain_str][0] balance = balances[old_address] logg.debug('checking {} -> {} = {}'.format(old_address, new_address, balance))