kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Output raw txs in all import modules

Browse Source
This commit is contained in:
nolash 2021-04-08 11:30:50 +02:00
parent 7e816e5f66
commit 4097f17030
Signed by untrusted user who does not match committer: lash
GPG Key ID: 21D2E7BB88C2A746
10 changed files with 44 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-ussd/cic_ussd/metadata/phone.py
View File

@ -31,7 +31,7 @@ class PhonePointerMetadata(Metadata):
self.identifier = identifier
self.metadata_pointer = generate_metadata_pointer(
identifier=self.identifier,
cic_type='cic.phone'
cic_type=':cic.phone'
)
if self.base_url:
self.url = os.path.join(self.base_url, self.metadata_pointer)

2
apps/cic-ussd/cic_ussd/metadata/user.py
View File

@ -33,7 +33,7 @@ class UserMetadata(Metadata):
self.identifier = identifier
self.metadata_pointer = generate_metadata_pointer(
identifier=self.identifier,
cic_type='cic.person'
cic_type=':cic.person'
)
if self.base_url:
self.url = os.path.join(self.base_url, self.metadata_pointer)

2
apps/cic-ussd/cic_ussd/runnable/server.py
View File

@ -86,7 +86,7 @@ UssdMenu.ussd_menu_db = ussd_menu_db
# set up db
data_source_name = dsn_from_config(config)
SessionBase.connect(data_source_name=data_source_name)
SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
# create session for the life time of http request
SessionBase.session = SessionBase.create_session()

10
apps/contract-migration/scripts/README.md
View File

@ -118,6 +118,8 @@ If you did not change the docker-compose setup, your `eth_provider` the you need
The keystore file is relative to the directory you found this README in. Of course you can use a different wallet, but then you will have to provide it with tokens yourself (hint: `../reset.sh`)
All external balance transactions are saved in raw wire format in `<datadir>/txs`, with transaction hash as file name.
Only run _one_ of the following.
@ -153,7 +155,11 @@ The `redis_hostname_in_docker` value is the hostname required to reach the redis
#### Alternative 3 - USSD import - `cic_ussd`
Run in sequence, in first terminal:
If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`.
`celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker <redis_url>`
Then, in sequence, run in first terminal:
`python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
@ -165,8 +171,6 @@ The balance script is a celery task worker, and will not exit by itself in its c
The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
All external balance transactions are saved to `<datadir>/txs`
### Step 4 - Metadata import (optional)

9
apps/contract-migration/scripts/cic_eth/import_balance.py
View File

@ -162,6 +162,15 @@ class Handler:
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
# except TypeError as e:
# logg.warning('typerror {}'.format(e))
# pass

3
apps/contract-migration/scripts/cic_eth/import_users.py
View File

@ -82,6 +82,9 @@ os.makedirs(os.path.join(phone_dir, 'meta'))
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec)

2
apps/contract-migration/scripts/cic_ussd/import_balance.py
View File

@ -48,7 +48,7 @@ argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to
argparser.add_argument('--offset', type=int, default=0, help='block offset to start syncer from')
argparser.add_argument('-v', help='be verbose', action='store_true')
argparser.add_argument('-vv', help='be more verbose', action='store_true')
argparser.add_argument('user_dir', type=str, help='user export directory')
argparser.add_argument('user_dir', default='out', type=str, help='user export directory')
args = argparser.parse_args(sys.argv[1:])
if args.v == True:

15
apps/contract-migration/scripts/cic_ussd/import_task.py
View File

@ -20,7 +20,8 @@ from chainlib.eth.tx import (
from cic_types.processor import generate_metadata_pointer
from cic_types.models.person import Person
logg = logging.getLogger().getChild(__name__)
#logg = logging.getLogger().getChild(__name__)
logg = logging.getLogger()
celery_app = celery.current_app
@ -58,7 +59,7 @@ class MetadataTask(ImportTask):
def old_address_from_phone(base_path, phone):
pidx = generate_metadata_pointer(phone.encode('utf-8'), 'cic.phone')
pidx = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
phone_idx_path = os.path.join('{}/phone/{}/{}/{}'.format(
base_path,
pidx[:2],
@ -75,7 +76,7 @@ def old_address_from_phone(base_path, phone):
@celery_app.task(bind=True, base=MetadataTask)
def resolve_phone(self, phone):
identifier = generate_metadata_pointer(phone.encode('utf-8'), 'cic.phone')
identifier = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
url = urllib.parse.urljoin(self.meta_url(), identifier)
logg.debug('attempt getting phone pointer at {} for phone {}'.format(url, phone))
r = urllib.request.urlopen(url)
@ -90,6 +91,7 @@ def resolve_phone(self, phone):
def generate_metadata(self, address, phone):
old_address = old_address_from_phone(self.import_dir, phone)
logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
logg.debug('address {}'.format(address))
old_address_upper = strip_0x(old_address).upper()
metadata_path = '{}/old/{}/{}/{}.json'.format(
@ -125,7 +127,7 @@ def generate_metadata(self, address, phone):
f.write(json.dumps(o))
f.close()
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
meta_filepath = os.path.join(
self.import_dir,
'meta',
@ -162,7 +164,7 @@ def opening_balance_tx(self, address, phone, serial):
)
f = open(tx_path, 'w')
f.write(o)
f.write(strip_0x(o))
f.close()
tx_nonce_path = os.path.join(
@ -183,6 +185,7 @@ def send_txs(self, nonce):
return
logg.debug('attempt to open symlink for nonce {}'.format(nonce))
tx_nonce_path = os.path.join(
self.import_dir,
'txs',
@ -194,7 +197,7 @@ def send_txs(self, nonce):
os.unlink(tx_nonce_path)
o = raw(tx_signed_raw_hex)
o = raw(add_0x(tx_signed_raw_hex))
tx_hash_hex = self.balance_processor.conn.do(o)
logg.info('sent nonce {} tx hash {}'.format(nonce, tx_hash_hex)) #tx_signed_raw_hex))

9
apps/contract-migration/scripts/eth/import_balance.py
View File

@ -162,6 +162,15 @@ class Handler:
(tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
r = conn.do(o)
tx_path = os.path.join(
user_dir,
'txs',
strip_0x(tx_hash_hex),
)
f = open(tx_path, 'w')
f.write(strip_0x(o['params'][0]))
f.close()
# except TypeError as e:
# logg.warning('typerror {}'.format(e))
# pass

3
apps/contract-migration/scripts/eth/import_users.py
View File

@ -74,6 +74,9 @@ os.makedirs(os.path.join(phone_dir, 'meta'))
user_old_dir = os.path.join(args.user_dir, 'old')
os.stat(user_old_dir)
txs_dir = os.path.join(args.user_dir, 'txs')
os.makedirs(txs_dir)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
chain_str = str(chain_spec)