Output raw txs in all import modules

apps/cic-ussd/cic_ussd/metadata/phone.py

@@ -31,7 +31,7 @@ class PhonePointerMetadata(Metadata):
         self.identifier = identifier
         self.metadata_pointer = generate_metadata_pointer(
                 identifier=self.identifier,
-                cic_type='cic.phone'
+                cic_type=':cic.phone'
         )
         if self.base_url:
             self.url = os.path.join(self.base_url, self.metadata_pointer)

apps/cic-ussd/cic_ussd/metadata/user.py

@@ -33,7 +33,7 @@ class UserMetadata(Metadata):
         self.identifier = identifier
         self.metadata_pointer = generate_metadata_pointer(
                 identifier=self.identifier,
-                cic_type='cic.person'
+                cic_type=':cic.person'
         )
         if self.base_url:
             self.url = os.path.join(self.base_url, self.metadata_pointer)

apps/cic-ussd/cic_ussd/runnable/server.py

@@ -86,7 +86,7 @@ UssdMenu.ussd_menu_db = ussd_menu_db
 
 # set up db
 data_source_name = dsn_from_config(config)
-SessionBase.connect(data_source_name=data_source_name)
+SessionBase.connect(data_source_name, pool_size=int(config.get('DATABASE_POOL_SIZE')), debug=config.true('DATABASE_DEBUG'))
 # create session for the life time of http request
 SessionBase.session = SessionBase.create_session()
 

apps/contract-migration/scripts/README.md

@@ -118,6 +118,8 @@ If you did not change the docker-compose setup, your `eth_provider` the you need
 
 The keystore file is relative to the directory you found this README in. Of course you can use a different wallet, but then you will have to provide it with tokens yourself (hint: `../reset.sh`)
 
+All external balance transactions are saved in raw wire format in `<datadir>/txs`, with transaction hash as file name.
+
 Only run _one_ of the following.
 
 
@@ -153,7 +155,11 @@ The `redis_hostname_in_docker` value is the hostname required to reach the redis
 
 #### Alternative 3 - USSD import - `cic_ussd`
 
-Run in sequence, in first terminal:
+If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`.
+
+`celery -A cic_ussd.import_task purge -Q cic-import-ussd --broker <redis_url>`
+
+Then, in sequence, run in first terminal:
 
 `python cic_eth/import_balance.py -v -c config -p <eth_provider> -r <cic_registry_address> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c out`
 
@@ -165,8 +171,6 @@ The balance script is a celery task worker, and will not exit by itself in its c
 
 The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
 
-All external balance transactions are saved to `<datadir>/txs`
-
 
 ### Step 4 - Metadata import (optional)
 

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -162,6 +162,15 @@ class Handler:
             (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
             logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
             r = conn.do(o)
+
+            tx_path = os.path.join(
+                    user_dir,
+                    'txs',
+                    strip_0x(tx_hash_hex),
+                    )
+            f = open(tx_path, 'w')
+            f.write(strip_0x(o['params'][0]))
+            f.close()
 #        except TypeError as e:
 #            logg.warning('typerror {}'.format(e))
 #            pass

apps/contract-migration/scripts/cic_eth/import_users.py

@@ -82,6 +82,9 @@ os.makedirs(os.path.join(phone_dir, 'meta'))
 user_old_dir = os.path.join(args.user_dir, 'old')
 os.stat(user_old_dir)
 
+txs_dir = os.path.join(args.user_dir, 'txs')
+os.makedirs(txs_dir)
+
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)
 

apps/contract-migration/scripts/cic_ussd/import_balance.py

@@ -48,7 +48,7 @@ argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to
 argparser.add_argument('--offset', type=int, default=0, help='block offset to start syncer from')
 argparser.add_argument('-v', help='be verbose', action='store_true')
 argparser.add_argument('-vv', help='be more verbose', action='store_true')
-argparser.add_argument('user_dir', type=str, help='user export directory')
+argparser.add_argument('user_dir', default='out', type=str, help='user export directory')
 args = argparser.parse_args(sys.argv[1:])
 
 if args.v == True:

apps/contract-migration/scripts/cic_ussd/import_task.py

@@ -20,7 +20,8 @@ from chainlib.eth.tx import (
 from cic_types.processor import generate_metadata_pointer
 from cic_types.models.person import Person
 
-logg = logging.getLogger().getChild(__name__)
+#logg = logging.getLogger().getChild(__name__)
+logg = logging.getLogger()
 
 celery_app = celery.current_app
 
@@ -58,7 +59,7 @@ class MetadataTask(ImportTask):
 
 
 def old_address_from_phone(base_path, phone):
-    pidx = generate_metadata_pointer(phone.encode('utf-8'), 'cic.phone')
+    pidx = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
     phone_idx_path = os.path.join('{}/phone/{}/{}/{}'.format(
             base_path,
             pidx[:2],
@@ -75,7 +76,7 @@ def old_address_from_phone(base_path, phone):
 
 @celery_app.task(bind=True, base=MetadataTask)
 def resolve_phone(self, phone):
-    identifier = generate_metadata_pointer(phone.encode('utf-8'), 'cic.phone')
+    identifier = generate_metadata_pointer(phone.encode('utf-8'), ':cic.phone')
     url = urllib.parse.urljoin(self.meta_url(), identifier)
     logg.debug('attempt getting phone pointer at {} for phone {}'.format(url, phone))
     r = urllib.request.urlopen(url)
@@ -90,6 +91,7 @@ def resolve_phone(self, phone):
 def generate_metadata(self, address, phone):
     old_address = old_address_from_phone(self.import_dir, phone)
 
+    logg.debug('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> foo')
     logg.debug('address {}'.format(address))
     old_address_upper = strip_0x(old_address).upper()
     metadata_path = '{}/old/{}/{}/{}.json'.format(
@@ -125,7 +127,7 @@ def generate_metadata(self, address, phone):
     f.write(json.dumps(o))
     f.close()
 
-    meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), 'cic.person')
+    meta_key = generate_metadata_pointer(bytes.fromhex(new_address_clean), ':cic.person')
     meta_filepath = os.path.join(
             self.import_dir,
             'meta',
@@ -162,7 +164,7 @@ def opening_balance_tx(self, address, phone, serial):
             )
             
     f = open(tx_path, 'w')
-    f.write(o)
+    f.write(strip_0x(o))
     f.close()
 
     tx_nonce_path = os.path.join(
@@ -183,6 +185,7 @@ def send_txs(self, nonce):
         return
 
 
+    logg.debug('attempt to open symlink for nonce {}'.format(nonce))
     tx_nonce_path = os.path.join(
             self.import_dir,
             'txs',
@@ -194,7 +197,7 @@ def send_txs(self, nonce):
 
     os.unlink(tx_nonce_path)
 
-    o = raw(tx_signed_raw_hex)
+    o = raw(add_0x(tx_signed_raw_hex))
     tx_hash_hex = self.balance_processor.conn.do(o)
 
     logg.info('sent nonce {} tx hash {}'.format(nonce, tx_hash_hex)) #tx_signed_raw_hex))

apps/contract-migration/scripts/eth/import_balance.py

@@ -162,6 +162,15 @@ class Handler:
             (tx_hash_hex, o) = self.tx_factory.transfer(self.token_address, signer_address, recipient, balance_full)
             logg.info('submitting erc20 transfer tx {} for recipient {}'.format(tx_hash_hex, recipient))
             r = conn.do(o)
+
+            tx_path = os.path.join(
+                    user_dir,
+                    'txs',
+                    strip_0x(tx_hash_hex),
+                    )
+            f = open(tx_path, 'w')
+            f.write(strip_0x(o['params'][0]))
+            f.close()
 #        except TypeError as e:
 #            logg.warning('typerror {}'.format(e))
 #            pass

apps/contract-migration/scripts/eth/import_users.py

@@ -74,6 +74,9 @@ os.makedirs(os.path.join(phone_dir, 'meta'))
 user_old_dir = os.path.join(args.user_dir, 'old')
 os.stat(user_old_dir)
 
+txs_dir = os.path.join(args.user_dir, 'txs')
+os.makedirs(txs_dir)
+
 chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 chain_str = str(chain_spec)