Add gas transfer script

CHANGELOG

@@ -0,0 +1,5 @@
+- 0.0.1
+	* Add eth tx decode
+	* Add eth balance query
+	* Add eth checksum address
+	* Add eth subscribe monitor

cic_tools/cic_eth/runnable/users.py

@@ -0,0 +1,252 @@
+#!/usr/bin/python
+
+# standard imports
+import json
+import time
+import datetime
+import random
+import logging
+import os
+import base64
+import hashlib
+import sys
+import uuid
+import argparse
+
+# third-party imports
+import redis
+import vobject
+import celery
+from faker import Faker
+import cic_registry
+import confini
+from cic_eth.api import Api
+
+logging.basicConfig(level=logging.DEBUG)
+logg = logging.getLogger()
+
+fake = Faker(['sl', 'en_US', 'no', 'de', 'ro'])
+
+#f = open('cic.conf', 'r')
+#config = json.load(f)
+#f.close()
+#
+
+default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
+argparser.add_argument('-q', type=str, default='cic-eth', help='Task queue')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
+argparser.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
+argparser.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
+argparser.add_argument('--timeout', default=1.0, type=int, help='timeout to wait for account create callback')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-vv', help='be more verbose', action='store_true')
+argparser.add_argument('count', help='Number of users to generate', type=int)
+args = argparser.parse_args()
+
+if args.v == True:
+    logging.getLogger().setLevel(logging.INFO)
+elif args.vv == True:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+config_dir = os.path.join(args.c)
+config = confini.Config(config_dir, os.environ.get('CONFINI_ENV_PREFIX'))
+config.process()
+args_override = {
+        'ETH_PROVIDER': getattr(args, 'p'),
+        'CIC_CHAIN_SPEC': getattr(args, 'i'),
+}
+config.dict_override(args_override, 'cli flag')
+logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
+
+
+dt_now = datetime.datetime.utcnow()
+dt_then = dt_now - datetime.timedelta(weeks=150)
+ts_now = int(dt_now.timestamp())
+ts_then = int(dt_then.timestamp())
+
+queue = args.q
+
+celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
+
+redis_host = config.get('REDIS_HOST')
+redis_port = config.get('REDIS_PORT')
+redis_db = config.get('REDIS_DB')
+redis_channel = str(uuid.uuid4())
+r = redis.Redis(redis_host, redis_port, redis_db)
+ps = r.pubsub()
+ps.subscribe(redis_channel)
+ps.get_message()
+
+api = Api(
+        config.get('CIC_CHAIN_SPEC'),
+        queue=args.q,
+        callback_param='{}:{}:{}:{}'.format(args.redis_host_callback, args.redis_port_callback, redis_db, redis_channel),
+        callback_task='cic_eth.callbacks.redis.redis',
+        callback_queue=queue,
+        )
+
+gift_max = 10000
+gift_factor = (10**9)
+
+user_count = args.count
+
+categories = [
+        "food/water",
+        "fuel/energy",
+        "education",
+        "health",
+        "shop",
+        "environment",
+        "transport",
+        "farming/labor",
+        "savingsgroup",
+        ]
+
+phone_idx = []
+
+
+def genPhoneIndex(phone):
+    h = hashlib.new('sha256')
+    h.update(phone.encode('utf-8'))
+    h.update(b'cic.msisdn')
+    return h.digest().hex()
+
+
+def genId(addr, typ):
+    h = hashlib.new('sha256')
+    h.update(bytes.fromhex(addr[2:]))
+    h.update(typ.encode('utf-8'))
+    return h.digest().hex()
+
+
+def genDate():
+
+    logg.info(ts_then)
+    ts = random.randint(ts_then, ts_now)
+    return datetime.datetime.fromtimestamp(ts).timestamp()
+
+
+def genPhone():
+    return fake.msisdn()
+
+
+def genPersonal(phone):
+    fn = fake.first_name()
+    ln = fake.last_name()
+    e = fake.email()
+
+    v = vobject.vCard()
+    first_name = fake.first_name()
+    last_name = fake.last_name()
+    v.add('n')
+    v.n.value = vobject.vcard.Name(family=last_name, given=first_name)
+    v.add('fn')
+    v.fn.value = '{} {}'.format(first_name, last_name)
+    v.add('tel')
+    v.tel.typ_param = 'CELL'
+    v.tel.value = phone
+    v.add('email')
+    v.email.value = fake.email()
+
+    vcard_serialized = v.serialize()
+    vcard_base64 = base64.b64encode(vcard_serialized.encode('utf-8'))
+
+    return vcard_base64.decode('utf-8')
+
+
+def genCats():
+    i = random.randint(0, 3)
+    return random.choices(categories, k=i)
+
+
+def genAmount():
+    return random.randint(0, gift_max) * gift_factor
+
+
+def gen():
+    old_blockchain_address = '0x' + os.urandom(20).hex()
+    t = api.create_account(register=True)
+
+    ps.get_message()
+    m = ps.get_message(timeout=args.timeout)
+    new_blockchain_address = json.loads(m['data'])
+    
+    #new_blockchain_address = t.get()
+    gender = random.choice(['female', 'male', 'other'])
+    phone = genPhone()
+    v = genPersonal(phone)
+    o = {
+        'date_registered': genDate(),
+        'vcard': v,
+        'gender': gender,
+        'key': {
+            'ethereum': [
+                old_blockchain_address,
+                new_blockchain_address,
+                ],
+            },
+        'location': {
+            'latitude': str(fake.latitude()),
+            'longitude': str(fake.longitude()),
+            'external': { # add osm lookup
+                }
+            },
+        'selling': genCats(),
+            }
+    uid = genId(new_blockchain_address, 'cic.person')
+
+    return (uid, phone, o)
+
+
+def prepareLocalFilePath(datadir, address):
+    parts = [
+                address[:2],
+                address[2:4],
+            ]
+    dirs = '{}/{}/{}'.format(
+            datadir,
+            parts[0],
+            parts[1],
+            )
+    os.makedirs(dirs, exist_ok=True)
+    return dirs
+
+
+if __name__ == '__main__':
+
+    os.makedirs('data/person', exist_ok=True)
+    os.makedirs('data/phone', exist_ok=True)
+
+    fa = open('./data/amounts', 'w')
+    fb = open('./data/addresses', 'w')
+
+    #for i in range(10):
+    for i in range(int(user_count)):
+    
+        (uid, phone, o) = gen()
+        eth = o['key']['ethereum'][1]
+
+        print(o)
+
+        d = prepareLocalFilePath('./data/person', uid)
+        f = open('{}/{}'.format(d, uid), 'w')
+        json.dump(o, f)
+        f.close()
+
+        pidx = genPhoneIndex(phone)
+        d = prepareLocalFilePath('./data/phone', uid)
+        f = open('{}/{}'.format(d, pidx), 'w')
+        f.write(eth)
+        f.close()
+
+        amount = genAmount()
+        fa.write('{},{}\n'.format(eth,amount))
+        fb.write('{}\n'.format(eth))
+        logg.debug('pidx {}, uid {}, eth {}, amount {}'.format(pidx, uid, eth, amount))
+
+    fb.close()
+    fa.close()

cic_tools/eth/constant.py

@@ -1,2 +1,3 @@
-zero_address = '0x{:040x}'.format(0)
-zero_content = '0x{:064x}'.format(0)
+ZERO_ADDRESS = '0x{:040x}'.format(0)
+ZERO_CONTENT = '0x{:064x}'.format(0)
+MINIMUM_FEE_UNITS = 21000

cic_tools/eth/gas.py

@@ -0,0 +1,56 @@
+# third-party imports
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+from crypto_dev_signer.eth.transaction import EIP155Transaction
+
+# local imports
+from cic_tools.eth.method import (
+        jsonrpc_template,
+        )
+from cic_tools.eth.tx import TxFactory
+from cic_tools.eth.hash import keccak256_hex_to_hex
+
+
+def price():
+    o = jsonrpc_template()
+    o['method'] = 'eth_gasPrice'
+    return o
+
+
+def balance(address):
+    o = jsonrpc_template()
+    o['method'] = 'eth_getBalance'
+    o['params'].append(address)
+    return o
+
+
+class GasTxFactory(TxFactory):
+
+    def create(self, sender, recipient, value):
+        tx = self.template(sender, recipient)
+        txe = EIP155Transaction(tx, tx['nonce'], tx['chainId'])
+        self.signer.signTransaction(txe)
+        tx_raw = txe.rlp_serialize()
+        tx_raw_hex = add_0x(tx_raw.hex())
+        tx_hash_hex = add_0x(keccak256_hex_to_hex(tx_raw_hex))
+
+        o = jsonrpc_template()
+        o['method'] = 'eth_sendRawTransaction'
+        o['params'].append(tx_raw_hex)
+
+        return (tx_hash_hex, o)
+
+
+class DefaultGasOracle:
+
+    def __init__(self, conn):
+        self.conn = conn
+
+
+    def get(self):
+        o = price()
+        r = self.conn.do(o)
+        n = strip_0x(r)
+        return int(n, 16)

cic_tools/eth/hash.py

@@ -1,7 +1,29 @@
+# third-party imports
 import sha3
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
 
 
 def keccak256_hex(s):
     h = sha3.keccak_256()
     h.update(s.encode('utf-8'))
     return h.digest().hex()
+
+
+def keccak256_string_to_hex(s):
+    return keccak256_hex(s)
+
+
+def keecak256_bytes_to_hex(b):
+    h = sha3.keccak_256()
+    h.update(b)
+    return h.digest().hex()
+
+
+def keccak256_hex_to_hex(hx):
+    h = sha3.keccak_256()
+    b = bytes.fromhex(strip_0x(hx))
+    h.update(b)
+    return h.digest().hex()

cic_tools/eth/method.py

@@ -4,13 +4,13 @@ import uuid
 from hexathon import add_0x
 from eth_abi import encode_single
 
-from .hash import keccak256_hex
-from .constant import zero_address
+from .hash import keccak256_string_to_hex
+from .constant import ZERO_ADDRESS
 
 
 # TODO: move to cic-contracts
-erc20_balance_signature = keccak256_hex('balanceOf(address)')[:8]
-erc20_decimals_signature = keccak256_hex('decimals()')[:8]
+erc20_balance_signature = keccak256_string_to_hex('balanceOf(address)')[:8]
+erc20_decimals_signature = keccak256_string_to_hex('decimals()')[:8]
 
 
 def jsonrpc_template():
@@ -22,7 +22,7 @@ def jsonrpc_template():
             }
 
 
-def erc20_balance(contract_address, address, sender_address=zero_address):
+def erc20_balance(contract_address, address, sender_address=ZERO_ADDRESS):
     o = jsonrpc_template()
     o['method'] = 'eth_call'
     data = erc20_balance_signature
@@ -34,7 +34,7 @@ def erc20_balance(contract_address, address, sender_address=zero_address):
     return o
 
 
-def erc20_decimals(contract_address, sender_address=zero_address):
+def erc20_decimals(contract_address, sender_address=ZERO_ADDRESS):
     o = jsonrpc_template()
     o['method'] = 'eth_call'
     arg = add_0x(erc20_decimals_signature)
@@ -45,7 +45,7 @@ def erc20_decimals(contract_address, sender_address=zero_address):
     return o
 
 
-def call(contract_address, data, sender_address=zero_address):
+def call(contract_address, data, sender_address=ZERO_ADDRESS):
     return {
         'from': sender_address,
         'to': contract_address,

cic_tools/eth/nonce.py

@@ -0,0 +1,32 @@
+# third-party imports
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+
+# local imports
+from cic_tools.eth.method import (
+        jsonrpc_template,
+        )
+
+
+def nonce(address):
+    o = jsonrpc_template()
+    o['method'] = 'eth_getTransactionCount'
+    o['params'].append(address)
+    o['params'].append('pending')
+    return o
+
+
+class DefaultNonceOracle:
+
+    def __init__(self, address, conn):
+        self.address = address
+        self.conn = conn
+
+
+    def next(self):
+        o = nonce(self.address)
+        r = self.conn.do(o)
+        n = strip_0x(r)
+        return int(n, 16)

cic_tools/eth/runnable/balance.py

@@ -57,9 +57,6 @@ if args.v:
 conn = HTTPConnection(args.p)
 
 def main():
-#    w3 = web3.Web3(web3.Web3.HTTPProvider(args.p))
-# REPLACE WITH URLLIB
-  
     account = to_checksum(args.account)
     if not args.u and account != add_0x(args.account):
         raise ValueError('invalid checksum address')

cic_tools/eth/runnable/gas.py

@@ -0,0 +1,112 @@
+#!python3
+
+"""Gas transfer script
+
+.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
+.. pgp:: 0826EDA1702D1E87C6E2875121D2E7BB88C2A746 
+
+"""
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# standard imports
+import sys
+import os
+import json
+import argparse
+import logging
+
+# third-party imports
+from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
+from crypto_dev_signer.keystore import DictKeystore
+from crypto_dev_signer.eth.helper import EthTxExecutor
+from hexathon import (
+        add_0x,
+        strip_0x,
+        )
+
+# local imports
+from cic_tools.eth.address import to_checksum
+from cic_tools.eth.connection import HTTPConnection
+from cic_tools.eth.method import (
+        jsonrpc_template,
+        )
+from cic_tools.eth.nonce import DefaultNonceOracle
+from cic_tools.eth.gas import (
+        DefaultGasOracle,
+        GasTxFactory,
+        )
+from cic_tools.eth.gas import balance as gas_balance
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+
+default_abi_dir = '/usr/share/local/cic/solidity/abi'
+default_eth_provider = os.environ.get('ETH_PROVIDER', 'http://localhost:8545')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
+argparser.add_argument('-w', action='store_true', help='Wait for the last transaction to be confirmed')
+argparser.add_argument('-ww', action='store_true', help='Wait for every transaction to be confirmed')
+argparser.add_argument('-i', '--chain-spec', dest='i', type=str, default='Ethereum:1', help='Chain specification string')
+argparser.add_argument('-a', '--signer-address', dest='a', type=str, help='Signing address')
+argparser.add_argument('-y', '--key-file', dest='y', type=str, help='Ethereum keystore file to use for signing')
+argparser.add_argument('-v', action='store_true', help='Be verbose')
+argparser.add_argument('-u', '--unsafe', dest='u', action='store_true', help='Auto-convert address to checksum adddress')
+argparser.add_argument('-vv', action='store_true', help='Be more verbose')
+argparser.add_argument('recipient', type=str, help='Ethereum address of recipient')
+argparser.add_argument('amount', type=int, help='Amount of tokens to mint and gift')
+args = argparser.parse_args()
+
+
+if args.vv:
+    logg.setLevel(logging.DEBUG)
+elif args.v:
+    logg.setLevel(logging.INFO)
+
+block_last = args.w
+block_all = args.ww
+
+signer_address = None
+keystore = DictKeystore()
+if args.y != None:
+    logg.debug('loading keystore file {}'.format(args.y))
+    signer_address = keystore.import_keystore_file(args.y)
+    logg.debug('now have key for signer address {}'.format(signer_address))
+signer = EIP155Signer(keystore)
+
+conn = HTTPConnection(args.p)
+nonce_oracle = DefaultNonceOracle(signer_address, conn)
+gas_oracle = DefaultGasOracle(conn)
+
+chain_pair = args.i.split(':')
+chain_id = int(chain_pair[1])
+
+
+def balance(address):
+    o = gas_balance(address)
+    r = conn.do(o)
+    hx = strip_0x(r)
+    return int(hx, 16)
+
+
+def main():
+    recipient = to_checksum(args.recipient)
+    if not args.u and recipient != add_0x(args.recipient):
+        raise ValueError('invalid checksum address')
+
+    value = args.amount
+
+    logg.debug('sender {} balance before: {}'.format(signer_address, balance(signer_address)))
+    logg.debug('recipient {} balance before: {}'.format(recipient, balance(recipient)))
+ 
+    g = GasTxFactory(signer, gas_oracle, nonce_oracle, chain_id=chain_id)
+    (tx_hash_hex, o) = g.create(signer_address, recipient, value)
+    conn.do(o)
+
+    print(tx_hash_hex)
+
+
+if __name__ == '__main__':
+    main()

cic_tools/eth/tx.py

@@ -10,6 +10,7 @@ from rlp import encode as rlp_encode
 
 # local imports
 from .address import to_checksum
+from .constant import MINIMUM_FEE_UNITS
 
 logg = logging.getLogger(__name__)
 
@@ -81,3 +82,27 @@ def unpack_signed(tx_raw_bytes, chain_id=1):
             }
 
 
+class TxFactory:
+
+    def __init__(self, signer, gas_oracle, nonce_oracle, chain_id=1):
+        self.gas_oracle = gas_oracle
+        self.nonce_oracle = nonce_oracle
+        self.chain_id = chain_id
+        self.signer = signer
+
+
+    def template(self, sender, recipient):
+        gas_price = self.gas_oracle.get()
+        logg.debug('using gas price {}'.format(gas_price))
+        nonce = self.nonce_oracle.next()
+        logg.debug('using nonce {} for address {}'.format(nonce, sender))
+        return {
+                'from': sender,
+                'to': recipient,
+                'value': 0,
+                'data': '0x',
+                'nonce': nonce,
+                'gasPrice': gas_price,
+                'gas': MINIMUM_FEE_UNITS,
+                'chainId': self.chain_id,
+                }

requirements.txt

@@ -13,3 +13,4 @@ hexathon==0.0.1a2
 eth-abi==2.1.1
 eth-keys==0.3.3
 websocket-client==0.57.0
+redis==3.5.3