Correct check test chainspec var

apps/cic-cache/cic_cache/api.py

@@ -55,37 +55,15 @@ class Api:
                     queue=callback_queue,
                     )       
 
-    def list(self, offset=0, limit=100, address=None, oldest=False):
+    def list(self, offset, limit, address=None):
         s = celery.signature(
         'cic_cache.tasks.tx.tx_filter',
         [
-            offset,
-            limit,
+            0,
+            100,
             address,
-            oldest,
             ],
-            queue=self.queue,
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t
-
-
-    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter_content',
-        [
-            offset,
-            limit,
-            address,
-            block_offset,
-            block_limit,
-            oldest,
-            ],
-            queue=self.queue,
+            queue=None
         )
         if self.callback_param != None:
             s.link(self.callback_success).on_error(self.callback_error)

apps/cic-cache/cic_cache/cache.py

@@ -10,16 +10,12 @@ from cic_cache.db.list import (
         list_transactions_mined,
         list_transactions_account_mined,
         list_transactions_mined_with_data,
-        list_transactions_mined_with_data_index,
-        list_transactions_account_mined_with_data_index,
-        list_transactions_account_mined_with_data,
         )
 
 logg = logging.getLogger()
 
 
 DEFAULT_FILTER_SIZE = 8192 * 8
-DEFAULT_LIMIT = 100
 
 class Cache:
 
@@ -36,7 +32,7 @@ class BloomCache(Cache):
         return n
 
 
-    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions(self, offset, limit):
         """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
 
         Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -53,7 +49,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+        rows = list_transactions_mined(self.session, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -62,12 +58,7 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -76,7 +67,7 @@ class BloomCache(Cache):
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
 
 
-    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions_account(self, address, offset, limit):
         """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
 
         :param address: Address to retrieve transactions for.
@@ -88,7 +79,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -97,12 +88,7 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -113,21 +99,8 @@ class BloomCache(Cache):
 
 class DataCache(Cache):
 
-    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def __process_rows(self, rows, oldest):
+    def load_transactions_with_data(self, offset, end):
+        rows = list_transactions_mined_with_data(self.session, offset, end) 
         tx_cache = []
         highest_block = -1;
         lowest_block = -1;
@@ -135,12 +108,7 @@ class DataCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r['block_number']
-                lowest_block = r['block_number']
-            else:
-                if oldest:
-                    highest_block = r['block_number']
-                else:
-                    lowest_block = r['block_number']
+            lowest_block = r['block_number']
             tx_type = 'unknown'
 
             if r['value'] != None:

apps/cic-cache/cic_cache/cli/arg.py

@@ -12,7 +12,7 @@ class ArgumentParser(BaseArgumentParser):
 
     def process_local_flags(self, local_arg_flags):
         if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
+            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
         if local_arg_flags & CICFlag.SYNCER:
             self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
             self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -1,5 +1,5 @@
 [celery]
 broker_url = redis://localhost:6379
 result_url = 
-queue = cic-cache
+queue = cic-eth
 debug = 0

apps/cic-cache/cic_cache/db/list.py

@@ -13,9 +13,6 @@ def list_transactions_mined(
         session,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -26,62 +23,15 @@ def list_transactions_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
+    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
     r = session.execute(s)
     return r
 
 
 def list_transactions_mined_with_data(
-        session,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
-
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data_index(
         session,
         offset,
         end,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -92,87 +42,7 @@ def list_transactions_mined_with_data_index(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined_with_data_index(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
-    r = session.execute(s)
-    return r
-
-def list_transactions_account_mined_with_data(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
+    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
 
     r = session.execute(s)
     return r
@@ -183,9 +53,6 @@ def list_transactions_account_mined(
         address,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
 
@@ -198,20 +65,7 @@ def list_transactions_account_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-
-    else:
-        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
+    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
     r = session.execute(s)
     return r
 

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -91,14 +91,13 @@ def process_transactions_all_data(session, env):
     if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
         return None
 
-    logg.debug('got data request {}'.format(env))
-    block_offset = r[1]
-    block_end = r[2]
+    offset = r[1]
+    end = r[2]
     if int(r[2]) < int(r[1]):
         raise ValueError('cart before the horse, dude')
 
     c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
+    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
 
     for r in tx_cache:
         r['date_block'] = r['date_block'].timestamp()

apps/cic-cache/cic_cache/tasks/tx.py

@@ -2,17 +2,14 @@
 import celery
 
 # local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-        )
+from cic_cache.cache import BloomCache
 from cic_cache.db.models.base import SessionBase
 
 celery_app = celery.current_app
 
 
 @celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, encoding='hex'):
     queue = self.request.delivery_info.get('routing_key')
 
     session = SessionBase.create_session()
@@ -20,9 +17,9 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     c = BloomCache(session)
     b = None
     if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
     else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
 
     session.close()
 
@@ -38,17 +35,4 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     return o
 
 
-@celery_app.task(bind=True)
-def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
-    session = SessionBase.create_session()
 
-    c = DataCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
-    else:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
-
-    session.close()
-
-    return (lowest_block, highest_block, tx_cache,)

apps/cic-cache/cic_cache/version.py

@@ -4,8 +4,8 @@ import semver
 version = (
         0,
         2,
-        1,
-        'alpha.1',
+        0,
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/requirements.txt

@@ -2,14 +2,13 @@ alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 uwsgi==2.0.19.1
 moolb~=0.1.1b2
-cic-eth-registry~=0.6.1a1
+cic-eth-registry~=0.5.8a1
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
 chainsyncer[sql]>=0.0.6a1,<0.1.0
-erc20-faucet>=0.3.2a1, <0.4.0
-chainlib-eth==0.0.9a2
-chainlib==0.0.9a2
-eth-address-index>=0.2.3a1,<0.3.0
+erc20-faucet~=0.2.4a1
+chainlib>=0.0.7a1,<0.1.0
+eth-address-index>=0.1.4a1,<0.2.0

apps/cic-cache/setup.cfg

@@ -41,4 +41,3 @@ console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
-	cic-cache-list = cic_cache.runable.list:main

apps/cic-cache/test_requirements.txt

@@ -6,5 +6,5 @@ sqlparse==0.4.1
 pytest-celery==0.0.0a1
 eth_tester==0.5.0b3
 py-evm==0.3.0a20
-sarafu-faucet~=0.0.7a1
-erc20-transfer-authorization>=0.3.5a1,<0.4.0
+sarafu-faucet~=0.0.5a2
+erc20-transfer-authorization>=0.3.4a1,<0.4.0

apps/cic-cache/tests/cli/test_cli_args.py

@@ -1,40 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_cache.cli
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
-
-
-def test_argumentparserto_config():
-
-    argparser = cic_cache.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-

apps/cic-cache/tests/cli/test_cli_celery.py

@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_cache.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_cache.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_cache.cli.CeleryApp.from_config(config)

apps/cic-cache/tests/cli/test_cli_chain.py

@@ -1,68 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-from chainlib.eth.pytest.fixtures_chain import default_chain_spec
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from hexathon import add_0x
-
-# local imports
-import cic_cache.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        ):
-    ifc = cic_cache.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    recipient = add_0x(os.urandom(20).hex())
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-

apps/cic-cache/tests/conftest.py

@@ -64,6 +64,7 @@ def txs(
         dt.timestamp(),
             )
 
+
     tx_number = 42
     tx_hash_second = '0x' + os.urandom(32).hex()
     tx_signed_second = '0x' + os.urandom(128).hex()
@@ -92,44 +93,6 @@ def txs(
             ]
 
 
-@pytest.fixture(scope='function')
-def more_txs(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        ):
-
-    session = init_database
-
-    tx_number = 666
-    tx_hash = '0x' + os.urandom(32).hex()
-    tx_signed = '0x' + os.urandom(128).hex()
-    nonce = 3
-
-    dt = datetime.datetime.utcnow()
-    dt += datetime.timedelta(hours=1)
-    db.add_transaction(
-        session,
-        tx_hash,
-        list_defaults['block']+2,
-        tx_number,
-        list_actors['alice'],
-        list_actors['diane'],
-        list_tokens['bar'],
-        list_tokens['bar'],
-        2048,
-        4096,
-        False,
-        dt.timestamp(),
-        )
-
-    session.commit()
-
-    return [tx_hash] + txs
-
-
 @pytest.fixture(scope='function')
 def tag_txs(
         init_database,

apps/cic-cache/tests/test_cache.py

@@ -8,7 +8,6 @@ import json
 import pytest
 
 # local imports
-from cic_cache import db
 from cic_cache import BloomCache
 from cic_cache.cache import DataCache
 
@@ -19,6 +18,7 @@ def test_cache(
         init_database,
         list_defaults,
         list_actors,
+        list_tokens,
         txs,
         ):
 
@@ -37,6 +37,9 @@ def test_cache(
 
 def test_cache_data(
         init_database,
+        list_defaults,
+        list_actors,
+        list_tokens,
         txs,
         tag_txs,
         ):
@@ -44,209 +47,10 @@ def test_cache_data(
     session = init_database
 
     c = DataCache(session)
-    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
+    b = c.load_transactions_with_data(410000, 420000)
 
     assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[0]
-    assert b[2][0]['tx_type'] == 'unknown'
-    assert b[2][1]['tx_type'] == 'test.taag'
-
-
-def test_cache_ranges(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = BloomCache(session)
-    b = c.load_transactions(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    # now check when supplying account
-    b = c.load_transactions_account(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    # add block filter to the mix
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
+    assert b[2][0]['tx_hash'] == txs[1]
+    assert b[2][1]['tx_type'] == 'unknown'
+    assert b[2][0]['tx_type'] == 'test.taag'
     
-    b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-
-    b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-
-
-def test_cache_ranges_data( 
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = DataCache(session)
-
-    b = c.load_transactions_with_data(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    # now check when supplying account
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    # add block filter to the mix
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[2]

apps/cic-cache/tests/testdata/config/test.ini

@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy

apps/cic-eth/admin_requirements.txt

@@ -1,5 +1,5 @@
 SQLAlchemy==1.3.20
-cic-eth-registry>=0.6.1a1,<0.7.0
-hexathon~=0.0.1a8
-chainqueue>=0.0.3a2,<0.1.0
-eth-erc20>=0.1.2a1,<0.2.0
+cic-eth-registry>=0.5.6a2,<0.6.0
+hexathon~=0.0.1a7
+chainqueue>=0.0.3a1,<0.1.0
+eth-erc20>=0.0.10a3,<0.1.0

apps/cic-eth/cic_eth/api/api_task.py

@@ -520,9 +520,9 @@ class Api(ApiBase):
             s_external_get = celery.signature(
                 external_task,
                 [
+                    address,
                     offset,
                     limit,
-                    address,
                     ],
                 queue=external_queue,
                 )

apps/cic-eth/cic_eth/cli/rpc.py

@@ -24,24 +24,20 @@ class RPC:
 
 
     def get_default(self):
-        return self.get_by_label('default')
-
-
-    def get_by_label(self, label):
-        return RPCConnection.connect(self.chain_spec, label)
+        return RPCConnection.connect(self.chain_spec, 'default')
 
 
     @staticmethod
-    def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
+    def from_config(config, use_signer=False):
         chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
 
-        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
+        RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
         if use_signer:
 
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
-            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
-            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label) 
+            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, 'signer')
+            RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, 'signer')
+            RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, 'signer')
+            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
         rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
         logg.info('set up rpc: {}'.format(rpc))
         return rpc

apps/cic-eth/cic_eth/data/config/dispatcher.ini

@@ -1,2 +0,0 @@
-[dispatcher]
-loop_interval = 1

apps/cic-eth/cic_eth/data/config/eth.ini

@@ -1,2 +1,2 @@
 [eth]
-gas_gifter_minimum_balance = 10000000000000000000000
+gas_gifter_minimum_balance = 10000000000000000000

apps/cic-eth/cic_eth/data/config/retry.ini

@@ -1,3 +0,0 @@
-[retry]
-delay =
-batch_size =

apps/cic-eth/cic_eth/ext/tx.py

@@ -12,11 +12,9 @@ from chainlib.eth.tx import (
         transaction_by_block,
         receipt,
         )
-from chainlib.eth.error import RequestMismatchException
 from chainlib.eth.block import block_by_number
 from chainlib.eth.contract import abi_decode_single
 from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.eth.tx import Tx
 from hexathon import strip_0x
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.erc20 import ERC20Token
@@ -25,8 +23,6 @@ from chainqueue.db.models.otx import Otx
 from chainqueue.db.enum import StatusEnum
 from chainqueue.sql.query import get_tx_cache
 from eth_erc20 import ERC20
-from erc20_faucet import Faucet
-from potaahto.symbols import snake_and_camel
 
 # local imports
 from cic_eth.queue.time import tx_times
@@ -39,32 +35,6 @@ logg = logging.getLogger()
 MAX_BLOCK_TX = 250
 
 
-def parse_transaction(chain_spec, rpc, tx, sender_address=None):
-    try:
-        transfer_data = ERC20.parse_transfer_request(tx['input'])
-        tx_address = transfer_data[0]
-        tx_token_value = transfer_data[1]
-        logg.debug('matched transfer transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
-        return (tx_address, tx_token_value)
-    except RequestMismatchException:
-        pass
-
-    try:
-        transfer_data = Faucet.parse_give_to_request(tx['input'])
-        tx_address = transfer_data[0]
-        c = Faucet(chain_spec)
-        o = c.token_amount(tx['to'], sender_address=sender_address, height=tx['block_number'])
-        r = rpc.do(o)
-        tx_token_value = Faucet.parse_token_amount(r)
-        logg.debug('matched giveto transaction {} in block {} sender {} recipient {} value {}'.format(tx['hash'], tx['block_number'], tx['from'], tx_address, tx_token_value))
-        return (tx_address, tx_token_value)
-
-    except RequestMismatchException:
-        pass
-
-    return None
-
-
 # TODO: Make this method easier to read
 @celery_app.task(bind=True, base=BaseTask)
 def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
@@ -101,39 +71,36 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     tx_filter = moolb.Bloom(databitlen, bloomspec['filter_rounds'], default_data=tx_filter_data)
 
     txs = {}
-    logg.debug('processing filter with span low {} to high {}'.format(bloomspec['low'], bloomspec['high']))
     for block_height in range(bloomspec['low'], bloomspec['high']):
         block_height_bytes = block_height.to_bytes(4, 'big')
         if block_filter.check(block_height_bytes):
             logg.debug('filter matched block {}'.format(block_height))
             o = block_by_number(block_height)
             block = rpc.do(o)
+            logg.debug('block {}'.format(block))
 
             for tx_index in range(0, len(block['transactions'])):
-                tx_index_bytes = tx_index.to_bytes(4, 'big')
-                composite = block_height_bytes + tx_index_bytes
-                if tx_filter.check(composite):
+                composite = tx_index + block_height
+                tx_index_bytes = composite.to_bytes(4, 'big')
+                if tx_filter.check(tx_index_bytes):
                     logg.debug('filter matched block {} tx {}'.format(block_height, tx_index))
 
-                    o = transaction_by_block(block['hash'], tx_index)
                     try:
+                        #tx = c.w3.eth.getTransactionByBlock(block_height, tx_index)
+                        o = transaction_by_block(block['hash'], tx_index)
                         tx = rpc.do(o)
                     except Exception as e:
                         logg.debug('false positive on block {} tx {} ({})'.format(block_height, tx_index, e))
                         continue
-
-                    tx = Tx(tx).src()
-
-                    logg.debug('got tx {}'.format(tx))
                     tx_address = None
                     tx_token_value = 0
-
-                    transfer_data = parse_transaction(chain_spec, rpc, tx, sender_address=BaseTask.call_address)
-                    if transfer_data == None:
+                    try:
+                        transfer_data = ERC20.parse_transfer_request(tx['data'])
+                        tx_address = transfer_data[0]
+                        tx_token_value = transfer_data[1]
+                    except ValueError:
+                        logg.debug('not a transfer transaction, skipping {}'.format(tx))
                         continue
-                    tx_address = transfer_data[0]
-                    tx_token_value = transfer_data[1]
-                    
                     if address == tx_address:
                         status = StatusEnum.SENT
                         try:
@@ -169,7 +136,6 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
     return txs
 
 
-
 # TODO: Surely it must be possible to optimize this
 # TODO: DRY this with callback filter in cic_eth/runnable/manager
 # TODO: Remove redundant fields from end representation (timestamp, tx_hash)

apps/cic-eth/cic_eth/runnable/tag.py

@@ -18,7 +18,7 @@ from cic_eth.db.models.base import SessionBase
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
+arg_flags = cic_eth.cli.argflag_std_base
 local_arg_flags = cic_eth.cli.argflag_local_taskcallback
 argparser = cic_eth.cli.ArgumentParser(arg_flags)
 argparser.add_positional('tag', type=str, help='address tag')

apps/cic-eth/cic_eth/version.py

@@ -9,8 +9,8 @@ import semver
 version = (
         0,
         12,
-        4,
-        'alpha.3',
+        2,
+        'alpha.4',
         )
 
 version_object = semver.VersionInfo(

apps/cic-eth/config/database.ini

@@ -0,0 +1,10 @@
+[database]
+NAME=cic_eth
+USER=postgres
+PASSWORD=tralala
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+POOL_SIZE=50
+DEBUG=0

apps/cic-eth/config/docker/chain.ini

@@ -1,2 +0,0 @@
-[chain]
-spec = evm:bloxberg:8996

apps/cic-eth/config/docker/database.ini

@@ -0,0 +1,10 @@
+[database]
+NAME=cic_eth
+USER=postgres
+PASSWORD=tralala
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+POOL_SIZE=50
+DEBUG=0

apps/cic-eth/requirements.txt

@@ -1,3 +1,3 @@
 celery==4.4.7
-chainlib-eth>=0.0.9a2,<0.1.0
+chainlib-eth>=0.0.7a1,<0.1.0
 semver==2.13.0

apps/cic-eth/scripts/migrate.py

@@ -10,7 +10,6 @@ from alembic.config import Config as AlembicConfig
 import confini
 
 from cic_eth.db import dsn_from_config
-import cic_eth.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
@@ -20,20 +19,25 @@ rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 dbdir = os.path.join(rootdir, 'cic_eth', 'db')
 migrationsdir = os.path.join(dbdir, 'migrations')
 
-arg_flags = cic_eth.cli.argflag_std_base
+config_dir = os.path.join('/usr/local/etc/cic-eth')
 
-argparser = cic_eth.cli.ArgumentParser(arg_flags)
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
 argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
 argparser.add_argument('-f', action='store_true', help='force action')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-extra_args = {
-    'migrations_dir': None,
-    'reset': None,
-    'f': '_FORCE_ACTION',
-        }
-config = cic_eth.cli.Config.from_args(args, arg_flags, 0, extra_args=extra_args)
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
 config.censor('PASSWORD', 'DATABASE')
 config.censor('PASSWORD', 'SSL')
 logg.debug('config:\n{}'.format(config))

apps/cic-eth/services_requirements.txt

@@ -3,13 +3,13 @@ chainsyncer[sql]>=0.0.6a1,<0.1.0
 alembic==1.4.2
 confini>=0.3.6rc4,<0.5.0
 redis==3.5.3
-hexathon~=0.0.1a8
+hexathon~=0.0.1a7
 pycryptodome==3.10.1
 liveness~=0.0.1a7
-eth-address-index>=0.2.3a1,<0.3.0
-eth-accounts-index>=0.1.2a1,<0.2.0
-cic-eth-registry>=0.6.1a1,<0.7.0
-erc20-faucet>=0.3.2a1,<0.4.0
-erc20-transfer-authorization>=0.3.5a1,<0.4.0
-sarafu-faucet>=0.0.7a1,<0.1.0
+eth-address-index>=0.1.4a1,<0.2.0
+eth-accounts-index>=0.0.14a1,<0.1.0
+cic-eth-registry>=0.5.8a1,<0.6.0
+erc20-faucet>=0.2.4a1,<0.3.0
+erc20-transfer-authorization>=0.3.4a1,<0.4.0
+sarafu-faucet>=0.0.5a2,<0.1.0
 moolb~=0.1.1b2

apps/cic-eth/test_requirements.txt

@@ -6,4 +6,4 @@ pytest-redis==2.0.0
 redis==3.5.3
 eth-tester==0.5.0b3
 py-evm==0.3.0a20
-eth-erc20~=0.1.2a1
+eth-erc20~=0.0.12a1

apps/cic-eth/tests/filters/test_callback_filter.py

@@ -200,7 +200,6 @@ def test_callback_filter(
     assert r['status'] == 1
 
     rcpt = snake_and_camel(r)
-    tx.block.hash = rcpt['block_hash']
     tx.apply_receipt(rcpt)
 
     fltr = CallbackFilter(default_chain_spec, None, None, caller_address=contract_roles['CONTRACT_DEPLOYER'])

apps/cic-eth/tests/testdata/config/test.ini

@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy

apps/cic-eth/tests/unit/cli/test_cli_args.py

@@ -1,56 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_eth.cli
-
-logg = logging.getLogger()
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-#config_dir = os.path.join(script_dir, '..', '..', 'testdata', 'config')
-
-
-def test_argumentparser_to_config():
-
-    argparser = cic_eth.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '--redis-host', 'foo',
-        '--redis-port', '123',
-        '--redis-db', '0',
-        '--redis-host-callback', 'bar',
-        '--redis-port-callback', '456',
-        '--redis-timeout', '10.0',
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    #config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-    config = cic_eth.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('REDIS_HOST') == 'foo'
-    assert config.get('REDIS_PORT') == 123
-    assert config.get('REDIS_DB') == 0
-    assert config.get('_REDIS_HOST_CALLBACK') == 'bar'
-    assert config.get('_REDIS_PORT_CALLBACK') == 456
-    assert config.get('REDIS_TIMEOUT') == 10.0
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-

apps/cic-eth/tests/unit/cli/test_cli_celery.py

@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_eth.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_eth.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_eth.cli.CeleryApp.from_config(config)

apps/cic-eth/tests/unit/cli/test_cli_chain.py

@@ -1,64 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-
-# local imports
-import cic_eth.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_eth.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        agent_roles,
-        ):
-    ifc = cic_eth.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], agent_roles['ALICE'], 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-

apps/cic-eth/tests/unit/ext/test_ext_tx.py

@@ -20,6 +20,7 @@ from cic_eth.db.models.nonce import (
 logg = logging.getLogger()
 
 
+# TODO: This test fails when not run alone. Identify which fixture leaves a dirty state
 def test_filter_process(
         init_database,
         default_chain_spec,
@@ -47,10 +48,10 @@ def test_filter_process(
     eth_rpc.do(o)
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
-    block_bytes = r['block_number'].to_bytes(4, 'big')
-    b.add(block_bytes)
-    tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
-    t.add(block_bytes + tx_index_bytes)
+    a = r['block_number']
+    b.add(a.to_bytes(4, 'big'))
+    a = r['block_number'] + r['transaction_index']
+    t.add(a.to_bytes(4, 'big'))
     tx_hashes.append(tx_hash_hex)
 
     # external tx
@@ -60,10 +61,10 @@ def test_filter_process(
     eth_rpc.do(o)
     o = receipt(tx_hash_hex)
     r = eth_rpc.do(o)
-    block_bytes = r['block_number'].to_bytes(4, 'big')
-    b.add(block_bytes)
-    tx_index_bytes = r['transaction_index'].to_bytes(4, 'big')
-    t.add(block_bytes + tx_index_bytes)
+    a = r['block_number']
+    b.add(a.to_bytes(4, 'big'))
+    a = r['block_number'] + r['transaction_index']
+    t.add(a.to_bytes(4, 'big'))
     tx_hashes.append(tx_hash_hex)
 
     init_eth_tester.mine_blocks(10)

apps/cic-eth/tools_requirements.txt

@@ -1,7 +1,8 @@
-crypto-dev-signer>=0.4.15a1,<=0.4.15
+crypto-dev-signer>=0.4.14b7,<=0.4.14
 chainqueue>=0.0.3a1,<0.1.0
-cic-eth-registry>=0.6.1a1,<0.7.0
+confini>=0.3.6rc4,<0.5.0
+cic-eth-registry>=0.5.8a1,<0.6.0
 redis==3.5.3
-hexathon~=0.0.1a8
+hexathon~=0.0.1a7
 pycryptodome==3.10.1
 pyxdg==0.27

apps/cic-ussd/cic_ussd/version.py

@@ -1,7 +1,7 @@
 # standard imports
 import semver
 
-version = (0, 3, 1, 'alpha.1')
+version = (0, 3, 0, 'alpha.10')
 
 version_object = semver.VersionInfo(
         major=version[0],

apps/cic-ussd/requirements.txt

@@ -1,10 +1,10 @@
 alembic==1.4.2
 bcrypt==3.2.0
 celery==4.4.7
-cic-eth[services]~=0.12.4a3
+cic-eth[services]==0.12.2a3
 cic-notify~=0.4.0a10
 cic-types~=0.1.0a14
-confini>=0.4.1a1,<0.5.0
+confini~=0.4.1a1
 phonenumbers==8.12.12
 psycopg2==2.8.6
 python-i18n[YAML]==0.3.9

apps/contract-migration/docker/Dockerfile

@@ -27,26 +27,17 @@ ENV CONFINI_DIR /usr/local/etc/cic/
 
 COPY config_template/ /usr/local/etc/cic/
 COPY requirements.txt . 
-COPY override_requirements.txt . 
 
 ARG pip_index_url=https://pypi.org/simple
 ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
-ARG EXTRA_PIP_ARGS=""
 ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
 ARG pip_trusted_host=pypi.org
 RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
     pip install --index-url https://pypi.org/simple \
     --force-reinstall \
     --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
+    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
     -r requirements.txt 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url https://pypi.org/simple \
-    --force-reinstall \
-    --trusted-host $pip_trusted_host \
-    --extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
-    -r override_requirements.txt 
-
 
 COPY . .
 RUN chmod +x *.sh

apps/contract-migration/override_requirements.txt

@@ -1,4 +1,3 @@
-eth-contract-registry==0.6.3a2
+#eth-contract-registry==0.5.5a3
 #erc20-demurrage-token==0.0.2a3
 #eth-address-index==0.1.1a12
-

apps/contract-migration/requirements.txt

@@ -1,10 +1,10 @@
-cic-eth[tools]==0.12.4a3
-chainlib-eth>=0.0.9a2,<0.1.0
-eth-erc20>=0.1.2a1,<0.2.0
-erc20-demurrage-token>=0.0.5a1,<0.1.0
-eth-accounts-index>=0.1.2a1,<0.2.0
-eth-address-index>=0.2.3a3,<0.3.0
-cic-eth-registry>=0.6.1a1,<0.7.0
-erc20-faucet>=0.3.2a1,<0.4.0
-erc20-transfer-authorization>=0.3.5a1,<0.4.0
-sarafu-faucet>=0.0.7a1,<0.1.0
+cic-eth[tools]==0.12.2a4
+eth-erc20>=0.0.12a1,<0.1.0
+erc20-demurrage-token>=0.0.3a1,<0.1.0
+eth-address-index>=0.1.4a1,<0.2.0
+eth-accounts-index>=0.0.14a1,<0.1.0
+cic-eth-registry>=0.5.8a1,<0.6.0
+erc20-faucet>=0.2.4a1,<0.3.0
+erc20-transfer-authorization>=0.3.4a1,<0.4.0
+sarafu-faucet>=0.0.5a5,<0.1.0
+chainlib-eth>=0.0.7a1,<0.1.0

apps/contract-migration/reset.sh

@@ -67,9 +67,6 @@ echo -n 1 > $init_level_file
 # Wait for the backend to be up, if we know where it is.
 if [[ -n "${ETH_PROVIDER}" ]]; then
 
-	export CONFINI_DIR=$_CONFINI_DIR
-	unset CONFINI_DIR
-
 	if [ ! -z "$DEV_USE_DOCKER_WAIT_SCRIPT" ]; then
 		echo "waiting for ${ETH_PROVIDER}..."
 		./wait-for-it.sh "${ETH_PROVIDER_HOST}:${ETH_PROVIDER_PORT}"
@@ -85,8 +82,8 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 			TOKEN_NAME=$TOKEN_SYMBOL
 		fi
 		>&2 echo deploying default token $TOKEN_TYPE
-		echo giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
-		DEV_RESERVE_ADDRESS=`giftable-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -s -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
+		echo giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv
+		DEV_RESERVE_ADDRESS=`giftable-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -ww --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL --decimals 6 -vv`
 	elif [ "$TOKEN_TYPE" == "erc20_demurrage_token" ]; then
 		if [ -z "$TOKEN_SYMBOL" ]; then
 			>&2 echo token symbol not set, setting defaults for type $TOKEN_TYPE
@@ -102,57 +99,61 @@ if [[ -n "${ETH_PROVIDER}" ]]; then
 				>&2 echo -e "\033[;93mtoken sink address not set, so redistribution will be BURNED\033[;39m"
 			fi
 		fi
-		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww -s` 
+		DEV_RESERVE_ADDRESS=`erc20-demurrage-token-deploy $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC --name "$TOKEN_NAME" --symbol $TOKEN_SYMBOL -vv -ww`
 	else
 		>&2 echo unknown token type $TOKEN_TYPE
 		exit 1
 	fi
-
-	echo "giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT"
-	giftable-token-gift $fee_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -u -vv -s -w -e $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
+	giftable-token-gift $gas_price_arg -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -vv -w -a $DEV_RESERVE_ADDRESS $DEV_RESERVE_AMOUNT
 
 	>&2 echo "deploy account index contract"
-	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -s -u -w`
+	DEV_ACCOUNT_INDEX_ADDRESS=`eth-accounts-index-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -y $DEV_ETH_KEYSTORE_FILE -vv -w`
 	>&2 echo "add deployer address as account index writer"
-	eth-accounts-index-writer $fee_price_arg -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
+	eth-accounts-index-writer $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww -vv $debug $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER
 
 	>&2 echo "deploy contract registry contract"
-	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $fee_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization --identifier ContractRegistry -p $ETH_PROVIDER -vv -s -u -w`
-	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier ContractRegistry $CIC_REGISTRY_ADDRESS
-	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
+	CIC_REGISTRY_ADDRESS=`eth-contract-registry-deploy $gas_price_arg -i $CIC_CHAIN_SPEC -y $DEV_ETH_KEYSTORE_FILE --identifier BancorRegistry --identifier AccountRegistry --identifier TokenRegistry --identifier AddressDeclarator --identifier Faucet --identifier TransferAuthorization -p $ETH_PROVIDER -vv -w`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv ContractRegistry $CIC_REGISTRY_ADDRESS
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AccountRegistry $DEV_ACCOUNT_INDEX_ADDRESS
 
 	# Deploy address declarator registry
 	>&2 echo "deploy address declarator contract"
 	declarator_description=0x546869732069732074686520434943206e6574776f726b000000000000000000
-	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -s -u -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
-	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier AddressDeclarator $DEV_DECLARATOR_ADDRESS
+	DEV_DECLARATOR_ADDRESS=`eth-address-declarator-deploy -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv $declarator_description`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv AddressDeclarator $DEV_DECLARATOR_ADDRESS
 
 	# Deploy transfer authorization contact
 	>&2 echo "deploy transfer auth contract"
 	DEV_TRANSFER_AUTHORIZATION_ADDRESS=`erc20-transfer-auth-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
-	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv --identifier TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC  -p $ETH_PROVIDER -vv TransferAuthorization $DEV_TRANSFER_AUTHORIZATION_ADDRESS
 
 	# Deploy token index contract
 	>&2 echo "deploy token index contract"
-	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy -s -u $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
-	eth-contract-registry-set $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
+	DEV_TOKEN_INDEX_ADDRESS=`eth-token-index-deploy $gas_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv`
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv TokenRegistry $DEV_TOKEN_INDEX_ADDRESS 
 	>&2 echo "add reserve token to token index"
-	eth-token-index-add $fee_price_arg -s -u -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -e $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
+	eth-token-index-add $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE  -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv -a $DEV_TOKEN_INDEX_ADDRESS $DEV_RESERVE_ADDRESS
 
 	# Sarafu faucet contract
 	>&2 echo "deploy token faucet contract"
+	export _CONFINI_DIR=$CONFINI_DIR
+	unset CONFINI_DIR
 	DEV_FAUCET_ADDRESS=`sarafu-faucet-deploy $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -w -vv --account-index-address $DEV_ACCOUNT_INDEX_ADDRESS $DEV_RESERVE_ADDRESS -s`
 
 	>&2 echo "set token faucet amount"
 	sarafu-faucet-set $fee_price_arg -y $DEV_ETH_KEYSTORE_FILE -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_FAUCET_ADDRESS -vv -s --fee-limit 100000 $DEV_FAUCET_AMOUNT
 
+	export CONFINI_DIR=$_CONFINI_DIR
+
 	>&2 echo "register faucet in registry"
-	eth-contract-registry-set -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv --identifier Faucet $DEV_FAUCET_ADDRESS
+	eth-contract-registry-set $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -r $CIC_REGISTRY_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv Faucet $DEV_FAUCET_ADDRESS
 
 	>&2 echo "set faucet as token minter"
-	giftable-token-minter -s -u $fee_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -e $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+	giftable-token-minter $gas_price_arg -w -y $DEV_ETH_KEYSTORE_FILE -a $DEV_RESERVE_ADDRESS -i $CIC_CHAIN_SPEC -p $ETH_PROVIDER -vv $DEV_FAUCET_ADDRESS
+
+
+
 
-	export CONFINI_DIR=$_CONFINI_DIR
 else
 	echo "\$ETH_PROVIDER not set!"
 	exit 1

apps/contract-migration/seed_cic_eth.sh

@@ -35,13 +35,10 @@ set -a
 
 #pip install --extra-index-url $DEV_PIP_EXTRA_INDEX_URL  eth-address-index==0.1.1a7
 
-export CONFINI_DIR=$_CONFINI_DIR
-unset CONFINI_DIR
-
 # get required addresses from registries
-DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw TokenRegistry`
-DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -u -i $CHAIN_SPEC -p $ETH_PROVIDER -e $CIC_REGISTRY_ADDRESS -vv --raw AccountRegistry`
-DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -u -p $ETH_PROVIDER -e $DEV_TOKEN_INDEX_ADDRESS -vv --raw $CIC_DEFAULT_TOKEN_SYMBOL`
+DEV_TOKEN_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief TokenRegistry`
+DEV_ACCOUNT_INDEX_ADDRESS=`eth-contract-registry-list -i $CHAIN_SPEC -p $ETH_PROVIDER -r $CIC_REGISTRY_ADDRESS -f brief AccountRegistry`
+DEV_RESERVE_ADDRESS=`eth-token-index-list -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_TOKEN_INDEX_ADDRESS -f brief $CIC_DEFAULT_TOKEN_SYMBOL`
 cat <<EOF
 Token registry: $DEV_TOKEN_INDEX_ADDRESS
 Account reigstry: $DEV_ACCOUNT_INDEX_ADDRESS
@@ -75,29 +72,28 @@ DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=`CONFINI_DIR=$empty_config_dir cic-eth-c
 echo DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER=$DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER >> $env_out_file
 cic-eth-tag -i $CHAIN_SPEC ACCOUNT_REGISTRY_WRITER $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 >&2 echo "add acccounts index writer account as writer on contract"
-eth-accounts-index-writer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
+eth-accounts-index-writer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_ACCOUNT_INDEX_ADDRESS -ww $debug $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER
 
 # Transfer gas to custodial gas provider adddress
 _CONFINI_DIR=$CONFINI_DIR
 unset CONFINI_DIR
 >&2 echo gift gas to gas gifter
->&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_GAS_GIFTER $gas_amount
 
 >&2 echo gift gas to sarafu token owner
->&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER $gas_amount
 
 >&2 echo gift gas to account index owner
->&2 eth-gas -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
+>&2 eth-gas --send -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -w $debug -a $DEV_ETH_ACCOUNT_ACCOUNT_REGISTRY_WRITER $gas_amount
 
 
 # Send token to token creator
 >&2 echo "gift tokens to sarafu owner"
-echo "giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount"
->&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount
+>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_SARAFU_GIFTER -w $debug $token_amount
 
 # Send token to token gifter
 >&2 echo "gift tokens to keystore address"
->&2 giftable-token-gift -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -e $DEV_RESERVE_ADDRESS -a $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount
+>&2 giftable-token-gift -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER -a $DEV_RESERVE_ADDRESS --recipient $DEV_ETH_ACCOUNT_CONTRACT_DEPLOYER -w $debug $token_amount
 
 >&2 echo "set sarafu token to reserve token (temporarily while bancor contracts are not connected)"
 echo DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS >> $env_out_file
@@ -105,17 +101,15 @@ export DEV_ETH_SARAFU_TOKEN_ADDRESS=$DEV_ETH_RESERVE_ADDRESS
 
 # Transfer tokens to gifter address
 >&2 echo "transfer tokens to token gifter address"
->&2 erc20-transfer -s -u -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --fee-limit 100000 -e $DEV_RESERVE_ADDRESS -w $debug -a $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
+>&2 erc20-transfer -y $keystore_file -i $CHAIN_SPEC -p $ETH_PROVIDER --gas-limit 100000 --token-address $DEV_RESERVE_ADDRESS -w $debug $DEV_ETH_ACCOUNT_SARAFU_GIFTER ${token_amount:0:-1}
 
 #echo -n 0 > $init_level_file
 
-#CONFINI_DIR=$_CONFINI_DIR
+CONFINI_DIR=$_CONFINI_DIR
 # Remove the SEND (8), QUEUE (16) and INIT (2) locks (or'ed), set by default at migration
-cic-eth-ctl -i $CHAIN_SPEC unlock INIT
-cic-eth-ctl -i $CHAIN_SPEC unlock SEND
-cic-eth-ctl -i $CHAIN_SPEC unlock QUEUE
-
-export CONFINI_DIR=$_CONFINI_DIR
+cic-eth-ctl -i :: unlock INIT
+cic-eth-ctl -i :: unlock SEND
+cic-eth-ctl -i :: unlock QUEUE
 
 set +a
 set +e

apps/data-seeding/eth/import_balance.py

@@ -37,8 +37,7 @@ from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.dict import DictKeystore
 from cic_types.models.person import Person
 from eth_erc20 import ERC20
-#from cic_base.eth.syncer import chain_interface
-from cic_eth.cli.chain import chain_interface
+from cic_base.eth.syncer import chain_interface
 from eth_accounts_index import AccountsIndex
 from eth_contract_registry import Registry
 from eth_token_index import TokenUniqueSymbolIndex
@@ -173,7 +172,6 @@ class Handler:
         erc20 = ERC20(self.chain_spec, signer=self.signer, gas_oracle=self.gas_oracle, nonce_oracle=self.nonce_oracle)
         o = erc20.decimals(self.token_address)
         r = conn.do(o)
-        logg.debug('parse dec {}'.format(r))
         decimals = erc20.parse_decimals(r)
         multiplier = 10 ** decimals
         balance_full = balance * multiplier

apps/data-seeding/requirements.txt

@@ -1,13 +1,13 @@
-sarafu-faucet~=0.0.7a1
-cic-eth[tools]~=0.12.4a2
+sarafu-faucet~=0.0.5a2
+cic-eth[tools]~=0.12.2a3
 cic-types~=0.1.0a14
-crypto-dev-signer>=0.4.15a1,<=0.4.15
+crypto-dev-signer~=0.4.14rc1
 faker==4.17.1
 chainsyncer~=0.0.6a1
-chainlib-eth~=0.0.9a2
-eth-address-index~=0.2.3a1
-eth-contract-registry~=0.6.3a1
-eth-accounts-index~=0.1.2a1
-eth-erc20~=0.1.2a1
-erc20-faucet~=0.3.2a1
+chainlib-eth~=0.0.7a1
+eth-address-index~=0.1.4a1
+eth-contract-registry~=0.5.8a1
+eth-accounts-index~=0.0.14a1
+eth-erc20~=0.0.12a1
+erc20-faucet~=0.2.4a1
 psycopg2==2.8.6

apps/data-seeding/verify.py

@@ -205,9 +205,8 @@ def send_ussd_request(address, data_dir):
 
 class VerifierState:
 
-    def __init__(self, item_keys, target_count, active_tests=None):
+    def __init__(self, item_keys, active_tests=None):
         self.items = {}
-        self.target_count = target_count
         for k in item_keys:
             self.items[k] = 0
         if active_tests == None:
@@ -224,7 +223,7 @@ class VerifierState:
         r = ''
         for k in self.items.keys():
             if k in self.active_tests:
-                r += '{}: {}/{}\n'.format(k, self.items[k], self.target_count)
+                r += '{}: {}\n'.format(k, self.items[k])
             else:
                 r += '{}: skipped\n'.format(k)
         return r
@@ -245,7 +244,7 @@ class VerifierError(Exception):
 class Verifier:
 
     # TODO: what an awful function signature
-    def __init__(self, conn, target_count, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
+    def __init__(self, conn, cic_eth_api, gas_oracle, chain_spec, index_address, token_address, faucet_address, data_dir, exit_on_error=False):
         self.conn = conn
         self.gas_oracle = gas_oracle
         self.chain_spec = chain_spec
@@ -269,7 +268,7 @@ class Verifier:
         self.faucet_amount = self.faucet_tx_factory.parse_token_amount(r)
         logg.info('faucet amount set to {} at verify initialization time'.format(self.faucet_amount))
 
-        self.state = VerifierState(verifymethods, target_count, active_tests=active_tests)
+        self.state = VerifierState(verifymethods, active_tests=active_tests)
 
 
     def verify_accounts_index(self, address, balance=None):
@@ -482,7 +481,7 @@ def main():
 
     f.close()
 
-    verifier = Verifier(conn, i, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
+    verifier = Verifier(conn, api, gas_oracle, chain_spec, account_index_address, token_address, faucet_address, user_dir, exit_on_error)
 
     user_new_dir = os.path.join(user_dir, 'new')
     i = 0
@@ -518,7 +517,6 @@ def main():
             verifier.verify(new_address, balance, debug_stem=s)
             i += 1
 
-    print()
     print(verifier)
 
 

docker-compose.yml

@@ -80,7 +80,6 @@ services:
         pip_index_url: ${PIP_DEFAULT_INDEX_URL:-https://pypi.org/simple}
         pip_extra_args: $PIP_EXTRA_ARGS
         EXTRA_INDEX_URL: ${EXTRA_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
-        EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
     # image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/contract-migration:latest
     environment:
       CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
@@ -239,7 +238,7 @@ services:
       - |
         if [[ -f /tmp/cic/config/.env ]]; then source /tmp/cic/config/.env; fi 
         "/usr/local/bin/uwsgi" \
-        --wsgi-file /root/cic_cache/runnable/daemons/server.py \
+        --wsgi-file /usr/src/cic-cache/cic_cache/runnable/daemons/server.py \
         --http :8000 \
         --pyargv "-vv"