Add temporary db commit per tx in tracker

Correct chainlib import paths

See merge request grassrootseconomics/cic-internal-integration!101

.gitignore

@@ -1,2 +1,6 @@
 service-configs/*
 !service-configs/.gitkeep
+**/node_modules/
+__pycache__
+*.pyc
+*.o

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -67,6 +67,7 @@ class ERC20TransferFilter(SyncFilter):
                 tx.status == Status.SUCCESS,
                 block.timestamp,
                 )
-        db_session.flush()
+        #db_session.flush()
+        db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -77,7 +77,7 @@ def main():
 
     if len(syncer_backends) == 0:
         logg.info('found no backends to resume')
-        syncers.append(SQLBackend.initial(chain_spec, block_offset))
+        syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))

apps/cic-eth/cic_eth/eth/meta.py

@@ -66,10 +66,10 @@ class ExtendedTx:
         self.status_code = n
 
 
-    def to_dict(self):
+    def asdict(self):
         o = {}
         for attr in dir(self):
-            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'to_dict']:
+            if attr[0] == '_' or attr in ['set_actors', 'set_tokens', 'set_status', 'asdict', 'rpc']:
                 continue
             o[attr] = getattr(self, attr)
         return o 

apps/cic-eth/cic_eth/runnable/daemons/filters/callback.py

@@ -78,19 +78,19 @@ class CallbackFilter(SyncFilter):
 
         o = c.amount(faucet_contract, sender_address=self.caller_address)
         r = conn.do(o)
-        transfer_data['amount'] = add_0x(c.parse_amount(r))
+        transfer_data['value'] = c.parse_amount(r)
 
         return ('tokengift', transfer_data)
 
 
     def call_back(self, transfer_type, result):
-        logg.debug('result {}'.format(result))
+        result['chain_spec'] = result['chain_spec'].asdict()
         s = celery.signature(
             self.method,
             [
                 result,
                 transfer_type,
-                int(result['status_code'] == 0),
+                int(result['status_code'] != 0),
             ],
             queue=self.queue,
             )
@@ -106,7 +106,7 @@ class CallbackFilter(SyncFilter):
 #        s_translate.link(s)
 #        s_translate.apply_async()
         t = s.apply_async()
-        return s
+        return t
 
 
     def parse_data(self, tx, conn):
@@ -167,8 +167,9 @@ class CallbackFilter(SyncFilter):
                     tokentx.set_status(1)
                 else:
                     tokentx.set_status(0)
-                t = self.call_back(transfer_type, tokentx.to_dict())
-                logg.info('callback success task id {} tx {}'.format(t, tx.hash))
+                result = tokentx.asdict()
+                t = self.call_back(transfer_type, result)
+                logg.info('callback success task id {} tx {} queue {}'.format(t, tx.hash, t.queue))
             except UnknownContractError:
                 logg.debug('callback filter {}:{} skipping "transfer" method on unknown contract {} tx {}'.format(tx.queue, tx.method, transfer_data['to'], tx.hash))
 

apps/cic-eth/cic_eth/runnable/daemons/tasker.py

@@ -39,6 +39,7 @@ from cic_eth.queue import (
 from cic_eth.callbacks import (
         Callback,
         http,
+        noop,
         #tcp,
         redis,
         )

apps/cic-notify/cic_notify/runnable/tasker.py

@@ -33,7 +33,9 @@ elif args.v:
 
 config = confini.Config(args.c, args.env_prefix)
 config.process()
+config.add(args.q, '_CELERY_QUEUE', True)
 config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 dsn = dsn_from_config(config)

apps/cic-notify/cic_notify/version.py

@@ -6,11 +6,10 @@ import time
 import semver
 
 # local imports
-from cic_notify.error import PleaseCommitFirstError
 
 logg = logging.getLogger()
 
-version = (0, 4, 0, 'alpha.3')
+version = (0, 4, 0, 'alpha.4')
 
 version_object = semver.VersionInfo(
         major=version[0],
@@ -18,27 +17,4 @@ version_object = semver.VersionInfo(
         patch=version[2],
         prerelease=version[3],
         )
-
 version_string = str(version_object)
-
-
-def git_hash():
-    import subprocess
-
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-        time_string_pair[0],
-        int(time_string_pair[1]),
-    )
-logg.info(f'Final version string will be {version_string}')
-
-__version_string__ = version_string

apps/cic-notify/setup.cfg

@@ -1,6 +1,5 @@
 [metadata]
 name = cic-notify
-version= 0.4.0a3
 description = CIC notifications service
 author = Louis Holbrook
 author_email = dev@holbrook.no

apps/cic-notify/setup.py

@@ -1,9 +1,31 @@
 # standard imports
+import logging
+import subprocess
+import time
 from setuptools import setup
 
-# third-party imports
-
 # local imports
+from cic_notify.version import version_string
+
+logg = logging.getLogger()
+
+
+def git_hash():
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+            time_string_pair[0],
+            int(time_string_pair[1]),
+            )
+logg.info(f'Final version string will be {version_string}')
 
 
 requirements = []
@@ -25,6 +47,6 @@ while True:
 test_requirements_file.close()
 
 setup(
+    version=version_string,
     install_requires=requirements,
-    tests_require=test_requirements,
-)
+    tests_require=test_requirements)

apps/cic-ussd/.config/redis.ini

@@ -1,6 +1,6 @@
 [celery]
-BROKER_URL=redis://redis:6379
-RESULT_URL=redis://redis:6379
+BROKER_URL=redis://
+RESULT_URL=redis://
 
 [redis]
 HOSTNAME=redis

apps/cic-ussd/cic_ussd/runnable/server.py

@@ -57,19 +57,17 @@ arg_parser.add_argument('--env-prefix',
                         help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
-# parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-
 # define log levels
 if args.vv:
     logging.getLogger().setLevel(logging.DEBUG)
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-# log config vars
-logg.debug(config)
+# parse config
+config = Config(config_dir=args.c, env_prefix=args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # initialize elements
 # set up translations

apps/cic-ussd/cic_ussd/runnable/tasker.py

@@ -6,6 +6,7 @@ import tempfile
 
 # third party imports
 import celery
+import i18n
 import redis
 from confini import Config
 
@@ -33,18 +34,18 @@ arg_parser.add_argument('-vv', action='store_true', help='be more verbose')
 arg_parser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
 args = arg_parser.parse_args()
 
-# parse config
-config = Config(config_dir=args.c, env_prefix=args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-
 # define log levels
 if args.vv:
     logging.getLogger().setLevel(logging.DEBUG)
 elif args.v:
     logging.getLogger().setLevel(logging.INFO)
 
-logg.debug(config)
+# parse config
+config = Config(args.c, args.env_prefix)
+config.process()
+config.add(args.q, '_CELERY_QUEUE', True)
+config.censor('PASSWORD', 'DATABASE')
+logg.debug('config loaded from {}:\n{}'.format(args.c, config))
 
 # connect to database
 data_source_name = dsn_from_config(config)
@@ -77,6 +78,10 @@ if key_file_path:
     validate_presence(path=key_file_path)
 Signer.key_file_path = key_file_path
 
+# set up translations
+i18n.load_path.append(config.get('APP_LOCALE_PATH'))
+i18n.set('fallback', config.get('APP_LOCALE_FALLBACK'))
+
 # set up celery
 current_app = celery.Celery(__name__)
 

apps/cic-ussd/cic_ussd/tasks/callback_handler.py

@@ -83,8 +83,8 @@ def process_incoming_transfer_callback(result: dict, param: str, status_code: in
         # collect result data
         recipient_blockchain_address = result.get('recipient')
         sender_blockchain_address = result.get('sender')
-        token_symbol = result.get('token_symbol')
-        value = result.get('destination_value')
+        token_symbol = result.get('destination_token_symbol')
+        value = result.get('destination_token_value')
 
         # try to find users in system
         recipient_user = session.query(User).filter_by(blockchain_address=recipient_blockchain_address).first()

apps/cic-ussd/requirements.txt

@@ -1,4 +1,4 @@
 cic_base[full_graph]~=0.1.2a68
 cic-eth~=0.11.0b3
-cic-notify~=0.4.0a3
+cic-notify~=0.4.0a4
 cic-types~=0.1.0a10

apps/contract-migration/scripts/cic_eth/import_balance.py

@@ -25,13 +25,13 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer

apps/contract-migration/scripts/eth/import_balance.py

@@ -25,13 +25,13 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import OverrideGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from chainlib.chain import ChainSpec
 from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer

apps/contract-migration/scripts/eth/import_users.py

@@ -24,7 +24,7 @@ from chainlib.eth.gas import RPCGasOracle
 from chainlib.eth.nonce import RPCNonceOracle
 from cic_types.processor import generate_metadata_pointer
 from eth_accounts_index import AccountRegistry
-from contract_registry import Registry
+from eth_contract_registry import Registry
 from crypto_dev_signer.keystore.dict import DictKeystore
 from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer
 from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict

apps/contract-migration/scripts/package.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "cic-client-meta": "^0.0.7-alpha.6",
+    "cic-client-meta": "0.0.7-alpha.6",
     "vcard-parser": "^1.0.0"
   }
 }

apps/contract-migration/scripts/verify.py

@@ -22,7 +22,7 @@ from hexathon import (
         strip_0x,
         add_0x,
         )
-from chainsyncer.backend import MemBackend
+from chainsyncer.backend.memory import MemBackend
 from chainsyncer.driver import HeadSyncer
 from chainlib.chain import ChainSpec
 from chainlib.eth.connection import EthHTTPConnection
@@ -32,7 +32,7 @@ from chainlib.eth.block import (
         block_by_number,
         Block,
         )
-from chainlib.eth.hash import keccak256_string_to_hex
+from chainlib.hash import keccak256_string_to_hex
 from chainlib.eth.address import to_checksum_address
 from chainlib.eth.erc20 import ERC20
 from chainlib.eth.gas import (
@@ -40,7 +40,7 @@ from chainlib.eth.gas import (
         balance,
         )
 from chainlib.eth.tx import TxFactory
-from chainlib.eth.rpc import jsonrpc_template
+from chainlib.jsonrpc import jsonrpc_template
 from chainlib.eth.error import EthException
 from cic_types.models.person import (
         Person,
@@ -57,6 +57,7 @@ custodial_tests = [
         'local_key',
         'gas',
         'faucet',
+        'ussd'
         ]
 
 metadata_tests = [

docker-compose.yml

@@ -446,7 +446,7 @@ services:
     deploy:
       restart_policy:
         condition: on-failure
-    command: "/root/start_tasker.sh -q cic-notify"
+    command: "/root/start_tasker.sh -q cic-notify -vv"
 
 
   cic-meta-server:
@@ -494,6 +494,8 @@ services:
       DATABASE_NAME: cic_ussd
       DATABASE_ENGINE: postgresql
       DATABASE_DRIVER: psycopg2
+      CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis}
+      CELERY_RESULT_URL: ${CELERY_BROKER_URL:-redis://redis}
       PGP_PASSPHRASE: merman
       SERVER_PORT: 9000
       CIC_META_URL: ${CIC_META_URL:-http://meta:8000}

scripts/initdb/create_db.sql

@@ -5,6 +5,7 @@ CREATE DATABASE "cic_notify";
 CREATE DATABASE "cic_meta";
 CREATE DATABASE "cic_signer";
 CREATE DATABASE "cic_ussd";
+CREATE DATABASE "chain_sync";
 GRANT ALL PRIVILEGES 
-ON DATABASE "cic_cache", "cic_eth", "cic_notify", "cic_meta", "cic_signer", "cic_ussd"
+ON DATABASE "cic_cache", "cic_eth", "cic_notify", "cic_meta", "cic_signer", "cic_ussd", "chain_sync"
 TO grassroots;