Dirty refactors to get integration working.

.dockerignore

@@ -0,0 +1,4 @@
+.git
+.cache
+.dot
+**/doc

.gitignore

@@ -1,17 +1,2 @@
 service-configs/*
 !service-configs/.gitkeep
-**/node_modules/
-__pycache__
-*.pyc
-*.o
-gmon.out
-*.egg-info
-dist/
-build/
-**/*sqlite
-**/.nyc_output
-**/coverage
-**/.venv
-**/venv
-**/dist
-.idea

.gitlab-ci.yml

@@ -2,13 +2,8 @@ include:
   - local: 'ci_templates/.cic-template.yml'
   - local: 'apps/contract-migration/.gitlab-ci.yml'
   - local: 'apps/cic-eth/.gitlab-ci.yml'
-  - local: 'apps/cic-ussd/.gitlab-ci.yml'
-  - local: 'apps/cic-notify/.gitlab-ci.yml'
-  - local: 'apps/cic-meta/.gitlab-ci.yml'
-  - local: 'apps/cic-cache/.gitlab-ci.yml'
-  - local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
   - build
   - test
-  - publish
+  - release

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "apps/cic-notify"]
+	path = apps/cic-notify
+	url = git@gitlab.com:grassrootseconomics/cic-notify.git
+[submodule "apps/cic-cache"]
+	path = apps/cic-cache
+	url = git@gitlab.com:grassrootseconomics/cic-cache.git
+[submodule "apps/cic-meta"]
+	path = apps/cic-meta
+	url = git@gitlab.com:grassrootseconomics/cic-meta.git

README.md

@@ -2,22 +2,6 @@
 
 ## Getting started 
 
-## Make some keys
-
-```
-docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
-```
-
-
-### Prepare the repo
-
-This is stuff we need to put in  makefile but for now...
-
-File mounts and permisssions need to be set
-```
-chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
-````
-
 start cluster
 ```
 docker-compose up
@@ -36,7 +20,6 @@ docker-compose down -v
 rebuild an images
 ```
 docker-compose up --build <service_name>
-```
-
-Deployment variables are writtend to service-configs/.env after everthing is up.
+``
 
+Deployment variables are writtend to service-configs/.env after everthing is up.`

apps/bloxbergValidatorSetup/.gitignore

@@ -1,6 +1,3 @@
 /validator/bloxbergData
 /validator/bloxberg.log
-keys/*
-!keys/Bloxberg
-keys/Bloxberg/*
-!keys/Bloxberg/UTC--2021-02-10T16-57-35Z--03512a62-5334-20cc-4e44-71156f33cff6
+keys/**/*

apps/bloxbergValidatorSetup/Dockerfile

@@ -17,12 +17,10 @@ COPY ./validator/bloxberg.json \
      ./validator/validator.toml \
      /root/
 
-COPY keys/ /root/keys/
-
 # RUN chown -R parity:parity $HOME/ && \
 #     chmod -R 775 $HOME/ && \
 #     chmod g+s $HOME/
 # USER parity
 
 ENTRYPOINT [ "parity" ]
-CMD [ "--config", "/root/validator.toml", "--keys-path", "/root/keys/", "--password", "/root/validator.pwd" ]
+CMD [ "--config", "/root/validator.toml", "--keys-path", "/root/keys/" ]

apps/bloxbergValidatorSetup/README.md

@@ -4,7 +4,7 @@
 The original bloxberg node config was kind of annoying so I am running it more like vanilla parity. This way you can pass command flags directly to parity.
 ## Make some keys
 ```
-docker build -t bloxie . && docker run -v ${PWD}/keys:/root/keys --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys --password /root/validator.pwd
+docker build -t bloxie . && docker run -v ${PWD}/keys:/root/keys --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
 ```
 
 ## Enter the signer address and passwords in the config files

apps/bloxbergValidatorSetup/keys/Bloxberg/UTC--2021-02-10T16-57-35Z--03512a62-5334-20cc-4e44-71156f33cff6

@@ -1 +0,0 @@
-{"id":"03512a62-5334-20cc-4e44-71156f33cff6","version":3,"crypto":{"cipher":"aes-128-ctr","cipherparams":{"iv":"dc388338c4d4e3203604aeb3d1c6bbfa"},"ciphertext":"8a945775b87089ce94537e011799f3abc1577c5dd1f3fbaebe1cd96dfdfc8b5a","kdf":"pbkdf2","kdfparams":{"c":10240,"dklen":32,"prf":"hmac-sha256","salt":"e8585836540caca01282381f5c1fe128e53b15b40f9d152fbc5a4f82a7967398"},"mac":"a7c7815e84a632ecf6d8f18c981bea73d50cd2e2a855a3e90477fc84ed14f906"},"address":"4f2a5902158c3969b245247f4154971d393301f2","name":"","meta":"{}"}

apps/bloxbergValidatorSetup/validator/bloxberg.json

@@ -7,7 +7,7 @@
                 "maximumUncleCount": 0,
                 "stepDuration": "5",
                 "validators" : {
-                     "list": ["0x4f2a5902158c3969b245247f4154971d393301f2"]
+                     "list": ["0x0ade5bd2065debbb250e816f614bc342a6ef71c1"]
                 }
             }
         }

apps/bloxbergValidatorSetup/validator/validator.toml

@@ -16,17 +16,15 @@ interface = "all"
 [websockets]
 disable = false
 port = 8546
-#apis = ["web3", "eth", "net", "personal", "parity", "parity_set", "traces", "rpc", "parity_accounts"]
-apis = ["all"]
 interface = "all"
-origins = ["*"]
+origins = ["none"]
 
 [account]
 password = ["/root/validator.pwd"]
 
 [mining]
 #CHANGE ENGINE SIGNER TO VALIDATOR ADDRESS
-engine_signer = "0x4f2a5902158c3969b245247f4154971d393301f2"
+engine_signer = "0x0ade5bd2065debbb250e816f614bc342a6ef71c1"
 reseal_on_txs = "none"
 force_sealing = true
 min_gas_price = 1000000

apps/cic-base-os/Dockerfile

@@ -0,0 +1,34 @@
+# The solc image messes up the alpine environment, so we have to go all over again
+FROM python:3.8.6-slim-buster
+
+LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
+LABEL spdx-license-identifier="GPL-3.0-or-later"
+LABEL description="Base layer for buiding development images for the cic component suite"
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
+
+
+RUN echo installing nodejs tooling
+
+COPY ./dev/nvm.sh /root/
+
+# Install nvm with node and npm
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
+ENV NVM_DIR /root/.nvm
+ENV NODE_VERSION 15.3.0
+ENV BANCOR_NODE_VERSION 10.16.0
+
+RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
+	&& . $NVM_DIR/nvm.sh \
+	&& nvm install $NODE_VERSION \
+	&& nvm alias default $NODE_VERSION \
+	&& nvm use $NODE_VERSION \
+# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
+# owner of these files is "1001" by default - wtf
+	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
+

apps/cic-base-os/README.md

@@ -0,0 +1 @@
+## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-base/.gitignore

@@ -1,7 +0,0 @@
-gmon.out
-__pycache__
-*.pyc
-inc.sh
-*.egg-info
-build/
-dist/

apps/cic-base/.gitlab-ci.yml

@@ -1,20 +0,0 @@
-.cic_base_variables:
-    variables:
-        APP_NAME: cic-base
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_base_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
-
-build-mr-cic-base:
-    extends:
-        - .cic_base_changes_target
-        - .py_build_merge_request
-        - .cic_base_variables
-
-publish_python:
-  extends:
-    - .publish_python
-    - .cic_base_variables

apps/cic-base/MANIFEST.in

@@ -1 +0,0 @@
-include *requirements.txt

apps/cic-base/cic_base/__init__.py

@@ -1,8 +0,0 @@
-#from . import (
-#        config,
-#        argparse,
-#        rpc,
-#        signer,
-#        log,
-#        version,
-#        )

apps/cic-base/cic_base/argparse.py

@@ -1,87 +0,0 @@
-# standard imports
-import logging
-import argparse
-import os
-import sys
-
-# external imports
-from xdg.BaseDirectory import (
-        xdg_config_dirs,
-        load_first_config,
-        )
-
-logg = logging.getLogger(__file__)
-
-fallback_config_path = '/usr/local/etc'
-xdg_config_dirs += [fallback_config_path]
-default_config_dir = load_first_config('cic')
-if default_config_dir == None:
-    default_config_dir = os.path.join('.', '.cic')
-env_config_dir = os.environ.get('CONFINI_DIR', default_config_dir)
-
-full_template = { 
-        # (long arg and key name, short var, type, default, help,)
-        'provider': ('p', str, None, 'RPC provider url',),
-        'registry_address': ('r', str, None, 'CIC registry address',),
-        'keystore_file': ('y', str, None, 'Keystore file',),
-        'config_dir': ('c', str, env_config_dir, 'Configuration directory',),
-        'queue': ('q', str, 'cic-eth', 'Celery task queue',),
-        'chain_spec': ('i', str, None, 'Chain spec string',),
-        'env_prefix': (None, str, os.environ.get('CONFINI_ENV_PREFIX'), 'Environment prefix for variables to overwrite configuration',),
-        }
-
-default_include_args = [
-    'config_dir',
-    'provider',
-    'env_prefix',
-    ]
-
-sub = None
-
-def create(caller_dir, include_args=default_include_args):
-
-    argparser = argparse.ArgumentParser()
-
-    for k in include_args:
-        a = full_template[k]
-        long_flag = '--' + k.replace('_', '-')
-        short_flag = None
-        dest = None
-        if a[0] != None:
-            short_flag = '-' + a[0]
-            dest = a[0]
-        else:
-            dest = k
-        default = a[2]
-        if default == None and k == 'config_dir':
-            default = os.path.join(os.getcwd(), 'config')
-        
-        if short_flag == None:
-            argparser.add_argument(long_flag, dest=dest, type=a[1], default=default, help=a[3])
-        else:
-            argparser.add_argument(short_flag, long_flag, dest=dest, type=a[1], default=default, help=a[3])
-
-    argparser.add_argument('-v', action='store_true', help='Be verbose')
-    argparser.add_argument('-vv', action='store_true', help='Be more verbose')
-
-    return argparser
-
-
-def add(argparser, processor, name, description=None):
-    processor(argparser)
-
-    return argparser
-
-
-def parse(argparser, logger=None):
-
-    args = argparser.parse_args(sys.argv[1:])
-
-    # handle logging input
-    if logger != None:
-        if args.vv:
-            logger.setLevel(logging.DEBUG)
-        elif args.v:
-            logger.setLevel(logging.INFO)
-
-    return args

apps/cic-base/cic_base/config.py

@@ -1,52 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import confini
-
-# local imports
-from .error import ConfigError
-
-logg = logging.getLogger(__name__)
-
-
-default_arg_overrides = {
-    'p': 'ETH_PROVIDER',
-    'i': 'CIC_CHAIN_SPEC',
-    'r': 'CIC_REGISTRY_ADDRESS',
-    }
-
-
-def override(config, override_dict, label):
-    config.dict_override(override_dict, label)
-    config.validate()
-    return config
-
-
-def create(config_dir, args, env_prefix=None, arg_overrides=default_arg_overrides):
-    # handle config input
-    config = None
-    try:
-        config = confini.Config(config_dir, env_prefix)
-    except OSError:
-        pass
-
-    if config == None:
-        raise ConfigError('directory {} not found'.format(config_dir))
-
-    config.process()
-    if arg_overrides != None and args != None:
-        override_dict = {}
-        for k in arg_overrides:
-            v = getattr(args, k)
-            if v != None:
-                override_dict[arg_overrides[k]] = v
-        config = override(config, override_dict, 'args')
-    else:
-        config.validate()
-
-    return config
-
-
-def log(config):
-    logg.debug('config loaded:\n{}'.format(config))

apps/cic-base/cic_base/error.py

@@ -1,2 +0,0 @@
-class ConfigError(Exception):
-    pass

apps/cic-base/cic_base/log.py

@@ -1,18 +0,0 @@
-# standard imports
-import logging
-
-logging.basicConfig(level=logging.WARNING)
-
-default_mutelist = [
-        'urllib3',
-        'websockets.protocol',
-        'web3.RequestManager',
-        'web3.providers.WebsocketProvider',
-        'web3.providers.HTTPProvider',
-        ]
-
-def create(name=None, mutelist=default_mutelist):
-    logg = logging.getLogger(name)
-    for m in mutelist:
-        logging.getLogger(m).setLevel(logging.CRITICAL)
-    return logg

apps/cic-base/cic_base/rpc.py

@@ -1,13 +0,0 @@
-# external imports
-from chainlib.connection import RPCConnection
-from chainlib.eth.connection import EthUnixSignerConnection
-from chainlib.eth.sign import (
-        sign_transaction,
-        sign_message,
-        )
-
-
-def setup(chain_spec, evm_provider, signer_provider=None):
-    RPCConnection.register_location(evm_provider, chain_spec, 'default')
-    if signer_provider != None:
-        RPCConnection.register_location(signer_provider, chain_spec, 'signer', constructor=EthUnixSignerConnection)

apps/cic-base/cic_base/signer.py

@@ -1,27 +0,0 @@
-# standard imports
-import logging
-import os
-
-# external imports 
-from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
-from crypto_dev_signer.keystore.dict import DictKeystore
-
-logg = logging.getLogger(__name__)
-
-keystore = DictKeystore()
-
-default_passphrase = os.environ.get('ETH_PASSPHRASE', '')
-
-
-def from_keystore(keyfile, passphrase=default_passphrase):
-    global keystore
-
-    # signer
-    if keyfile == None:
-        raise ValueError('please specify signer keystore file')
-
-    logg.debug('loading keystore file {}'.format(keyfile))
-    address = keystore.import_keystore_file(keyfile, password=passphrase)
-
-    signer = EIP155Signer(keystore)
-    return (address, signer,)

apps/cic-base/cic_base/sqlalchemy.py

@@ -1,122 +0,0 @@
-# stanard imports
-import logging
-
-# third-party imports
-from sqlalchemy import Column, Integer
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import (
-        StaticPool,
-        QueuePool,
-        AssertionPool,
-        )
-
-logg = logging.getLogger()
-
-Model = declarative_base(name='Model')
-
-
-class SessionBase(Model):
-    """The base object for all SQLAlchemy enabled models. All other models must extend this.
-    """
-    __abstract__ = True
- 
-    id = Column(Integer, primary_key=True)
-
-    engine = None
-    """Database connection engine of the running aplication"""
-    sessionmaker = None
-    """Factory object responsible for creating sessions from the  connection pool"""
-    transactional = True
-    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
-    poolable = True
-    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
-    procedural = True
-    """Whether the database backend supports stored procedures"""
-    localsessions = {}
-    """Contains dictionary of sessions initiated by db model components"""
-
-
-    @staticmethod
-    def create_session():
-        """Creates a new database session.
-        """
-        return SessionBase.sessionmaker()
-
-
-    @staticmethod
-    def _set_engine(engine):
-        """Sets the database engine static property
-        """
-        SessionBase.engine = engine
-        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
-
-
-    @staticmethod
-    def connect(dsn, pool_size=8, debug=False):
-        """Create new database connection engine and connect to database backend.
-
-        :param dsn: DSN string defining connection.
-        :type dsn: str
-        """
-        e = None
-        if SessionBase.poolable:
-            poolclass = QueuePool
-            if pool_size > 1:
-                e = create_engine(
-                        dsn,
-                        max_overflow=pool_size*3,
-                        pool_pre_ping=True,
-                        pool_size=pool_size,
-                        pool_recycle=60,
-                        poolclass=poolclass,
-                        echo=debug,
-                    )
-            else:
-                if debug:
-                    poolclass = AssertionPool
-                else:
-                    poolclass = StaticPool
-
-                e = create_engine(
-                        dsn,
-                        poolclass=poolclass,
-                        echo=debug,
-                    )
-        else:
-            e = create_engine(
-                    dsn,
-                    echo=debug,
-                    )
-
-        SessionBase._set_engine(e)
-
-
-    @staticmethod
-    def disconnect():
-        """Disconnect from database and free resources.
-        """
-        SessionBase.engine.dispose()
-        SessionBase.engine = None
-
-
-    @staticmethod
-    def bind_session(session=None):
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
-            localsession_key = str(id(localsession))
-            logg.debug('creating new session {}'.format(localsession_key))
-            SessionBase.localsessions[localsession_key] = localsession
-        return localsession
-
-
-    @staticmethod
-    def release_session(session=None):
-        session.flush()
-        session_key = str(id(session))
-        if SessionBase.localsessions.get(session_key) != None:
-            logg.debug('destroying session {}'.format(session_key))
-            session.commit()
-            session.close()

apps/cic-base/cic_base/version.py

@@ -1,43 +0,0 @@
-# standard imports
-import os
-import time
-import logging
-
-# third-party imports
-import semver
-
-version = (
-        0,
-        1,
-        2,
-        'beta.22',
-        )
-
-version_object = semver.VersionInfo(
-        major=version[0],
-        minor=version[1],
-        patch=version[2],
-        prerelease=version[3],
-        )
-
-
-def git_hash():
-    import subprocess
-    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-    return git_hash_brief
-
-version_string = str(version_object)
-
-try:
-    version_git = git_hash()
-    version_string += '+build.{}'.format(version_git)
-except FileNotFoundError:
-    time_string_pair = str(time.time()).split('.')
-    version_string += '+build.{}{:<09d}'.format(
-            time_string_pair[0],
-            int(time_string_pair[1]),
-            )
-
-__version_string__ = version_string

apps/cic-base/docker/Dockerfile

@@ -1,26 +0,0 @@
-FROM python:3.8.6-slim-buster
-
-RUN apt-get update && \
-	apt-get install -y git gcc g++ libpq-dev && \
-	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client cargo
-
-WORKDIR /usr/src/cic-base
-COPY . /usr/src/cic-base/
-
-#RUN mkdir python 
-#WORKDIR ./python
-#COPY ./pep503.sh .
-#RUN pip download --no-cache-dir --extra-index-url https://pip.grassrootseconomics.net:8433 cic-base[full_graph]==0.1.1a6
-RUN pip install -r requirements.txt
-RUN python setup.py bdist_wheel
-RUN pip download --extra-index-url https://pip.grassrootseconomics.net:8433 dist/$(basename $(ls dist/*))
-RUN mkdir packages && \
-	cd packages && \
-	bash ../docker/pep503.sh ..
-
-WORKDIR /usr/src/cic-base/packages
-
-RUN ls
-RUN ls ..
-
-ENTRYPOINT ["python", "-m", "http.server", "8080"]

apps/cic-base/docker/pep503.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env 
-
-dest=`pwd`
-d=$1
-for df in `find $d -name "*.whl" -type f`; do
-	f=`basename $df`
-	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
-	mkdir -v $dest/$pd
-	mv -v $df $dest/$pd/
-done
-for df in `find $d -name "*.tar.gz" -type f`; do
-	f=`basename $df`
-	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
-	mkdir -v $dest/$pd
-	mv -v $df $dest/$pd/
-done
-for df in `find $d -name "*.zip" -type f`; do
-	f=`basename $df`
-	pd=`echo $f | sed -e "s/^\(.*\)-[[:digit:]]*\.[[:digit:]].*$/\1/g" | tr "[:upper:]" "[:lower:]" | tr "_" "-"`
-	mkdir -v $dest/$pd
-	mv -v $df $dest/$pd/
-done

apps/cic-base/examples/cli.py

@@ -1,20 +0,0 @@
-import logging
-import os
-
-import cic_base.argparse
-import cic_base.config
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-
-def more_argparse(argparser):
-    argparser.add_argument('--foo', type=str, help='foo')
-
-script_dir = os.path.realpath(os.path.dirname(__file__))
-
-argparser = cic_base.argparse.create(script_dir, include_args=cic_base.argparse.full_template)
-args = cic_base.argparse.parse(argparser, logger=logg)
-config = cic_base.config.create(args.c, args, env_prefix=args.env_prefix)
-
-cic_base.config.log(config)

apps/cic-base/examples/config/foo.ini

@@ -1,3 +0,0 @@
-[foo]
-bar = 42
-baz = xyzzy

apps/cic-base/full_graph_requirements.txt

@@ -1,98 +0,0 @@
-africastalking==1.2.3
-alembic==1.4.2
-amqp==2.6.1
-attrs==20.3.0
-base58==2.1.0
-bcrypt==3.2.0
-billiard==3.6.3.0
-bip-utils==1.4.0
-bitarray==1.2.2
-blake2b-py==0.1.4
-cached-property==1.5.2
-celery==4.4.7
-certifi==2020.12.5
-cffi==1.14.3
-chainlib==0.0.1a18
-chainsyncer==0.0.1a18
-chardet==3.0.4
-confini==0.3.6rc3
-contextlib2==0.6.0.post1
-coverage==5.4
-cryptography==3.2.1
-cytoolz==0.11.0
-ecdsa==0.16.1
-ecuth==0.4.5a1
-Faker==4.17.1
-hexathon==0.0.1a3
-hexbytes==0.2.1
-http-hoba-auth==0.2.0
-idna==2.10
-iniconfig==1.1.1
-ipfshttpclient==0.6.1
-json-rpc==1.13.0
-jsonschema==3.2.0
-kombu==4.6.11
-lru-dict==1.1.7
-Mako==1.1.3
-MarkupSafe==1.1.1
-mirakuru==2.3.0
-moolb==0.1.1b2
-more-itertools==8.7.0
-multiaddr==0.0.9
-mypy-extensions==0.4.3
-netaddr==0.8.0
-packaging==20.9
-parsimonious==0.8.1
-phonenumbers==8.12.12
-pluggy==0.13.1
-port-for==0.4
-protobuf==3.15.1
-psutil==5.8.0
-psycopg2==2.8.6
-py==1.9.0
-py-ecc==4.1.0
-py-eth==0.1.1
-pycparser==2.20
-pycryptodome==3.10.1
-pyethash==0.1.27
-pyparsing==2.4.7
-pyrsistent==0.17.3
-pysha3==1.0.2
-pytest==6.0.1
-pytest-alembic==0.2.5
-pytest-celery==0.0.0a1
-pytest-cov==2.10.1
-pytest-mock==3.3.1
-pytest-redis==2.0.0
-python-dateutil==2.8.1
-python-editor==1.0.4
-python-gnupg==0.4.6
-python-i18n==0.3.9
-pytz==2021.1
-PyYAML==5.3.1
-redis==3.5.3
-requests==2.24.0
-rlp==2.0.1
-schema==0.7.4
-semantic-version==2.8.5
-semver==2.13.0
-six==1.15.0
-sortedcontainers==2.3.0
-SQLAlchemy==1.3.20
-sqlparse==0.4.1
-text-unidecode==1.3
-tinydb==4.2.0
-toml==0.10.2
-toolz==0.11.1
-transitions==0.8.4
-trie==2.0.0a5
-typing-extensions==3.7.4.3
-urllib3==1.25.11
-uWSGI==2.0.19.1
-varint==1.0.2
-vine==1.3.0
-vobject==0.9.6.1
-web3==5.12.2
-websocket-client==0.57.0
-websockets==8.1
-yaml-acl==0.0.1

apps/cic-base/mac_requirements.txt

@@ -1,40 +0,0 @@
-confini==0.3.6rc3
-crypto-dev-signer==0.4.14b1
-semver==2.13.0
-SQLAlchemy==1.3.20
-pyxdg==0.27
-chainlib==0.0.2a10
-alembic==1.4.2
-celery==4.4.7
-cryptography==3.2.1
-ecuth==0.4.5a1
-eth-accounts-index==0.0.11a8
-eth-address-index==0.1.1a8
-eth-contract-registry==0.5.4a9
-erc20-transfer-authorization==0.3.1a4
-erc20-single-shot-faucet==0.2.0a11
-faker==4.17.1
-http-hoba-auth==0.2.0
-moolb==0.1.1b2
-phonenumbers==8.12.12
-psycopg2==2.8.6
-python-i18n==0.3.9
-PyYAML==5.3.1
-redis==3.5.3
-requests==2.24.0
-sqlparse==0.4.1
-transitions==0.8.4
-uWSGI==2.0.19.1
-vobject==0.9.6.1
-web3==5.12.2
-websockets==8.1
-yaml-acl==0.0.1
-rlp==2.0.1
-cryptocurrency-cli-tools==0.0.4
-giftable-erc20-token==0.0.8a8
-websocket-client==0.57.0
-hexathon==0.0.1a7
-chainsyncer==0.0.2b1
-sarafu-faucet==0.0.2a20
-cic-types==0.1.0a10
-cic-eth-registry==0.5.4a13

apps/cic-base/requirements.txt

@@ -1,9 +0,0 @@
-confini==0.3.6rc3
-crypto-dev-signer==0.4.14b4
-semver==2.13.0
-SQLAlchemy==1.3.20
-pyxdg==0.27
-chainlib==0.0.3rc3
-eth-erc20==0.0.9a4
-liveness==0.0.1a7
-requirements-magic~=0.0.1a2

apps/cic-base/setup.py

@@ -1,50 +0,0 @@
-from setuptools import setup
-import configparser
-import os
-import logging
-import re
-
-logg = logging.getLogger(__name__)
-
-
-re_v = r'[~><=]='
-def merge(requirements_files, base_dir='.'):
-
-    requirements = {}
-    for r in requirements_files:
-        filepath = os.path.join(base_dir, r)
-        logg.debug('reading {}'.format(filepath))
-        f = open(filepath, 'r')
-        while True:
-            l = f.readline()
-            if l == '':
-                break
-            l = l.rstrip()
-            m = re.split(re_v, l)
-            k = m[0]
-            if k == None:
-                raise ValueError('invalid requirement line {}'.format(l))
-            if requirements.get(k) == None:
-                logg.info('adding {} -> {}'.format(k, l))
-                requirements[k] = l
-            else:
-                logg.debug('skipping {}'.format(l))
-        f.close()
-
-    return list(requirements.values())
-
-
-requirements = []
-f = open('requirements.txt', 'r')
-while True:
-    l = f.readline()
-    if l == '':
-        break
-    requirements.append(l.rstrip())
-f.close()
-
-
-
-setup(
-    install_requires=requirements,
-        )

apps/cic-base/tests/base.py

@@ -1,22 +0,0 @@
-# standard imports
-import logging
-import unittest
-
-# external imports
-from chainlib.chain import ChainSpec
-
-# local imports
-from cic_base.rpc import setup as rpc_setup
-
-logging.basicConfig(level=logging.DEBUG)
-logg = logging.getLogger()
-
-
-class TestBase(unittest.TestCase):
-
-    def setUp(self):
-        self.chain_spec = ChainSpec('evm', 'foo', 42)
-        rpc_setup(self.chain_spec, 'http://localhost:8545', signer_provider='ipc://tmp/foo')
-
-    def tearDown(self):
-        pass

apps/cic-base/tests/test_basic.py

@@ -1,14 +0,0 @@
-# standard imports
-import unittest
-
-# local imports
-from tests.base import TestBase
-
-
-class TestBasic(TestBase):
-    
-    def test_basic(self):
-        pass
-
-if __name__ == '__main__':
-    unittest.main()

apps/cic-cache/.config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/.config/cic.ini

@@ -1,2 +0,0 @@
-[cic]
-registry_address = 

apps/cic-cache/.config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=

apps/cic-cache/.config/eth.ini

@@ -1,6 +0,0 @@
-[eth]
-provider = ws://localhost:8545
-#ttp_provider = http://localhost:8545
-#provider = http://localhost:8545
-gas_provider_address = 
-#chain_id = 

apps/cic-cache/.config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/.config/test/cic.ini

@@ -1,2 +0,0 @@
-[cic]
-registry_address = 

apps/cic-cache/.config/test/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic-cache-test
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite
-DEBUG=

apps/cic-cache/.config/test/eth.ini

@@ -1,5 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-#chain_id = 

apps/cic-cache/.coveragerc

@@ -1,5 +0,0 @@
-[report]
-omit = 
-	.venv/*
-	scripts/*
-	cic_cache/db/postgres/*

apps/cic-cache/.envrc_example

@@ -1,7 +0,0 @@
-set -a
-CICTEST_DATABASE_ENGINE=postgresql
-CICTEST_DATABASE_DRIVER=psycopg2
-CICTEST_DATABASE_HOST=localhost
-CICTEST_DATABASE_PORT=5432
-CICTEST_DATABASE_NAME=cic-eth-test
-set +a

apps/cic-cache/.gitignore

@@ -1,8 +0,0 @@
-.envrc
-.envrc_dev
-.venv
-__pycache__
-*.pyc
-_build
-doc/**/*.png
-doc/**/html

apps/cic-cache/.gitlab-ci.yml

@@ -1,22 +0,0 @@
-.cic_cache_variables:
-    variables:
-        APP_NAME: cic-cache
-        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
-
-.cic_cache_changes_target:
-    rules:
-        - changes:
-            - $CONTEXT/$APP_NAME/*
-
-build-mr-cic-cache:
-    extends:
-        - .cic_cache_changes_target
-        - .py_build_merge_request
-        - .cic_cache_variables
-
-build-push-cic-cache:
-    extends:
-        - .py_build_push
-        - .cic_cache_variables
-        
-

apps/cic-cache/CHANGELOG

@@ -1,13 +0,0 @@
-- 0.1.2
-	* Revert to alembic migrations
-- 0.1.1
-	* Add missing modules to setup 
-- 0.1.0
-	* Remove old APIs
-	* Add bloom filter output APIs for all txs and per-account txs
-- 0.0.2
-	* UWSGI server endpoint example
-	* OpenAPI spec
-	* stored procedures, test fixture for database schema
-- 0.0.1
-	* Add json translators of transaction_list and balances stored procedure queries

apps/cic-cache/cic_cache/__init__.py

@@ -1 +0,0 @@
-from .cache import BloomCache

apps/cic-cache/cic_cache/api.py

@@ -1,73 +0,0 @@
-"""API for cic-cache celery tasks
-
-.. moduleauthor:: Louis Holbrook <dev@holbrook.no>
-
-"""
-# standard imports
-import logging
-
-# third-party imports 
-import celery
-
-
-app = celery.current_app
-
-logg = logging.getLogger(__name__)
-
-
-class Api:
-    """Creates task chains to perform well-known CIC operations.
-
-    Each method that sends tasks returns details about the root task. The root task uuid can be provided in the callback, to enable to caller to correlate the result with individual calls. It can also be used to independently poll the completion of a task chain.
-
-    :param callback_param: Static value to pass to callback
-    :type callback_param: str
-    :param callback_task: Callback task that executes callback_param call. (Must be included by the celery worker)
-    :type callback_task: string
-    :param queue: Name of worker queue to submit tasks to
-    :type queue: str
-    """
-    def __init__(self, queue='cic-cache', callback_param=None, callback_task='cic_cache.callbacks.noop.noop', callback_queue=None):
-        self.callback_param = callback_param
-        self.callback_task = callback_task
-        self.queue = queue
-        logg.info('api using queue {}'.format(self.queue))
-        self.callback_success = None
-        self.callback_error = None
-        if callback_queue == None:
-            callback_queue=self.queue
-
-        if callback_param != None:
-            self.callback_success = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        0,
-                        ],
-                    queue=callback_queue,
-                    )
-            self.callback_error = celery.signature(
-                    callback_task,
-                    [
-                        callback_param,
-                        1,
-                        ],
-                    queue=callback_queue,
-                    )       
-
-    def list(self, offset, limit, address=None):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter',
-        [
-            0,
-            100,
-            address,
-            ],
-            queue=None
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t

apps/cic-cache/cic_cache/cache.py

@@ -1,136 +0,0 @@
-# standard imports
-import logging
-import datetime
-
-# external imports
-import moolb
-
-# local imports
-from cic_cache.db.list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        list_transactions_mined_with_data,
-        )
-
-logg = logging.getLogger()
-
-
-class Cache:
-
-    def __init__(self, session):
-        self.session = session
-
-
-class BloomCache(Cache):
-
-    @staticmethod
-    def __get_filter_size(n):
-        n = 8192 * 8
-        logg.warning('filter size hardcoded to {}'.format(n))
-        return n
-
-
-    def load_transactions(self, offset, limit):
-        """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
-
-        Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
-
-        For example, if the block number is 13 and the transaction index is 42, the input are:
-
-        block filter:       0x0d000000
-        block+tx filter:    0x0d0000002a0000000
-
-        :param offset: Offset in data set to return transactions from
-        :type offset: int
-        :param limit: Max number of transactions to retrieve
-        :type limit: int
-        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
-        :rtype: tuple
-        """
-        rows = list_transactions_mined(self.session, offset, limit) 
-
-        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        highest_block = -1
-        lowest_block = -1
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r[0]
-            lowest_block = r[0]
-            block = r[0].to_bytes(4, byteorder='big')
-            tx = r[1].to_bytes(4, byteorder='big')
-            f_block.add(block)
-            f_blocktx.add(block + tx)
-            logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
-        return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
-
-
-    def load_transactions_account(self, address, offset, limit):
-        """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
-
-        :param address: Address to retrieve transactions for.
-        :type address: str, 0x-hex
-        :param offset: Offset in data set to return transactions from
-        :type offset: int
-        :param limit: Max number of transactions to retrieve
-        :type limit: int
-        :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
-        :rtype: tuple
-        """
-        rows = list_transactions_account_mined(self.session, address, offset, limit) 
-
-        f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
-        highest_block = -1;
-        lowest_block = -1;
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r[0]
-            lowest_block = r[0]
-            block = r[0].to_bytes(4, byteorder='big')
-            tx = r[1].to_bytes(4, byteorder='big')
-            f_block.add(block)
-            f_blocktx.add(block + tx)
-            logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
-        return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
-
-
-class DataCache(Cache):
-
-    def load_transactions_with_data(self, offset, end):
-        rows = list_transactions_mined_with_data(self.session, offset, end) 
-        tx_cache = []
-        highest_block = -1;
-        lowest_block = -1;
-        date_is_str = None # stick this in startup
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r['block_number']
-            lowest_block = r['block_number']
-            tx_type = 'unknown'
-
-            if r['value'] != None:
-                tx_type = '{}.{}'.format(r['domain'], r['value'])
-
-            if date_is_str == None:
-                date_is_str = type(r['date_block']).__name__ == 'str'
-
-            o = {
-                'block_number': r['block_number'],
-                'tx_hash': r['tx_hash'],
-                'date_block': r['date_block'],
-                'sender': r['sender'],
-                'recipient': r['recipient'],
-                'from_value': int(r['from_value']),
-                'to_value': int(r['to_value']),
-                'source_token': r['source_token'],
-                'destination_token': r['destination_token'],
-                'success': r['success'],
-                'tx_type': tx_type,
-            }
-
-            if date_is_str:
-                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
-
-            tx_cache.append(o)
-        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/db/__init__.py

@@ -1,40 +0,0 @@
-# standard imports
-import logging
-
-# local imports
-from .list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        add_transaction,
-        tag_transaction,
-        add_tag,
-    )
-
-
-logg = logging.getLogger()
-
-
-def dsn_from_config(config):
-    scheme = config.get('DATABASE_ENGINE')
-    if config.get('DATABASE_DRIVER') != None:
-        scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
-
-    dsn = ''
-    if config.get('DATABASE_ENGINE') == 'sqlite':
-        dsn = '{}:///{}'.format(
-                scheme,
-                config.get('DATABASE_NAME'),    
-            )
-
-    else:
-        dsn = '{}://{}:{}@{}:{}/{}'.format(
-                scheme,
-                config.get('DATABASE_USER'),
-                config.get('DATABASE_PASSWORD'),
-                config.get('DATABASE_HOST'),
-                config.get('DATABASE_PORT'),
-                config.get('DATABASE_NAME'),    
-            )
-    logg.debug('parsed dsn from config: {}'.format(dsn))
-    return dsn
-

apps/cic-cache/cic_cache/db/list.py

@@ -1,199 +0,0 @@
-# standard imports
-import logging
-import datetime
-
-# external imports
-from cic_cache.db.models.base import SessionBase
-from sqlalchemy import text
-
-logg = logging.getLogger()
-
-
-def list_transactions_mined(
-        session,
-        offset,
-        limit,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data(
-        session,
-        offset,
-        end,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined(
-        session,
-        address,
-        offset,
-        limit,
-        ):
-    """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
-
-    :param address: Address to retrieve transactions for.
-    :type address: str, 0x-hex
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
-    r = session.execute(s)
-    return r
-
-
-def add_transaction(
-        session,
-        tx_hash,
-        block_number,
-        tx_index,
-        sender,
-        receiver,
-        source_token,
-        destination_token,
-        from_value,
-        to_value,
-        success,
-        timestamp,
-        ):
-    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param block_number: Block number
-    :type block_number: int
-    :param tx_index: Transaction index in block
-    :type tx_index: int
-    :param sender: Ethereum address of effective sender
-    :type sender: str, 0x-hex
-    :param receiver: Ethereum address of effective recipient
-    :type receiver: str, 0x-hex
-    :param source_token: Ethereum address of token used by sender
-    :type source_token: str, 0x-hex
-    :param destination_token: Ethereum address of token received by recipient
-    :type destination_token: str, 0x-hex
-    :param from_value: Source token value spent in transaction
-    :type from_value: int
-    :param to_value: Destination token value received in transaction
-    :type to_value: int
-    :param success: True if code execution on network was successful
-    :type success: bool
-    :param date_block: Block timestamp
-    :type date_block: datetime
-    """
-    date_block = datetime.datetime.fromtimestamp(timestamp)
-    s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
-            tx_hash,
-            block_number,
-            tx_index,
-            sender,
-            receiver,
-            source_token,
-            destination_token,
-            from_value,
-            to_value,
-            success,
-            date_block,
-            )
-    session.execute(s)
-
-
-
-def tag_transaction(
-        session,
-        tx_hash,
-        name,
-        domain=None,
-        ):
-    """Tag a single transaction with a single tag.
-
-    Tag must already exist in storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises ValueError: Unknown tag or transaction hash
-
-    """
-
-    s = text("SELECT id from tx where tx_hash = :a")
-    r = session.execute(s, {'a': tx_hash}).fetchall()
-    tx_id = r[0].values()[0]
-
-    if tx_id == None:
-        raise ValueError('unknown tx hash {}'.format(tx_hash))
-
-    #s = text("SELECT id from tag where value = :a and domain = :b")
-    if domain == None:
-        s = text("SELECT id from tag where value = :a")
-    else:
-        s = text("SELECT id from tag where value = :a and domain = :b")
-    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
-    tag_id = r[0].values()[0]
-
-    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
-
-    if tag_id == None:
-        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
-
-    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
-    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
-
-
-def add_tag(
-        session,
-        name,
-        domain=None,
-        ):
-    """Add a single tag to storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises sqlalchemy.exc.IntegrityError: Tag already exists
-    """
-
-    s = None
-    if domain == None: 
-        s = text("INSERT INTO tag (value) VALUES (:b)")
-    else:
-        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
-    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/README

@@ -1 +0,0 @@
-Generic single-database configuration.

apps/cic-cache/cic_cache/db/migrations/default/versions/63b629f14a85_.py

@@ -1,52 +0,0 @@
-"""Base tables
-
-Revision ID: 63b629f14a85
-Revises: 
-Create Date: 2020-12-04 08:16:00.412189
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '63b629f14a85'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tx',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('date_registered', sa.DateTime, nullable=False, server_default=sa.func.current_timestamp()),
-            sa.Column('block_number', sa.Integer, nullable=False),
-            sa.Column('tx_index', sa.Integer, nullable=False),
-            sa.Column('tx_hash', sa.String(66), nullable=False),
-            sa.Column('sender', sa.String(42), nullable=False),
-            sa.Column('recipient', sa.String(42), nullable=False),
-            sa.Column('source_token', sa.String(42), nullable=False),
-            sa.Column('destination_token', sa.String(42), nullable=False),
-            sa.Column('success', sa.Boolean, nullable=False),
-            sa.Column('from_value', sa.NUMERIC(), nullable=False),
-            sa.Column('to_value', sa.NUMERIC(), nullable=False),
-            sa.Column('date_block', sa.DateTime, nullable=False),
-            )
-    op.create_table(
-            'tx_sync',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tx', sa.String(66), nullable=False),
-            )
-
-    op.execute("INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');")
-
-    op.create_index('sender_token_idx', 'tx', ['sender', 'source_token'])
-    op.create_index('recipient_token_idx', 'tx', ['recipient', 'destination_token'])
-            
-
-def downgrade():
-    op.drop_index('recipient_token_idx')
-    op.drop_index('sender_token_idx')
-    op.drop_table('tx_sync')
-    op.drop_table('tx')

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -1,28 +0,0 @@
-"""Add chain syncer
-
-Revision ID: 6604de4203e2
-Revises: 63b629f14a85
-Create Date: 2021-04-01 08:10:29.156243
-
-"""
-from alembic import op
-import sqlalchemy as sa
-from chainsyncer.db.migrations.sqlalchemy import (
-        chainsyncer_upgrade,
-        chainsyncer_downgrade,
-        )
-
-
-# revision identifiers, used by Alembic.
-revision = '6604de4203e2'
-down_revision = '63b629f14a85'
-branch_labels = None
-depends_on = None
-
-def upgrade():
-    chainsyncer_upgrade(0, 0, 1)
-
-
-def downgrade():
-    chainsyncer_downgrade(0, 0, 1)
-

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -1,38 +0,0 @@
-"""Transaction tags
-
-Revision ID: aaf2bdce7d6e
-Revises: 6604de4203e2
-Create Date: 2021-05-01 09:20:20.775082
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'aaf2bdce7d6e'
-down_revision = '6604de4203e2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tag',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('domain', sa.String(), nullable=True),
-            sa.Column('value', sa.String(), nullable=False),
-            )
-    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
-
-    op.create_table(
-            'tag_tx_link',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
-            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
-                )
-
-def downgrade():
-    op.drop_table('tag_tx_link')
-    op.drop_index('idx_tag_domain_value')
-    op.drop_table('tag')

apps/cic-cache/cic_cache/db/models/base.py

@@ -1,102 +0,0 @@
-# stanard imports
-import logging
-
-# third-party imports
-from sqlalchemy import Column, Integer
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-logg = logging.getLogger()
-
-Model = declarative_base(name='Model')
-
-
-class SessionBase(Model):
-    """The base object for all SQLAlchemy enabled models. All other models must extend this.
-    """
-    __abstract__ = True
- 
-    id = Column(Integer, primary_key=True)
-
-    engine = None
-    """Database connection engine of the running aplication"""
-    sessionmaker = None
-    """Factory object responsible for creating sessions from the  connection pool"""
-    transactional = True
-    """Whether the database backend supports query transactions. Should be explicitly set by initialization code"""
-    poolable = True
-    """Whether the database backend supports connection pools. Should be explicitly set by initialization code"""
-    procedural = True
-    """Whether the database backend supports stored procedures"""
-    localsessions = {}
-    """Contains dictionary of sessions initiated by db model components"""
-
-
-    @staticmethod
-    def create_session():
-        """Creates a new database session.
-        """
-        return SessionBase.sessionmaker()
-
-
-    @staticmethod
-    def _set_engine(engine):
-        """Sets the database engine static property
-        """
-        SessionBase.engine = engine
-        SessionBase.sessionmaker = sessionmaker(bind=SessionBase.engine)
-
-
-    @staticmethod
-    def connect(dsn, debug=False):
-        """Create new database connection engine and connect to database backend.
-
-        :param dsn: DSN string defining connection.
-        :type dsn: str
-        """
-        e = None
-        if SessionBase.poolable:
-            e = create_engine(
-                    dsn,
-                    max_overflow=50,
-                    pool_pre_ping=True,
-                    pool_size=20,
-                    pool_recycle=10,
-                    echo=debug,
-                )
-        else:
-            e = create_engine(
-                    dsn,
-                    echo=debug,
-                    )
-
-        SessionBase._set_engine(e)
-
-
-    @staticmethod
-    def disconnect():
-        """Disconnect from database and free resources.
-        """
-        SessionBase.engine.dispose()
-        SessionBase.engine = None
-
-
-    @staticmethod
-    def bind_session(session=None):
-        localsession = session
-        if localsession == None:
-            localsession = SessionBase.create_session()
-            localsession_key = str(id(localsession))
-            logg.debug('creating new session {}'.format(localsession_key))
-            SessionBase.localsessions[localsession_key] = localsession
-        return localsession
-
-
-    @staticmethod
-    def release_session(session=None):
-        session_key = str(id(session))
-        if SessionBase.localsessions.get(session_key) != None:
-            logg.debug('destroying session {}'.format(session_key))
-            session.commit()
-            session.close()

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1,2 +0,0 @@
-from .erc20 import *
-from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,27 +0,0 @@
-class TagSyncFilter:
-    """Holds tag name and domain for an implementing filter.
-
-    :param name: Tag value 
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    """
-
-    def __init__(self, name, domain=None):
-        self.tag_name = name
-        self.tag_domain = domain
-
-
-    def tag(self):
-        """Return tag value/domain.
-
-        :rtype: Tuple
-        :returns: tag value/domain.
-        """
-        return (self.tag_name, self.tag_domain)
-
-
-    def __str__(self):
-        if self.tag_domain == None:
-            return self.tag_name
-        return '{}.{}'.format(self.tag_domain, self.tag_name)

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -1,83 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.eth.address import (
-        to_checksum_address,
-        )
-from chainlib.eth.error import RequestMismatchException
-from chainlib.status import Status
-from cic_eth_registry.erc20 import ERC20Token
-from cic_eth_registry.error import (
-        NotAContractError,
-        ContractMismatchError,
-        )
-from eth_erc20 import ERC20
-
-# local imports
-from .base import TagSyncFilter
-from cic_cache import db as cic_cache_db
-
-logg = logging.getLogger().getChild(__name__)
-
-
-class ERC20TransferFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec):
-        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
-        self.chain_spec = chain_spec
-
-
-    # TODO: Verify token in declarator / token index
-    def filter(self, conn, block, tx, db_session=None):
-        logg.debug('filter {} {}'.format(block, tx))
-        token = None
-        try:
-            token = ERC20Token(self.chain_spec, conn, tx.inputs[0])
-        except NotAContractError:
-            logg.debug('not a contract {}'.format(tx.inputs[0]))
-            return False
-        except ContractMismatchError:
-            logg.debug('not an erc20 token  {}'.format(tx.inputs[0]))
-            return False
-
-        transfer_data = None
-        try:
-            transfer_data = ERC20.parse_transfer_request(tx.payload)
-        except RequestMismatchException:
-            logg.debug('erc20 match but not a transfer, skipping')
-            return False
-        except ValueError:
-            logg.debug('erc20 match but bogus data, skipping')
-            return False
-
-        token_sender = tx.outputs[0]
-        token_recipient = transfer_data[0]
-        token_value = transfer_data[1]
-
-        logg.debug('matched erc20 token transfer {} ({}) to {} value {}'.format(token.name, token.address, transfer_data[0], transfer_data[1]))
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token.address,
-                token.address,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -1,73 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from erc20_faucet import Faucet
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.status import Status
-from hexathon import strip_0x
-
-# local imports
-import cic_cache.db as cic_cache_db
-from .base import TagSyncFilter
-
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
-
-
-class FaucetFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
-        super(FaucetFilter, self).__init__('give_to', domain='faucet')
-        self.chain_spec = chain_spec
-        self.sender_address = sender_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        try:
-            data = strip_0x(tx.payload)
-        except ValueError:
-            return False
-        logg.debug('data {}'.format(data))
-        if Faucet.method_for(data[:8]) == None:
-            return False
-
-        token_sender = tx.inputs[0]
-        token_recipient = data[64+8-40:]
-        logg.debug('token recipient {}'.format(token_recipient))
- 
-        f = Faucet(self.chain_spec)
-        o = f.token(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token = f.parse_token(r)
-
-        f = Faucet(self.chain_spec)
-        o = f.token_amount(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token_value = f.parse_token_amount(r)
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token,
-                token,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -1,110 +0,0 @@
-# standard imports
-import logging
-import json
-import re
-import base64
-
-# local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-    )
-
-logg = logging.getLogger(__name__)
-
-re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
-re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
-
-DEFAULT_LIMIT = 100
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    address = r[1]
-    if r[2] == None:
-        address = '0x' + address
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 2:
-        offset = r[3]
-    limit = 0
-    if r.lastindex > 3:
-        limit = r[4]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    offset = DEFAULT_LIMIT
-    if r.lastindex > 0:
-        offset = r[1]
-    limit = 0
-    if r.lastindex > 1:
-        limit = r[2]
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_data(session, env):
-    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-        return None
-
-    offset = r[1]
-    end = r[2]
-    if int(r[2]) < int(r[1]):
-        raise ValueError('cart before the horse, dude')
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,83 +0,0 @@
-# standard imports
-import os
-import logging
-import argparse
-import base64
-
-# external imports
-import confini
-
-# local imports
-from cic_cache.db import dsn_from_config
-from cic_cache.db.models.base import SessionBase
-from cic_cache.runnable.daemons.query import (
-        process_transactions_account_bloom,
-        process_transactions_all_bloom,
-        process_transactions_all_data,
-        )
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
-
-
-# uwsgi application
-def application(env, start_response):
-
-    headers = []
-    content = b''
-
-    session = SessionBase.create_session()
-    for handler in [
-            process_transactions_all_data,
-            process_transactions_all_bloom,
-            process_transactions_account_bloom,
-            ]:
-        r = None
-        try:
-            r = handler(session, env)
-        except ValueError as e:
-            start_response('400 {}'.format(str(e)))
-            return []
-        if r != None:
-            (mime_type, content) = r
-            break
-    session.close()
-
-    headers.append(('Content-Length', str(len(content))),)
-    headers.append(('Access-Control-Allow-Origin', '*',));
-
-    if len(content) == 0:
-        headers.append(('Content-Type', 'text/plain, charset=UTF-8',))
-        start_response('404 Looked everywhere, sorry', headers)
-    else:
-        headers.append(('Content-Type', mime_type,))
-        start_response('200 OK', headers)
-
-    return [content]

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -1,98 +0,0 @@
-# standard imports
-import logging
-import os
-import sys
-import argparse
-
-# third-party imports
-import celery
-import confini
-
-# local imports
-from cic_cache.db import dsn_from_config
-from cic_cache.db.models.base import SessionBase
-from cic_cache.tasks.tx import *
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-
-# connect to database
-dsn = dsn_from_config(config)
-SessionBase.connect(dsn)
-
-# verify database connection with minimal sanity query
-#session = SessionBase.create_session()
-#session.execute('select version_num from alembic_version')
-#session.close()
-
-# set up celery
-current_app = celery.Celery(__name__)
-
-broker = config.get('CELERY_BROKER_URL')
-if broker[:4] == 'file':
-    bq = tempfile.mkdtemp()
-    bp = tempfile.mkdtemp()
-    current_app.conf.update({
-            'broker_url': broker,
-            'broker_transport_options': {
-                'data_folder_in': bq,
-                'data_folder_out': bq,
-                'data_folder_processed': bp,
-            },
-            },
-            )
-    logg.warning('celery broker dirs queue i/o {} processed {}, will NOT be deleted on shutdown'.format(bq, bp))
-else:
-    current_app.conf.update({
-        'broker_url': broker,
-        })
-
-result = config.get('CELERY_RESULT_URL')
-if result[:4] == 'file':
-    rq = tempfile.mkdtemp()
-    current_app.conf.update({
-        'result_backend': 'file://{}'.format(rq),
-        })
-    logg.warning('celery backend store dir {} created, will NOT be deleted on shutdown'.format(rq))
-else:
-    current_app.conf.update({
-        'result_backend': result,
-        })
-
-
-def main():
-    argv = ['worker']
-    if args.vv:
-        argv.append('--loglevel=DEBUG')
-    elif args.v:
-        argv.append('--loglevel=INFO')
-    argv.append('-Q')
-    argv.append(args.q)
-    argv.append('-n')
-    argv.append(args.q)
-
-    current_app.worker_main(argv)
-
-
-if __name__ == '__main__':
-    main()

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -1,152 +0,0 @@
-# standard imports
-import os
-import sys
-import logging
-import time
-import argparse
-import sys
-import re
-
-# external imports
-import confini
-import celery
-import sqlalchemy
-import rlp
-import cic_base.config
-import cic_base.log
-import cic_base.argparse
-import cic_base.rpc
-from cic_base.eth.syncer import chain_interface
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.error import UnknownContractError
-from chainlib.chain import ChainSpec
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.connection import RPCConnection
-from chainlib.eth.block import (
-        block_latest,
-        )
-from hexathon import (
-        strip_0x,
-        )
-from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
-from chainsyncer.db.models.base import SessionBase
-
-# local imports
-from cic_cache.db import (
-        dsn_from_config,
-        add_tag,
-        )
-from cic_cache.runnable.daemons.filters import (
-        ERC20TransferFilter,
-        FaucetFilter,
-        )
-
-script_dir = os.path.realpath(os.path.dirname(__file__))
-
-def add_block_args(argparser):
-    argparser.add_argument('--history-start', type=int, default=0, dest='history_start', help='Start block height for initial history sync')
-    argparser.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-    return argparser
-
-
-logg = cic_base.log.create()
-argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
-argparser = cic_base.argparse.add(argparser, add_block_args, 'block')
-args = cic_base.argparse.parse(argparser, logg)
-config = cic_base.config.create(args.c, args, args.env_prefix)
-
-config.add(args.history_start, 'SYNCER_HISTORY_START', True)
-config.add(args.no_history, '_NO_HISTORY', True)
-
-cic_base.config.log(config)
-
-dsn = dsn_from_config(config)
-
-SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
-
-chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
-
-cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
-
-
-def register_filter_tags(filters, session):
-    for f in filters:
-        tag = f.tag()
-        try:
-            add_tag(session, tag[0], domain=tag[1])
-            session.commit()
-            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-        except sqlalchemy.exc.IntegrityError:
-            session.rollback()
-            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-
-
-def main():
-    # Connect to blockchain with chainlib
-    rpc = RPCConnection.connect(chain_spec, 'default')
-
-    o = block_latest()
-    r = rpc.do(o)
-    block_offset = int(strip_0x(r), 16) + 1
-
-    logg.debug('current block height {}'.format(block_offset))
-
-    syncers = []
-
-    #if SQLBackend.first(chain_spec):
-    #    backend = SQLBackend.initial(chain_spec, block_offset)
-    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
-
-    if len(syncer_backends) == 0:
-        initial_block_start = config.get('SYNCER_HISTORY_START')
-        initial_block_offset = block_offset
-        if config.get('_NO_HISTORY'):
-            initial_block_start = block_offset
-            initial_block_offset += 1
-        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
-        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
-    else:
-        for syncer_backend in syncer_backends:
-            logg.info('resuming sync session {}'.format(syncer_backend))
-
-    for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, chain_interface))
-
-    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, chain_interface))
-
-    trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
-    if trusted_addresses_src == None:
-        logg.critical('At least one trusted address must be declared in CIC_TRUST_ADDRESS')
-        sys.exit(1)
-    trusted_addresses = trusted_addresses_src.split(',')
-    for address in trusted_addresses:
-        logg.info('using trusted address {}'.format(address))
-
-    erc20_transfer_filter = ERC20TransferFilter(chain_spec)
-    faucet_filter = FaucetFilter(chain_spec)
-
-    filters = [
-        erc20_transfer_filter,
-        faucet_filter,
-            ]
-
-    session = SessionBase.create_session()
-    register_filter_tags(filters, session)
-    session.close()
-
-    i = 0
-    for syncer in syncers:
-        logg.debug('running syncer index {}'.format(i))
-        for f in filters:
-            syncer.add_filter(f)
-        r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
-        sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
-
-        i += 1
-
-
-if __name__ == '__main__':
-    main()

apps/cic-cache/cic_cache/tasks/tx.py

@@ -1,38 +0,0 @@
-# third-party imports
-import celery
-
-# local imports
-from cic_cache.cache import BloomCache
-from cic_cache.db.models.base import SessionBase
-
-celery_app = celery.current_app
-
-
-@celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, encoding='hex'):
-    queue = self.request.delivery_info.get('routing_key')
-
-    session = SessionBase.create_session()
-
-    c = BloomCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-    else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    session.close()
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': bloom_filter_block.hex(), 
-        'blocktx_filter': bloom_filter_tx.hex(),
-        'filter_rounds': 3,
-            }
-
-    return o
-
-
-

apps/cic-cache/cic_cache/version.py

@@ -1,18 +0,0 @@
-import os
-import semver
-
-version = (
-        0,
-        2,
-        0,
-        'alpha.2',
-        )
-
-version_object = semver.VersionInfo(
-        major=version[0],
-        minor=version[1],
-        patch=version[2],
-        prerelease=version[3],
-        )
-
-version_string = str(version_object)

apps/cic-cache/config/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis:///
-result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec =
-trust_address = 

apps/cic-cache/config/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/bancor.ini

@@ -1,3 +0,0 @@
-[bancor]
-registry_address = 
-dir = /usr/local/share/bancor

apps/cic-cache/config/docker/celery.ini

@@ -1,3 +0,0 @@
-[celery]
-broker_url = redis://localhost:63379
-result_url = redis://localhost:63379

apps/cic-cache/config/docker/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-chain_spec =
-registry_address =  
-trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic_cache
-USER=grassroots
-PASSWORD=
-HOST=localhost
-PORT=63432
-ENGINE=postgresql
-DRIVER=psycopg2
-DEBUG=0

apps/cic-cache/config/docker/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = http://localhost:63545

apps/cic-cache/config/docker/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 1
-history_start = 0

apps/cic-cache/config/eth.ini

@@ -1,2 +0,0 @@
-[eth]
-provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -1,3 +0,0 @@
-[syncer]
-loop_interval = 5
-history_start = 0

apps/cic-cache/config/test/bancor.ini

@@ -1,2 +0,0 @@
-[bancor]
-dir =

apps/cic-cache/config/test/cic.ini

@@ -1,4 +0,0 @@
-[cic]
-registry_address = 
-chain_spec = 
-trust_address =

apps/cic-cache/config/test/database.ini

@@ -1,9 +0,0 @@
-[database]
-NAME=cic-cache-test
-USER=postgres
-PASSWORD=
-HOST=localhost
-PORT=5432
-ENGINE=sqlite
-DRIVER=pysqlite
-DEBUG=1

apps/cic-cache/config/test/eth.ini

@@ -1,5 +0,0 @@
-[eth]
-#ws_provider = ws://localhost:8546
-#ttp_provider = http://localhost:8545
-provider = http://localhost:8545
-#chain_id = 

apps/cic-cache/config/test/syncer.ini

@@ -1,2 +0,0 @@
-[syncer]
-loop_interval = 1

apps/cic-cache/db/psycopg2/db.sql

@@ -1,22 +0,0 @@
-CREATE TABLE tx (
-	id SERIAL PRIMARY KEY,
-	date_registered TIMESTAMP NOT NULL default CURRENT_TIMESTAMP,
-	block_number INTEGER NOT NULL,
-	tx_index INTEGER NOT NULL,
-	tx_hash VARCHAR(66) NOT NULL,
-	sender VARCHAR(42) NOT NULL, 
-	recipient VARCHAR(42) NOT NULL, 
-	source_token VARCHAR(42) NOT NULL,
-	destination_token VARCHAR(42) NOT NULL,
-	from_value BIGINT NOT NULL,
-	to_value BIGINT NOT NULL,
-	success BOOLEAN NOT NULL, 
-	date_block TIMESTAMP NOT NULL
-);
-
-CREATE TABLE tx_sync (
-	id SERIAL PRIMARY KEY,
-	tx VARCHAR(66) NOT NULL
-);
-
-INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');

apps/cic-cache/db/pysqlite/db.sql

@@ -1,23 +0,0 @@
-CREATE TABLE tx (
-	id SERIAL PRIMARY KEY,
-	date_registered DATETIME NOT NULL default CURRENT_DATE, 
-	block_number INTEGER NOT NULL,
-	tx_index INTEGER NOT NULL,
-	tx_hash VARCHAR(66) NOT NULL,
-	sender VARCHAR(42) NOT NULL, 
-	recipient VARCHAR(42) NOT NULL, 
-	source_token VARCHAR(42) NOT NULL,
-	destination_token VARCHAR(42) NOT NULL,
-	from_value INTEGER NOT NULL,
-	to_value INTEGER NOT NULL,
-	success BOOLEAN NOT NULL, 
-	date_block DATETIME NOT NULL,
-	CHECK (success IN (0, 1))
-);
-
-CREATE TABLE tx_sync (
-	id SERIAL PRIMARY_KEY,
-	tx VARCHAR(66) NOT NULL
-);
-
-INSERT INTO tx_sync (tx) VALUES('0x0000000000000000000000000000000000000000000000000000000000000000');

apps/cic-cache/doc/openapi/server.yml

@@ -1,102 +0,0 @@
-openapi: "3.0.3"
-info:
-  title: Grassroots Economics CIC Cache
-  description: Cache of processed transaction data from Ethereum blockchain and worker queues
-  termsOfService: bzz://grassrootseconomics.eth/terms
-  contact:
-    name: Grassroots Economics
-    url: https://www.grassrootseconomics.org
-    email: will@grassecon.org
-  license:
-    name: GPLv3
-  version: 0.1.0
-
-paths:
-  /tx/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-
-    parameters:
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
-    - name: limit
-      in: path
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}/{offset}/{limit}:
-    description: Bloom filter for batch of latest transactions by account
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: offset
-      in: path
-      schema:
-        type: integer
-        format: int32
-    - name: limit
-      in: path
-      schema:
-        type: integer
-        format: int32
-
-components:
-  schemas:
-    BlocksBloom:
-      type: object
-      properties:
-        low:
-          type: int
-          format: int32
-          description: The lowest block number included in the filter
-        block_filter:
-          type: string
-          format: byte
-          description: Block number filter
-        blocktx_filter:
-          type: string
-          format: byte
-          description: Block and tx index filter
-        alg:
-          type: string
-          description: Hashing algorithm (currently only using sha256)
-        filter_rounds:
-          type: int
-          format: int32
-          description: Number of hash rounds used to create the filter

apps/cic-cache/docker/Dockerfile

@@ -1,52 +0,0 @@
-FROM python:3.8.6-slim-buster
-
-#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
-
-WORKDIR /usr/src/cic-cache
-
-ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
-ARG root_requirement_file='requirements.txt'
-
-#RUN apk update && \
-#	apk add gcc musl-dev gnupg libpq
-#RUN apk add postgresql-dev
-#RUN apk add linux-headers
-#RUN apk add libffi-dev
-RUN apt-get update && \
-	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
-
-# Copy shared requirements from top of mono-repo
-RUN echo "copying root req file ${root_requirement_file}"
-RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
-
-COPY cic-cache/requirements.txt ./
-COPY cic-cache/setup.cfg \
-	 cic-cache/setup.py \
- 	 ./
-COPY cic-cache/cic_cache/ ./cic_cache/
-COPY cic-cache/scripts/ ./scripts/
-COPY cic-cache/test_requirements.txt ./
-RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
-RUN pip install $pip_extra_index_url_flag .
-RUN pip install .[server]
-
-COPY cic-cache/tests/ ./tests/
-#COPY db/ cic-cache/db
-#RUN apk add postgresql-client
-
-# ini files in config directory defines the configurable parameters for the application
-# they can all be overridden by environment variables
-# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
-COPY cic-cache/config/ /usr/local/etc/cic-cache/
-
-# for db migrations
-RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
-COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
-
-COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
-COPY cic-cache/docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
-# Tracker
-# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
-# Server
-# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]

apps/cic-cache/docker/db.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
->&2 echo executing database migration
-python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
-set +e

apps/cic-cache/docker/start_tracker.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. ./db.sh
-
-if [ $? -ne "0" ]; then
-	>&2 echo db migrate fail
-	exit 1
-fi
-
-/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/examples/bloom_client/parse.js

@@ -1,40 +0,0 @@
-let xmlhttprequest = require('xhr2');
-let moolb = require('moolb');
-
-let xhr = new xmlhttprequest();
-xhr.responseType = 'json';
-xhr.open('GET', 'http://localhost:5555/tx/0/100');
-xhr.addEventListener('load', (e) => {
-
-	d = xhr.response;
-
-	b_one = Buffer.from(d.block_filter, 'base64');
-	b_two = Buffer.from(d.blocktx_filter, 'base64');
-
-	for (let i = 0; i < 8192; i++) {
-		if (b_two[i] > 0) {
-			console.debug('value on', i, b_two[i]);
-		}
-	}
-	console.log(b_one, b_two);
-
-	let f_block = moolb.fromBytes(b_one, d.filter_rounds);
-	let f_blocktx = moolb.fromBytes(b_two, d.filter_rounds);
-	let a = new ArrayBuffer(8);
-	let w = new DataView(a);
-	for (let i = 410000; i < 430000; i++) {
-		w.setInt32(0, i);
-		let r = new Uint8Array(a.slice(0, 4));
-		if (f_block.check(r)) {
-			for (let j = 0; j < 200; j++) {
-				w = new DataView(a);
-				w.setInt32(4, j);
-				r = new Uint8Array(a);
-				if (f_blocktx.check(r)) {
-					console.log('true', i, j);
-				}
-			}
-		}
-	}
-});
-let r = xhr.send();

apps/cic-cache/requirements.txt

@@ -1,14 +0,0 @@
-cic-base==0.1.3a3+build.984b5cff
-alembic==1.4.2
-confini~=0.3.6rc3
-uwsgi==2.0.19.1
-moolb~=0.1.0
-cic-eth-registry~=0.5.6a1
-SQLAlchemy==1.3.20
-semver==2.13.0
-psycopg2==2.8.6
-celery==4.4.7
-redis==3.5.3
-rlp==2.0.1
-chainsyncer[sql]~=0.0.3a3
-erc20-faucet~=0.2.2a1

apps/cic-cache/scripts/migrate.py

@@ -1,65 +0,0 @@
-#!/usr/bin/python
-import os
-import argparse
-import logging
-import re
-
-import alembic
-from alembic.config import Config as AlembicConfig
-import confini
-
-from cic_cache.db import dsn_from_config
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-# BUG: the dbdir doesn't work after script install
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-migrationsdir = os.path.join(dbdir, 'migrations')
-
-config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-argparser = argparse.ArgumentParser()
-argparser.add_argument('-c', type=str, default=config_dir, help='config file')
-argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', action='store_true', help='force action')
-argparser.add_argument('-v', action='store_true', help='be verbose')
-argparser.add_argument('-vv', action='store_true', help='be more verbose')
-args = argparser.parse_args()
-
-if args.vv:
-    logging.getLogger().setLevel(logging.DEBUG)
-elif args.v:
-    logging.getLogger().setLevel(logging.INFO)
-
-config = confini.Config(args.c, args.env_prefix)
-config.process()
-config.censor('PASSWORD', 'DATABASE')
-config.censor('PASSWORD', 'SSL')
-logg.debug('config:\n{}'.format(config))
-
-migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
-if not os.path.isdir(migrations_dir):
-    logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
-    migrations_dir = os.path.join(args.migrations_dir, 'default')
-
-# connect to database
-dsn = dsn_from_config(config)
-
-
-logg.info('using migrations dir {}'.format(migrations_dir))
-logg.info('using db {}'.format(dsn))
-ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
-ac.set_main_option('sqlalchemy.url', dsn)
-ac.set_main_option('script_location', migrations_dir)
-
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
-alembic.command.upgrade(ac, 'head')