Add phone pointer and ussd import verifiers

.gitignore

@@ -1,17 +1,2 @@
 service-configs/*
 !service-configs/.gitkeep
-**/node_modules/
-__pycache__
-*.pyc
-*.o
-gmon.out
-*.egg-info
-dist/
-build/
-**/*sqlite
-**/.nyc_output
-**/coverage
-**/.venv
-.idea
-**/.vim
-**/*secret.yaml

.gitlab-ci.yml

@@ -1,81 +1,13 @@
 include:
-  #- local: 'ci_templates/.cic-template.yml' #kaniko build templates
-  # these includes are app specific unit tests
+  - local: 'ci_templates/.cic-template.yml'
+  - local: 'apps/contract-migration/.gitlab-ci.yml'
   - local: 'apps/cic-eth/.gitlab-ci.yml'
   - local: 'apps/cic-ussd/.gitlab-ci.yml'
   - local: 'apps/cic-notify/.gitlab-ci.yml'
   - local: 'apps/cic-meta/.gitlab-ci.yml'
   - local: 'apps/cic-cache/.gitlab-ci.yml'
-    #- local: 'apps/contract-migration/.gitlab-ci.yml'
-    #- local: 'apps/data-seeding/.gitlab-ci.yml'
 
 stages:
-  - version
   - build
   - test
-  - deploy
-
-image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
-
-variables:
-  DOCKER_BUILDKIT: "1"
-  COMPOSE_DOCKER_CLI_BUILD: "1"
-  CI_DEBUG_TRACE: "true"
-  SEMVERBOT_VERSION: "0.2.0"
-
-  #before_script:
-  #  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-
-version:
-  #image: python:3.7-stretch
-  image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae 
-  stage: version
-  tags:
-    - integration
-  script:
-    - mkdir -p ~/.ssh && chmod 700 ~/.ssh
-    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
-    - eval $(ssh-agent -s)
-    - ssh-add <(echo "$SSH_PRIVATE_KEY")
-    - git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
-    - export TAG=$(sbot predict version -m auto)
-    - |
-      if [[ -z $TAG ]]
-      then
-        echo "tag could not be set $@"
-        exit 1
-      fi
-    - echo $TAG > version
-    - git tag -a v$TAG -m "ci tagged"
-    - git push origin v$TAG
-  artifacts:
-    paths:
-      - version
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
-
-# runs on protected branches and pushes to repo
-build-push:
-  stage: build
-  tags:
-    - integration
-  #script:
-  #  - TAG=$CI_Cbefore_script:
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  script:
-    - TAG=latest ./scripts/build-push.sh
-    - TAG=$(cat ./version) ./scripts/build-push.sh
-  rules:
-  - if: $CI_COMMIT_REF_PROTECTED == "true"
-    when: always
-  - if: $CI_COMMIT_REF_NAME == "master"
-    when: always
-
-deploy-dev:
-  stage: deploy
-  trigger: grassrootseconomics/devops
-  when: manual
+  - release

.gitlab/issue_templates/bug.md

@@ -1,44 +0,0 @@
-<!---
-Please read this!
-
-Before opening a new issue, make sure to search for keywords in the issues
-filtered by the "bug" label:
-
-- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
-
-and verify the issue you're about to submit isn't a duplicate.
---->
-
-### Summary
-
-<!-- Summarize the bug encountered concisely. -->
-
-### Steps to reproduce
-
-<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
-
-### Example Project
-
-<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic 
-behavior, and link to it here in the bug report. If you are using an older version of GitLab, this 
-will also determine whether the bug is fixed in a more recent version. -->
-
-### What is the current *bug* behavior?
-
-<!-- Describe what actually happens. -->
-
-### What is the expected *correct* behavior?
-
-<!-- Describe what you should see instead. -->
-
-### Relevant logs and/or screenshots
-
-<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
- as it's tough to read otherwise. -->
-
-
-### Possible fixes
-
-<!-- If you can, link to the line of code that might be responsible for the problem. -->
-
-/label ~"bug"

.semverbot.toml

@@ -1,16 +0,0 @@
-[git]
-
-[git.config]
-email = "semverbot@grassroots.org"
-name = "semvervot"
-
-[git.tags]
-prefix = "v"
-
-[semver]
-mode = "git-commit"
-
-[semver.detection]
-patch = ["fix", "[fix]", "patch", "[patch]"]
-minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
-major = ["BREAKING CHANGE"]

CODE_OF_CONDUCT.md

@@ -1,83 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-
-
-
-
-...Try to keep in mind the immortal
-words of Bill and Ted, "Be excellent to each other."
-

CONTRIBUTING.md

@@ -1,16 +0,0 @@
-Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider. 
-
-__To request a change to the code please fork this repository and sumbit a merge request.__
-
-__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
-
-__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
-
-__Ask a question in our dev chat:__
-
-[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
-
-[Discord](https://discord.gg/XWunwAsX)
-
-[Matrix, IRC soon?]
-

CONTRIBUTING_CORE.md

@@ -1,117 +0,0 @@
-# CORE TEAM CONTRIBUTION GUIDE
-
-# 1. Transparency
-
-1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
-
-1.2 A work log SHOULD be submitted after a "unit of work" is complete.
-
-1.2.1 A "unit of work" should not span more than one full day's worth of work.
-
-1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
-
-1.3 Individual logs are reviewed in weekly meetings
-
-<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
-
-```
-- Core
-	* fixed foo
-	* fixed bar
-- Frontend
-	* connected bar to baz
-
-```-->
-
-1.4 Work log format is defined in []()
-
-1.5 Link to issue/MR in bullet point where appropriate
-
-1.6 
-
-
-# 2. Code hygiene
-
-2.1 Keep function names and variable names short
-
-2.2 Keep code files, functions and test fixtures short
-
-2.3 The less magic the better. Recombinable and replaceable is king
-
-2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
-
-2.5 Only auto-import when necessary, and always with a minimum of side-effects
-
-2.6 Use custom errors. Let them bubble up
-
-2.7 No logs in tight loops
-
-2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
-
-2.9 Test coverage MUST be kept higher than 90% after changes
-
-2.10 Docstrings. Always. Always!
-
-
-# 3. Versioning
-
-3.1 Use [Semantic Versioning](https://semver.org/)
-
-3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
-
-
-# 4. Issues
-
-4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
-
-4.2 Issues need proper problem statement
-
-4.2.1. What is the current state
-
-4.2.2. If current state is not behaving as expected, what was the expected state
-
-4.2.3. What is the desired new state.
-
-4.3  Issues need proper resolution statement
-
-4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
-		
-4.3.2. Builet point list of external resources informing the issue and resolution
-
-4.4 Tasks needs to be appropriately labelled using GROUP labels.
-
-
-# 5. Code submission
-
-5.1 A branch and new MR is always created BEFORE THE WORK STARTS
-
-5.2 An MR should solve ONE SINGLE PART of a problem
-
-5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
-
-5.4 MRs should not be open for more than one week (during normal operation periods)
-
-5.5 MR should ideally not be longer than 400 lines of changes of logic
-
-5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
-
-
-# 6. Code reviews
-
-6.1 At least one peer review before merge
-
-6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
-
-6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
-
-6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
-
-6.5 If CI is not working, reviewer MUST make sure code builds and runs
-
-6.6 Behave!
-
-6.6.1 Don't be a jerk
-
-6.6.2 Don't block needlessly
-
-6.6.3 Say please

LICENSE

@@ -1,661 +0,0 @@
-                    GNU AFFERO GENERAL PUBLIC LICENSE
-                       Version 3, 19 November 2007
-
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The GNU Affero General Public License is a free, copyleft license for
-software and other kinds of works, specifically designed to ensure
-cooperation with the community in the case of network server software.
-
-  The licenses for most software and other practical works are designed
-to take away your freedom to share and change the works.  By contrast,
-our General Public Licenses are intended to guarantee your freedom to
-share and change all versions of a program--to make sure it remains free
-software for all its users.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-them if you wish), that you receive source code or can get it if you
-want it, that you can change the software or use pieces of it in new
-free programs, and that you know you can do these things.
-
-  Developers that use our General Public Licenses protect your rights
-with two steps: (1) assert copyright on the software, and (2) offer
-you this License which gives you legal permission to copy, distribute
-and/or modify the software.
-
-  A secondary benefit of defending all users' freedom is that
-improvements made in alternate versions of the program, if they
-receive widespread use, become available for other developers to
-incorporate.  Many developers of free software are heartened and
-encouraged by the resulting cooperation.  However, in the case of
-software used on network servers, this result may fail to come about.
-The GNU General Public License permits making a modified version and
-letting the public access it on a server without ever releasing its
-source code to the public.
-
-  The GNU Affero General Public License is designed specifically to
-ensure that, in such cases, the modified source code becomes available
-to the community.  It requires the operator of a network server to
-provide the source code of the modified version running there to the
-users of that server.  Therefore, public use of a modified version, on
-a publicly accessible server, gives the public access to the source
-code of the modified version.
-
-  An older license, called the Affero General Public License and
-published by Affero, was designed to accomplish similar goals.  This is
-a different license, not a version of the Affero GPL, but Affero has
-released a new version of the Affero GPL which permits relicensing under
-this license.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                       TERMS AND CONDITIONS
-
-  0. Definitions.
-
-  "This License" refers to version 3 of the GNU Affero General Public License.
-
-  "Copyright" also means copyright-like laws that apply to other kinds of
-works, such as semiconductor masks.
-
-  "The Program" refers to any copyrightable work licensed under this
-License.  Each licensee is addressed as "you".  "Licensees" and
-"recipients" may be individuals or organizations.
-
-  To "modify" a work means to copy from or adapt all or part of the work
-in a fashion requiring copyright permission, other than the making of an
-exact copy.  The resulting work is called a "modified version" of the
-earlier work or a work "based on" the earlier work.
-
-  A "covered work" means either the unmodified Program or a work based
-on the Program.
-
-  To "propagate" a work means to do anything with it that, without
-permission, would make you directly or secondarily liable for
-infringement under applicable copyright law, except executing it on a
-computer or modifying a private copy.  Propagation includes copying,
-distribution (with or without modification), making available to the
-public, and in some countries other activities as well.
-
-  To "convey" a work means any kind of propagation that enables other
-parties to make or receive copies.  Mere interaction with a user through
-a computer network, with no transfer of a copy, is not conveying.
-
-  An interactive user interface displays "Appropriate Legal Notices"
-to the extent that it includes a convenient and prominently visible
-feature that (1) displays an appropriate copyright notice, and (2)
-tells the user that there is no warranty for the work (except to the
-extent that warranties are provided), that licensees may convey the
-work under this License, and how to view a copy of this License.  If
-the interface presents a list of user commands or options, such as a
-menu, a prominent item in the list meets this criterion.
-
-  1. Source Code.
-
-  The "source code" for a work means the preferred form of the work
-for making modifications to it.  "Object code" means any non-source
-form of a work.
-
-  A "Standard Interface" means an interface that either is an official
-standard defined by a recognized standards body, or, in the case of
-interfaces specified for a particular programming language, one that
-is widely used among developers working in that language.
-
-  The "System Libraries" of an executable work include anything, other
-than the work as a whole, that (a) is included in the normal form of
-packaging a Major Component, but which is not part of that Major
-Component, and (b) serves only to enable use of the work with that
-Major Component, or to implement a Standard Interface for which an
-implementation is available to the public in source code form.  A
-"Major Component", in this context, means a major essential component
-(kernel, window system, and so on) of the specific operating system
-(if any) on which the executable work runs, or a compiler used to
-produce the work, or an object code interpreter used to run it.
-
-  The "Corresponding Source" for a work in object code form means all
-the source code needed to generate, install, and (for an executable
-work) run the object code and to modify the work, including scripts to
-control those activities.  However, it does not include the work's
-System Libraries, or general-purpose tools or generally available free
-programs which are used unmodified in performing those activities but
-which are not part of the work.  For example, Corresponding Source
-includes interface definition files associated with source files for
-the work, and the source code for shared libraries and dynamically
-linked subprograms that the work is specifically designed to require,
-such as by intimate data communication or control flow between those
-subprograms and other parts of the work.
-
-  The Corresponding Source need not include anything that users
-can regenerate automatically from other parts of the Corresponding
-Source.
-
-  The Corresponding Source for a work in source code form is that
-same work.
-
-  2. Basic Permissions.
-
-  All rights granted under this License are granted for the term of
-copyright on the Program, and are irrevocable provided the stated
-conditions are met.  This License explicitly affirms your unlimited
-permission to run the unmodified Program.  The output from running a
-covered work is covered by this License only if the output, given its
-content, constitutes a covered work.  This License acknowledges your
-rights of fair use or other equivalent, as provided by copyright law.
-
-  You may make, run and propagate covered works that you do not
-convey, without conditions so long as your license otherwise remains
-in force.  You may convey covered works to others for the sole purpose
-of having them make modifications exclusively for you, or provide you
-with facilities for running those works, provided that you comply with
-the terms of this License in conveying all material for which you do
-not control copyright.  Those thus making or running the covered works
-for you must do so exclusively on your behalf, under your direction
-and control, on terms that prohibit them from making any copies of
-your copyrighted material outside their relationship with you.
-
-  Conveying under any other circumstances is permitted solely under
-the conditions stated below.  Sublicensing is not allowed; section 10
-makes it unnecessary.
-
-  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
-
-  No covered work shall be deemed part of an effective technological
-measure under any applicable law fulfilling obligations under article
-11 of the WIPO copyright treaty adopted on 20 December 1996, or
-similar laws prohibiting or restricting circumvention of such
-measures.
-
-  When you convey a covered work, you waive any legal power to forbid
-circumvention of technological measures to the extent such circumvention
-is effected by exercising rights under this License with respect to
-the covered work, and you disclaim any intention to limit operation or
-modification of the work as a means of enforcing, against the work's
-users, your or third parties' legal rights to forbid circumvention of
-technological measures.
-
-  4. Conveying Verbatim Copies.
-
-  You may convey verbatim copies of the Program's source code as you
-receive it, in any medium, provided that you conspicuously and
-appropriately publish on each copy an appropriate copyright notice;
-keep intact all notices stating that this License and any
-non-permissive terms added in accord with section 7 apply to the code;
-keep intact all notices of the absence of any warranty; and give all
-recipients a copy of this License along with the Program.
-
-  You may charge any price or no price for each copy that you convey,
-and you may offer support or warranty protection for a fee.
-
-  5. Conveying Modified Source Versions.
-
-  You may convey a work based on the Program, or the modifications to
-produce it from the Program, in the form of source code under the
-terms of section 4, provided that you also meet all of these conditions:
-
-    a) The work must carry prominent notices stating that you modified
-    it, and giving a relevant date.
-
-    b) The work must carry prominent notices stating that it is
-    released under this License and any conditions added under section
-    7.  This requirement modifies the requirement in section 4 to
-    "keep intact all notices".
-
-    c) You must license the entire work, as a whole, under this
-    License to anyone who comes into possession of a copy.  This
-    License will therefore apply, along with any applicable section 7
-    additional terms, to the whole of the work, and all its parts,
-    regardless of how they are packaged.  This License gives no
-    permission to license the work in any other way, but it does not
-    invalidate such permission if you have separately received it.
-
-    d) If the work has interactive user interfaces, each must display
-    Appropriate Legal Notices; however, if the Program has interactive
-    interfaces that do not display Appropriate Legal Notices, your
-    work need not make them do so.
-
-  A compilation of a covered work with other separate and independent
-works, which are not by their nature extensions of the covered work,
-and which are not combined with it such as to form a larger program,
-in or on a volume of a storage or distribution medium, is called an
-"aggregate" if the compilation and its resulting copyright are not
-used to limit the access or legal rights of the compilation's users
-beyond what the individual works permit.  Inclusion of a covered work
-in an aggregate does not cause this License to apply to the other
-parts of the aggregate.
-
-  6. Conveying Non-Source Forms.
-
-  You may convey a covered work in object code form under the terms
-of sections 4 and 5, provided that you also convey the
-machine-readable Corresponding Source under the terms of this License,
-in one of these ways:
-
-    a) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by the
-    Corresponding Source fixed on a durable physical medium
-    customarily used for software interchange.
-
-    b) Convey the object code in, or embodied in, a physical product
-    (including a physical distribution medium), accompanied by a
-    written offer, valid for at least three years and valid for as
-    long as you offer spare parts or customer support for that product
-    model, to give anyone who possesses the object code either (1) a
-    copy of the Corresponding Source for all the software in the
-    product that is covered by this License, on a durable physical
-    medium customarily used for software interchange, for a price no
-    more than your reasonable cost of physically performing this
-    conveying of source, or (2) access to copy the
-    Corresponding Source from a network server at no charge.
-
-    c) Convey individual copies of the object code with a copy of the
-    written offer to provide the Corresponding Source.  This
-    alternative is allowed only occasionally and noncommercially, and
-    only if you received the object code with such an offer, in accord
-    with subsection 6b.
-
-    d) Convey the object code by offering access from a designated
-    place (gratis or for a charge), and offer equivalent access to the
-    Corresponding Source in the same way through the same place at no
-    further charge.  You need not require recipients to copy the
-    Corresponding Source along with the object code.  If the place to
-    copy the object code is a network server, the Corresponding Source
-    may be on a different server (operated by you or a third party)
-    that supports equivalent copying facilities, provided you maintain
-    clear directions next to the object code saying where to find the
-    Corresponding Source.  Regardless of what server hosts the
-    Corresponding Source, you remain obligated to ensure that it is
-    available for as long as needed to satisfy these requirements.
-
-    e) Convey the object code using peer-to-peer transmission, provided
-    you inform other peers where the object code and Corresponding
-    Source of the work are being offered to the general public at no
-    charge under subsection 6d.
-
-  A separable portion of the object code, whose source code is excluded
-from the Corresponding Source as a System Library, need not be
-included in conveying the object code work.
-
-  A "User Product" is either (1) a "consumer product", which means any
-tangible personal property which is normally used for personal, family,
-or household purposes, or (2) anything designed or sold for incorporation
-into a dwelling.  In determining whether a product is a consumer product,
-doubtful cases shall be resolved in favor of coverage.  For a particular
-product received by a particular user, "normally used" refers to a
-typical or common use of that class of product, regardless of the status
-of the particular user or of the way in which the particular user
-actually uses, or expects or is expected to use, the product.  A product
-is a consumer product regardless of whether the product has substantial
-commercial, industrial or non-consumer uses, unless such uses represent
-the only significant mode of use of the product.
-
-  "Installation Information" for a User Product means any methods,
-procedures, authorization keys, or other information required to install
-and execute modified versions of a covered work in that User Product from
-a modified version of its Corresponding Source.  The information must
-suffice to ensure that the continued functioning of the modified object
-code is in no case prevented or interfered with solely because
-modification has been made.
-
-  If you convey an object code work under this section in, or with, or
-specifically for use in, a User Product, and the conveying occurs as
-part of a transaction in which the right of possession and use of the
-User Product is transferred to the recipient in perpetuity or for a
-fixed term (regardless of how the transaction is characterized), the
-Corresponding Source conveyed under this section must be accompanied
-by the Installation Information.  But this requirement does not apply
-if neither you nor any third party retains the ability to install
-modified object code on the User Product (for example, the work has
-been installed in ROM).
-
-  The requirement to provide Installation Information does not include a
-requirement to continue to provide support service, warranty, or updates
-for a work that has been modified or installed by the recipient, or for
-the User Product in which it has been modified or installed.  Access to a
-network may be denied when the modification itself materially and
-adversely affects the operation of the network or violates the rules and
-protocols for communication across the network.
-
-  Corresponding Source conveyed, and Installation Information provided,
-in accord with this section must be in a format that is publicly
-documented (and with an implementation available to the public in
-source code form), and must require no special password or key for
-unpacking, reading or copying.
-
-  7. Additional Terms.
-
-  "Additional permissions" are terms that supplement the terms of this
-License by making exceptions from one or more of its conditions.
-Additional permissions that are applicable to the entire Program shall
-be treated as though they were included in this License, to the extent
-that they are valid under applicable law.  If additional permissions
-apply only to part of the Program, that part may be used separately
-under those permissions, but the entire Program remains governed by
-this License without regard to the additional permissions.
-
-  When you convey a copy of a covered work, you may at your option
-remove any additional permissions from that copy, or from any part of
-it.  (Additional permissions may be written to require their own
-removal in certain cases when you modify the work.)  You may place
-additional permissions on material, added by you to a covered work,
-for which you have or can give appropriate copyright permission.
-
-  Notwithstanding any other provision of this License, for material you
-add to a covered work, you may (if authorized by the copyright holders of
-that material) supplement the terms of this License with terms:
-
-    a) Disclaiming warranty or limiting liability differently from the
-    terms of sections 15 and 16 of this License; or
-
-    b) Requiring preservation of specified reasonable legal notices or
-    author attributions in that material or in the Appropriate Legal
-    Notices displayed by works containing it; or
-
-    c) Prohibiting misrepresentation of the origin of that material, or
-    requiring that modified versions of such material be marked in
-    reasonable ways as different from the original version; or
-
-    d) Limiting the use for publicity purposes of names of licensors or
-    authors of the material; or
-
-    e) Declining to grant rights under trademark law for use of some
-    trade names, trademarks, or service marks; or
-
-    f) Requiring indemnification of licensors and authors of that
-    material by anyone who conveys the material (or modified versions of
-    it) with contractual assumptions of liability to the recipient, for
-    any liability that these contractual assumptions directly impose on
-    those licensors and authors.
-
-  All other non-permissive additional terms are considered "further
-restrictions" within the meaning of section 10.  If the Program as you
-received it, or any part of it, contains a notice stating that it is
-governed by this License along with a term that is a further
-restriction, you may remove that term.  If a license document contains
-a further restriction but permits relicensing or conveying under this
-License, you may add to a covered work material governed by the terms
-of that license document, provided that the further restriction does
-not survive such relicensing or conveying.
-
-  If you add terms to a covered work in accord with this section, you
-must place, in the relevant source files, a statement of the
-additional terms that apply to those files, or a notice indicating
-where to find the applicable terms.
-
-  Additional terms, permissive or non-permissive, may be stated in the
-form of a separately written license, or stated as exceptions;
-the above requirements apply either way.
-
-  8. Termination.
-
-  You may not propagate or modify a covered work except as expressly
-provided under this License.  Any attempt otherwise to propagate or
-modify it is void, and will automatically terminate your rights under
-this License (including any patent licenses granted under the third
-paragraph of section 11).
-
-  However, if you cease all violation of this License, then your
-license from a particular copyright holder is reinstated (a)
-provisionally, unless and until the copyright holder explicitly and
-finally terminates your license, and (b) permanently, if the copyright
-holder fails to notify you of the violation by some reasonable means
-prior to 60 days after the cessation.
-
-  Moreover, your license from a particular copyright holder is
-reinstated permanently if the copyright holder notifies you of the
-violation by some reasonable means, this is the first time you have
-received notice of violation of this License (for any work) from that
-copyright holder, and you cure the violation prior to 30 days after
-your receipt of the notice.
-
-  Termination of your rights under this section does not terminate the
-licenses of parties who have received copies or rights from you under
-this License.  If your rights have been terminated and not permanently
-reinstated, you do not qualify to receive new licenses for the same
-material under section 10.
-
-  9. Acceptance Not Required for Having Copies.
-
-  You are not required to accept this License in order to receive or
-run a copy of the Program.  Ancillary propagation of a covered work
-occurring solely as a consequence of using peer-to-peer transmission
-to receive a copy likewise does not require acceptance.  However,
-nothing other than this License grants you permission to propagate or
-modify any covered work.  These actions infringe copyright if you do
-not accept this License.  Therefore, by modifying or propagating a
-covered work, you indicate your acceptance of this License to do so.
-
-  10. Automatic Licensing of Downstream Recipients.
-
-  Each time you convey a covered work, the recipient automatically
-receives a license from the original licensors, to run, modify and
-propagate that work, subject to this License.  You are not responsible
-for enforcing compliance by third parties with this License.
-
-  An "entity transaction" is a transaction transferring control of an
-organization, or substantially all assets of one, or subdividing an
-organization, or merging organizations.  If propagation of a covered
-work results from an entity transaction, each party to that
-transaction who receives a copy of the work also receives whatever
-licenses to the work the party's predecessor in interest had or could
-give under the previous paragraph, plus a right to possession of the
-Corresponding Source of the work from the predecessor in interest, if
-the predecessor has it or can get it with reasonable efforts.
-
-  You may not impose any further restrictions on the exercise of the
-rights granted or affirmed under this License.  For example, you may
-not impose a license fee, royalty, or other charge for exercise of
-rights granted under this License, and you may not initiate litigation
-(including a cross-claim or counterclaim in a lawsuit) alleging that
-any patent claim is infringed by making, using, selling, offering for
-sale, or importing the Program or any portion of it.
-
-  11. Patents.
-
-  A "contributor" is a copyright holder who authorizes use under this
-License of the Program or a work on which the Program is based.  The
-work thus licensed is called the contributor's "contributor version".
-
-  A contributor's "essential patent claims" are all patent claims
-owned or controlled by the contributor, whether already acquired or
-hereafter acquired, that would be infringed by some manner, permitted
-by this License, of making, using, or selling its contributor version,
-but do not include claims that would be infringed only as a
-consequence of further modification of the contributor version.  For
-purposes of this definition, "control" includes the right to grant
-patent sublicenses in a manner consistent with the requirements of
-this License.
-
-  Each contributor grants you a non-exclusive, worldwide, royalty-free
-patent license under the contributor's essential patent claims, to
-make, use, sell, offer for sale, import and otherwise run, modify and
-propagate the contents of its contributor version.
-
-  In the following three paragraphs, a "patent license" is any express
-agreement or commitment, however denominated, not to enforce a patent
-(such as an express permission to practice a patent or covenant not to
-sue for patent infringement).  To "grant" such a patent license to a
-party means to make such an agreement or commitment not to enforce a
-patent against the party.
-
-  If you convey a covered work, knowingly relying on a patent license,
-and the Corresponding Source of the work is not available for anyone
-to copy, free of charge and under the terms of this License, through a
-publicly available network server or other readily accessible means,
-then you must either (1) cause the Corresponding Source to be so
-available, or (2) arrange to deprive yourself of the benefit of the
-patent license for this particular work, or (3) arrange, in a manner
-consistent with the requirements of this License, to extend the patent
-license to downstream recipients.  "Knowingly relying" means you have
-actual knowledge that, but for the patent license, your conveying the
-covered work in a country, or your recipient's use of the covered work
-in a country, would infringe one or more identifiable patents in that
-country that you have reason to believe are valid.
-
-  If, pursuant to or in connection with a single transaction or
-arrangement, you convey, or propagate by procuring conveyance of, a
-covered work, and grant a patent license to some of the parties
-receiving the covered work authorizing them to use, propagate, modify
-or convey a specific copy of the covered work, then the patent license
-you grant is automatically extended to all recipients of the covered
-work and works based on it.
-
-  A patent license is "discriminatory" if it does not include within
-the scope of its coverage, prohibits the exercise of, or is
-conditioned on the non-exercise of one or more of the rights that are
-specifically granted under this License.  You may not convey a covered
-work if you are a party to an arrangement with a third party that is
-in the business of distributing software, under which you make payment
-to the third party based on the extent of your activity of conveying
-the work, and under which the third party grants, to any of the
-parties who would receive the covered work from you, a discriminatory
-patent license (a) in connection with copies of the covered work
-conveyed by you (or copies made from those copies), or (b) primarily
-for and in connection with specific products or compilations that
-contain the covered work, unless you entered into that arrangement,
-or that patent license was granted, prior to 28 March 2007.
-
-  Nothing in this License shall be construed as excluding or limiting
-any implied license or other defenses to infringement that may
-otherwise be available to you under applicable patent law.
-
-  12. No Surrender of Others' Freedom.
-
-  If conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot convey a
-covered work so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you may
-not convey it at all.  For example, if you agree to terms that obligate you
-to collect a royalty for further conveying from those to whom you convey
-the Program, the only way you could satisfy both those terms and this
-License would be to refrain entirely from conveying the Program.
-
-  13. Remote Network Interaction; Use with the GNU General Public License.
-
-  Notwithstanding any other provision of this License, if you modify the
-Program, your modified version must prominently offer all users
-interacting with it remotely through a computer network (if your version
-supports such interaction) an opportunity to receive the Corresponding
-Source of your version by providing access to the Corresponding Source
-from a network server at no charge, through some standard or customary
-means of facilitating copying of software.  This Corresponding Source
-shall include the Corresponding Source for any work covered by version 3
-of the GNU General Public License that is incorporated pursuant to the
-following paragraph.
-
-  Notwithstanding any other provision of this License, you have
-permission to link or combine any covered work with a work licensed
-under version 3 of the GNU General Public License into a single
-combined work, and to convey the resulting work.  The terms of this
-License will continue to apply to the part which is the covered work,
-but the work with which it is combined will remain governed by version
-3 of the GNU General Public License.
-
-  14. Revised Versions of this License.
-
-  The Free Software Foundation may publish revised and/or new versions of
-the GNU Affero General Public License from time to time.  Such new versions
-will be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-  Each version is given a distinguishing version number.  If the
-Program specifies that a certain numbered version of the GNU Affero General
-Public License "or any later version" applies to it, you have the
-option of following the terms and conditions either of that numbered
-version or of any later version published by the Free Software
-Foundation.  If the Program does not specify a version number of the
-GNU Affero General Public License, you may choose any version ever published
-by the Free Software Foundation.
-
-  If the Program specifies that a proxy can decide which future
-versions of the GNU Affero General Public License can be used, that proxy's
-public statement of acceptance of a version permanently authorizes you
-to choose that version for the Program.
-
-  Later license versions may give you additional or different
-permissions.  However, no additional obligations are imposed on any
-author or copyright holder as a result of your choosing to follow a
-later version.
-
-  15. Disclaimer of Warranty.
-
-  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
-APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
-HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
-OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
-THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
-IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
-ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. Limitation of Liability.
-
-  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
-THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
-GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
-USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
-DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
-PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
-EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGES.
-
-  17. Interpretation of Sections 15 and 16.
-
-  If the disclaimer of warranty and limitation of liability provided
-above cannot be given local legal effect according to their terms,
-reviewing courts shall apply local law that most closely approximates
-an absolute waiver of all civil liability in connection with the
-Program, unless a warranty or assumption of liability accompanies a
-copy of the Program in return for a fee.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    cic-internal-integration
-    Copyright (C) 2021  Grassroots Economics
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published
-    by the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

README.md

@@ -1,19 +1,42 @@
-# Community Inclusion Currency Stack (CIC Stack) 
-
-A custodial evm wallet for executing transactions via USSD
+# cic-internal-integration
 
 ## Getting started 
 
-This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
+## Make some keys
 
 ```
-export COMPOSE_DOCKER_CLI_BUILD=1
-export DOCKER_BUILDKIT=1
+docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
 ```
 
-To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
 
-## Documentation 
+### Prepare the repo
 
-[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
+This is stuff we need to put in  makefile but for now...
+
+File mounts and permisssions need to be set
+```
+chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
+````
+
+start cluster
+```
+docker-compose up
+```
+
+stop cluster
+```
+docker-compose down
+```
+
+delete data
+```
+docker-compose down -v
+```
+
+rebuild an images
+```
+docker-compose up --build <service_name>
+```
+
+Deployment variables are writtend to service-configs/.env after everthing is up.
 

apps/cic-base-os/Dockerfile

@@ -0,0 +1,34 @@
+# The solc image messes up the alpine environment, so we have to go all over again
+FROM python:3.8.6-slim-buster
+
+LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
+LABEL spdx-license-identifier="GPL-3.0-or-later"
+LABEL description="Base layer for buiding development images for the cic component suite"
+
+RUN apt-get update && \
+	apt-get install -y git gcc g++ libpq-dev && \
+	apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
+
+
+RUN echo installing nodejs tooling
+
+COPY ./dev/nvm.sh /root/
+
+# Install nvm with node and npm
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
+ENV NVM_DIR /root/.nvm
+ENV NODE_VERSION 15.3.0
+ENV BANCOR_NODE_VERSION 10.16.0
+
+RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
+	&& . $NVM_DIR/nvm.sh \
+	&& nvm install $NODE_VERSION \
+	&& nvm alias default $NODE_VERSION \
+	&& nvm use $NODE_VERSION \
+# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
+# owner of these files is "1001" by default - wtf
+	&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
+
+ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
+ENV PATH      $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
+

apps/cic-base-os/README.md

@@ -0,0 +1 @@
+## this is an example base image if we wanted one for all the other apps. Its just OS level things

apps/cic-base-os/aux/wait-for-it/.gitignore

@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/

apps/cic-base-os/aux/wait-for-it/.travis.yml

@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-

apps/cic-base-os/aux/wait-for-it/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

apps/cic-base-os/aux/wait-for-it/README.md

@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
--h HOST | --host=HOST       Host or IP under test
--p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
--s | --strict               Only execute subcommand if the test succeeds
--q | --quiet                Don't output any status messages
--t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
--- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-base-os/aux/wait-for-it/wait-for-it.sh

@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi

apps/cic-cache/.config/database.ini

@@ -6,4 +6,3 @@ HOST=localhost
 PORT=5432
 ENGINE=postgresql
 DRIVER=psycopg2
-DEBUG=

apps/cic-cache/.config/test/database.ini

@@ -6,4 +6,3 @@ HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=

apps/cic-cache/.coveragerc

@@ -2,6 +2,4 @@
 omit = 
 	.venv/*
 	scripts/*
-	cic_cache/db/migrations/*
-	cic_cache/version.py
-	cic_cache/cli
+	cic_cache/db/postgres/*

apps/cic-cache/.dockerignore

@@ -1,4 +0,0 @@
-.git
-.cache
-.dot
-**/doc

apps/cic-cache/.gitlab-ci.yml

@@ -1,17 +1,22 @@
-build-test-cic-cache:
-  stage: test 
-  tags:
-    - integration
-  variables:
-    APP_NAME: cic-cache
-    MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
-  script:
-    - cd apps/cic-cache
-    - docker build -t $MR_IMAGE_TAG -f docker/Dockerfile . 
-    - docker run $MR_IMAGE_TAG sh docker/run_tests.sh
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    changes:
-        - apps/$APP_NAME/**/*
-    when: always
+.cic_cache_variables:
+    variables:
+        APP_NAME: cic-cache
+        DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
+
+.cic_cache_changes_target:
+    rules:
+        - changes:
+            - $CONTEXT/$APP_NAME/*
+
+build-mr-cic-cache:
+    extends:
+        - .cic_cache_changes_target
+        - .py_build_merge_request
+        - .cic_cache_variables
+
+build-push-cic-cache:
+    extends:
+        - .py_build_push
+        - .cic_cache_variables
+        
+

apps/cic-cache/MANIFEST.in

@@ -1 +0,0 @@
-include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*

apps/cic-cache/README.md

@@ -1 +0,0 @@
-# CIC-CACHE

apps/cic-cache/aux/wait-for-it/.gitignore

@@ -1,3 +0,0 @@
-**/*.pyc
-.pydevproject
-/vendor/

apps/cic-cache/aux/wait-for-it/.travis.yml

@@ -1,7 +0,0 @@
-language: python
-python:
-  - "2.7"
-
-script:
-  - python test/wait-for-it.py
-

apps/cic-cache/aux/wait-for-it/LICENSE

@@ -1,20 +0,0 @@
-The MIT License (MIT)
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

apps/cic-cache/aux/wait-for-it/README.md

@@ -1,75 +0,0 @@
-# wait-for-it
-
-`wait-for-it.sh` is a pure bash script that will wait on the availability of a
-host and TCP port.  It is useful for synchronizing the spin-up of
-interdependent services, such as linked docker containers.  Since it is a pure
-bash script, it does not have any external dependencies.
-
-## Usage
-
-```text
-wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
--h HOST | --host=HOST       Host or IP under test
--p PORT | --port=PORT       TCP port under test
-                            Alternatively, you specify the host and port as host:port
--s | --strict               Only execute subcommand if the test succeeds
--q | --quiet                Don't output any status messages
--t TIMEOUT | --timeout=TIMEOUT
-                            Timeout in seconds, zero for no timeout
--- COMMAND ARGS             Execute command with args after the test finishes
-```
-
-## Examples
-
-For example, let's test to see if we can access port 80 on `www.google.com`,
-and if it is available, echo the message `google is up`.
-
-```text
-$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-You can set your own timeout with the `-t` or `--timeout=` option.  Setting
-the timeout value to 0 will disable the timeout:
-
-```text
-$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
-wait-for-it.sh: waiting for www.google.com:80 without a timeout
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-google is up
-```
-
-The subcommand will be executed regardless if the service is up or not.  If you
-wish to execute the subcommand only if the service is up, add the `--strict`
-argument. In this example, we will test port 81 on `www.google.com` which will
-fail:
-
-```text
-$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
-wait-for-it.sh: waiting 1 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
-wait-for-it.sh: strict mode, refusing to execute subprocess
-```
-
-If you don't want to execute a subcommand, leave off the `--` argument.  This
-way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
-and determine how to proceed:
-
-```text
-$ ./wait-for-it.sh www.google.com:80
-wait-for-it.sh: waiting 15 seconds for www.google.com:80
-wait-for-it.sh: www.google.com:80 is available after 0 seconds
-$ echo $?
-0
-$ ./wait-for-it.sh www.google.com:81
-wait-for-it.sh: waiting 15 seconds for www.google.com:81
-wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
-$ echo $?
-124
-```
-
-## Community
-
-*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).

apps/cic-cache/aux/wait-for-it/wait-for-it.sh

@@ -1,182 +0,0 @@
-#!/usr/bin/env bash
-# Use this script to test if a given TCP host/port are available
-
-WAITFORIT_cmdname=${0##*/}
-
-echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
-
-usage()
-{
-    cat << USAGE >&2
-Usage:
-    $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
-    -h HOST | --host=HOST       Host or IP under test
-    -p PORT | --port=PORT       TCP port under test
-                                Alternatively, you specify the host and port as host:port
-    -s | --strict               Only execute subcommand if the test succeeds
-    -q | --quiet                Don't output any status messages
-    -t TIMEOUT | --timeout=TIMEOUT
-                                Timeout in seconds, zero for no timeout
-    -- COMMAND ARGS             Execute command with args after the test finishes
-USAGE
-    exit 1
-}
-
-wait_for()
-{
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    else
-        echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
-    fi
-    WAITFORIT_start_ts=$(date +%s)
-    while :
-    do
-        if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
-            nc -z $WAITFORIT_HOST $WAITFORIT_PORT
-            WAITFORIT_result=$?
-        else
-            (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
-            WAITFORIT_result=$?
-        fi
-        if [[ $WAITFORIT_result -eq 0 ]]; then
-            WAITFORIT_end_ts=$(date +%s)
-            echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
-            break
-        fi
-        sleep 1
-    done
-    return $WAITFORIT_result
-}
-
-wait_for_wrapper()
-{
-    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
-    if [[ $WAITFORIT_QUIET -eq 1 ]]; then
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    else
-        timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
-    fi
-    WAITFORIT_PID=$!
-    trap "kill -INT -$WAITFORIT_PID" INT
-    wait $WAITFORIT_PID
-    WAITFORIT_RESULT=$?
-    if [[ $WAITFORIT_RESULT -ne 0 ]]; then
-        echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
-    fi
-    return $WAITFORIT_RESULT
-}
-
-# process arguments
-while [[ $# -gt 0 ]]
-do
-    case "$1" in
-        *:* )
-        WAITFORIT_hostport=(${1//:/ })
-        WAITFORIT_HOST=${WAITFORIT_hostport[0]}
-        WAITFORIT_PORT=${WAITFORIT_hostport[1]}
-        shift 1
-        ;;
-        --child)
-        WAITFORIT_CHILD=1
-        shift 1
-        ;;
-        -q | --quiet)
-        WAITFORIT_QUIET=1
-        shift 1
-        ;;
-        -s | --strict)
-        WAITFORIT_STRICT=1
-        shift 1
-        ;;
-        -h)
-        WAITFORIT_HOST="$2"
-        if [[ $WAITFORIT_HOST == "" ]]; then break; fi
-        shift 2
-        ;;
-        --host=*)
-        WAITFORIT_HOST="${1#*=}"
-        shift 1
-        ;;
-        -p)
-        WAITFORIT_PORT="$2"
-        if [[ $WAITFORIT_PORT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --port=*)
-        WAITFORIT_PORT="${1#*=}"
-        shift 1
-        ;;
-        -t)
-        WAITFORIT_TIMEOUT="$2"
-        if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
-        shift 2
-        ;;
-        --timeout=*)
-        WAITFORIT_TIMEOUT="${1#*=}"
-        shift 1
-        ;;
-        --)
-        shift
-        WAITFORIT_CLI=("$@")
-        break
-        ;;
-        --help)
-        usage
-        ;;
-        *)
-        echoerr "Unknown argument: $1"
-        usage
-        ;;
-    esac
-done
-
-if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
-    echoerr "Error: you need to provide a host and port to test."
-    usage
-fi
-
-WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
-WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
-WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
-WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
-
-# Check to see if timeout is from busybox?
-WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
-WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
-
-WAITFORIT_BUSYTIMEFLAG=""
-if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
-    WAITFORIT_ISBUSY=1
-    # Check if busybox timeout uses -t flag
-    # (recent Alpine versions don't support -t anymore)
-    if timeout &>/dev/stdout | grep -q -e '-t '; then
-        WAITFORIT_BUSYTIMEFLAG="-t"
-    fi
-else
-    WAITFORIT_ISBUSY=0
-fi
-
-if [[ $WAITFORIT_CHILD -gt 0 ]]; then
-    wait_for
-    WAITFORIT_RESULT=$?
-    exit $WAITFORIT_RESULT
-else
-    if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
-        wait_for_wrapper
-        WAITFORIT_RESULT=$?
-    else
-        wait_for
-        WAITFORIT_RESULT=$?
-    fi
-fi
-
-if [[ $WAITFORIT_CLI != "" ]]; then
-    if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
-        echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
-        exit $WAITFORIT_RESULT
-    fi
-    exec "${WAITFORIT_CLI[@]}"
-else
-    exit $WAITFORIT_RESULT
-fi

apps/cic-cache/cic_cache/api.py

@@ -55,37 +55,15 @@ class Api:
                     queue=callback_queue,
                     )       
 
-    def list(self, offset=0, limit=100, address=None, oldest=False):
+    def list(self, offset, limit, address=None):
         s = celery.signature(
         'cic_cache.tasks.tx.tx_filter',
         [
-            offset,
-            limit,
+            0,
+            100,
             address,
-            oldest,
             ],
-            queue=self.queue,
-        )
-        if self.callback_param != None:
-            s.link(self.callback_success).on_error(self.callback_error)
-
-        t = s.apply_async()
-
-        return t
-
-
-    def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
-        s = celery.signature(
-        'cic_cache.tasks.tx.tx_filter_content',
-        [
-            offset,
-            limit,
-            address,
-            block_offset,
-            block_limit,
-            oldest,
-            ],
-            queue=self.queue,
+            queue=None
         )
         if self.callback_param != None:
             s.link(self.callback_success).on_error(self.callback_error)

apps/cic-cache/cic_cache/cache.py

@@ -1,42 +1,30 @@
 # standard imports
 import logging
-import datetime
 
-# external imports
+# third-party imports
 import moolb
 
 # local imports
-from cic_cache.db.list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        list_transactions_mined_with_data,
-        list_transactions_mined_with_data_index,
-        list_transactions_account_mined_with_data_index,
-        list_transactions_account_mined_with_data,
-        )
+from cic_cache.db import list_transactions_mined
+from cic_cache.db import list_transactions_account_mined
 
 logg = logging.getLogger()
 
 
-DEFAULT_FILTER_SIZE = 8192 * 8
-DEFAULT_LIMIT = 100
-
-class Cache:
+class BloomCache:
 
     def __init__(self, session):
         self.session = session
 
 
-class BloomCache(Cache):
-
     @staticmethod
     def __get_filter_size(n):
-        n = DEFAULT_FILTER_SIZE
+        n = 8192 * 8
         logg.warning('filter size hardcoded to {}'.format(n))
         return n
 
 
-    def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions(self, offset, limit):
         """Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
 
         Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
@@ -53,7 +41,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
+        rows = list_transactions_mined(self.session, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -62,12 +50,7 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
@@ -76,7 +59,7 @@ class BloomCache(Cache):
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
 
 
-    def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
+    def load_transactions_account(self, address, offset, limit):
         """Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
 
         :param address: Address to retrieve transactions for.
@@ -88,7 +71,7 @@ class BloomCache(Cache):
         :return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
         :rtype: tuple
         """
-        rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest) 
+        rows = list_transactions_account_mined(self.session, address, offset, limit) 
 
         f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
         f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
@@ -97,74 +80,10 @@ class BloomCache(Cache):
         for r in rows:
             if highest_block == -1:
                 highest_block = r[0]
-                lowest_block = r[0]
-            else:
-                if oldest:
-                    highest_block = r[0]
-                else:
-                    lowest_block = r[0]
+            lowest_block = r[0]
             block = r[0].to_bytes(4, byteorder='big')
             tx = r[1].to_bytes(4, byteorder='big')
             f_block.add(block)
             f_blocktx.add(block + tx)
             logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
         return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
-
-
-class DataCache(Cache):
-
-    def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-        rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest) 
-        return self.__process_rows(rows, oldest)
-
-
-    def __process_rows(self, rows, oldest):
-        tx_cache = []
-        highest_block = -1;
-        lowest_block = -1;
-        date_is_str = None # stick this in startup
-        for r in rows:
-            if highest_block == -1:
-                highest_block = r['block_number']
-                lowest_block = r['block_number']
-            else:
-                if oldest:
-                    highest_block = r['block_number']
-                else:
-                    lowest_block = r['block_number']
-            tx_type = 'unknown'
-
-            if r['value'] != None:
-                tx_type = '{}.{}'.format(r['domain'], r['value'])
-
-            if date_is_str == None:
-                date_is_str = type(r['date_block']).__name__ == 'str'
-
-            o = {
-                'block_number': r['block_number'],
-                'tx_hash': r['tx_hash'],
-                'date_block': r['date_block'],
-                'sender': r['sender'],
-                'recipient': r['recipient'],
-                'from_value': int(r['from_value']),
-                'to_value': int(r['to_value']),
-                'source_token': r['source_token'],
-                'destination_token': r['destination_token'],
-                'success': r['success'],
-                'tx_type': tx_type,
-            }
-
-            if date_is_str:
-                o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
-
-            tx_cache.append(o)
-        return (lowest_block, highest_block, tx_cache)

apps/cic-cache/cic_cache/cli/__init__.py

@@ -1,15 +0,0 @@
-# local imports
-from .base import *
-from .chain import (
-        EthChainInterface,
-        chain_interface,
-        )
-from .rpc import RPC
-from .arg import ArgumentParser
-from .config import Config
-from .celery import CeleryApp
-from .registry import (
-        connect_registry,
-        connect_token_registry,
-        connect_declarator,
-        )

apps/cic-cache/cic_cache/cli/arg.py

@@ -1,20 +0,0 @@
-# external imports
-from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
-
-# local imports
-from .base import (
-        CICFlag,
-        Flag,
-        )
-
-
-class ArgumentParser(BaseArgumentParser):
-
-    def process_local_flags(self, local_arg_flags):
-        if local_arg_flags & CICFlag.CELERY:
-            self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
-        if local_arg_flags & CICFlag.SYNCER:
-            self.add_argument('--offset', type=int, help='Start block height for initial history sync')
-            self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
-        if local_arg_flags & CICFlag.CHAIN:
-            self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')

apps/cic-cache/cic_cache/cli/base.py

@@ -1,31 +0,0 @@
-# standard imports
-import enum
-
-# external imports
-from chainlib.eth.cli import (
-    argflag_std_read,
-    argflag_std_write,
-    argflag_std_base,
-    Flag,
-    )
-
-class CICFlag(enum.IntEnum):
-   
-    # celery - nibble 1
-    CELERY = 1
-
-    # redis - nibble 2
-#    REDIS = 16
-#    REDIS_CALLBACK = 32
-
-    # chain - nibble 3
-    CHAIN = 256
-
-    # sync - nibble 4
-    SYNCER = 4096
-
-
-argflag_local_task = CICFlag.CELERY 
-#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
-argflag_local_chain = CICFlag.CHAIN
-argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN

apps/cic-cache/cic_cache/cli/celery.py

@@ -1,24 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-import celery
-
-logg = logging.getLogger(__name__)
-
-
-class CeleryApp:
-   
-    @classmethod
-    def from_config(cls, config):
-        backend_url = config.get('CELERY_RESULT_URL')
-        broker_url = config.get('CELERY_BROKER_URL')
-        celery_app = None
-        if backend_url != None:
-            celery_app = celery.Celery(broker=broker_url, backend=backend_url)
-            logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
-        else:
-            celery_app = celery.Celery(broker=broker_url)
-            logg.info('creating celery app without results backend on {}'.format(broker_url))
-
-        return celery_app

apps/cic-cache/cic_cache/cli/chain.py

@@ -1,21 +0,0 @@
-# external imports
-from chainlib.eth.block import (
-        block_by_number,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        Tx,
-        )
-from chainlib.interface import ChainInterface
-
-
-class EthChainInterface(ChainInterface):
-    
-    def __init__(self):
-        self._tx_receipt = receipt
-        self._block_by_number = block_by_number
-        self._block_from_src = Block.from_src
-        self._src_normalize = Tx.src_normalize
-
-chain_interface = EthChainInterface()

apps/cic-cache/cic_cache/cli/config.py

@@ -1,63 +0,0 @@
-# standard imports
-import os
-import logging
-
-# external imports
-from chainlib.eth.cli import (
-        Config as BaseConfig,
-        Flag,
-        )
-
-# local imports
-from .base import CICFlag
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-
-logg = logging.getLogger(__name__)
-
-
-class Config(BaseConfig):
-
-    local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
-
-    @classmethod
-    def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
-        expanded_base_config_dir = [cls.local_base_config_dir]
-        if base_config_dir != None:
-            if isinstance(base_config_dir, str):
-                base_config_dir = [base_config_dir]
-            for d in base_config_dir:
-                expanded_base_config_dir.append(d)
-        config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
-
-        local_args_override = {}
-#        if local_arg_flags & CICFlag.REDIS:
-#            local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
-#            local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
-#            local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
-#            local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
-
-        if local_arg_flags & CICFlag.CHAIN:
-            local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
-
-        if local_arg_flags & CICFlag.CELERY:
-            local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
-
-        if local_arg_flags & CICFlag.SYNCER:
-            local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
-            local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
-
-        config.dict_override(local_args_override, 'local cli args')
-
-#        if local_arg_flags & CICFlag.REDIS_CALLBACK:
-#            config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
-#            config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
-
-        if local_arg_flags & CICFlag.CELERY:
-            config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
-
-        logg.debug('config loaded:\n{}'.format(config))
-
-        return config
-
-

apps/cic-cache/cic_cache/cli/registry.py

@@ -1,33 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from cic_eth_registry import CICRegistry
-from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
-from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
-from chainlib.eth.constant import ZERO_ADDRESS
-
-logg = logging.getLogger()
-
-
-def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
-    logg.debug('using token registry address {}'.format(token_registry_address))
-    lookup = TokenIndexLookup(chain_spec, token_registry_address)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
-    registry = CICRegistry(chain_spec, conn)
-    declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
-    logg.debug('using declarator address {}'.format(declarator_address))
-    lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
-    CICRegistry.add_lookup(lookup)
-
-
-def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
-    CICRegistry.address = registry_address
-    registry = CICRegistry(chain_spec, conn)
-    registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
-    return registry

apps/cic-cache/cic_cache/cli/rpc.py

@@ -1,43 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.connection import (
-        RPCConnection,
-        ConnType,
-        )
-from chainlib.eth.connection import EthUnixSignerConnection
-from chainlib.chain import ChainSpec
-
-logg = logging.getLogger(__name__)
-
-
-class RPC:
-
-    def __init__(self, chain_spec, rpc_provider, signer_provider=None):
-        self.chain_spec = chain_spec
-        self.rpc_provider = rpc_provider
-        self.signer_provider = signer_provider
-
-
-    def get_default(self):
-        return RPCConnection.connect(self.chain_spec, 'default')
-
-
-    @staticmethod
-    def from_config(config):
-        chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-        RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
-        if config.get('SIGNER_PROVIDER'):
-            RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
-            RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
-        rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
-        logg.info('set up rpc: {}'.format(rpc))
-        return rpc
-
-
-    def __str__(self):
-        return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
-
-
-

apps/cic-cache/cic_cache/data/config/celery.ini

@@ -1,5 +0,0 @@
-[celery]
-broker_url = redis://localhost:6379
-result_url = 
-queue = cic-cache
-debug = 0

apps/cic-cache/cic_cache/data/config/database.ini

@@ -1,11 +0,0 @@
-[database]
-engine =
-driver =
-host =
-port =
-#name = cic-cache
-prefix =
-user =
-password =
-debug = 0
-pool_size = 0

apps/cic-cache/cic_cache/data/config/signer.ini

@@ -1,2 +0,0 @@
-[signer]
-provider =

apps/cic-cache/cic_cache/data/config/syncer.ini

@@ -1,4 +0,0 @@
-[syncer]
-loop_interval = 1
-offset = 0
-no_history = 0

apps/cic-cache/cic_cache/db/__init__.py

@@ -2,33 +2,23 @@
 import logging
 
 # local imports
-from .list import (
-        list_transactions_mined,
-        list_transactions_account_mined,
-        add_transaction,
-        tag_transaction,
-        add_tag,
-    )
-from cic_cache.db.models.base import SessionBase
-
+from .list import list_transactions_mined
+from .list import list_transactions_account_mined
+from .list import add_transaction
 
 logg = logging.getLogger()
 
 
-def dsn_from_config(config, name):
+def dsn_from_config(config):
     scheme = config.get('DATABASE_ENGINE')
     if config.get('DATABASE_DRIVER') != None:
         scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
 
-    database_name = name
-    if config.get('DATABASE_PREFIX'):
-        database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
     dsn = ''
     if config.get('DATABASE_ENGINE') == 'sqlite':
-        SessionBase.poolable = False
         dsn = '{}:///{}'.format(
                 scheme,
-                database_name,
+                config.get('DATABASE_NAME'),    
             )
 
     else:
@@ -38,7 +28,7 @@ def dsn_from_config(config, name):
                 config.get('DATABASE_PASSWORD'),
                 config.get('DATABASE_HOST'),
                 config.get('DATABASE_PORT'),
-                database_name,
+                config.get('DATABASE_NAME'),    
             )
     logg.debug('parsed dsn from config: {}'.format(dsn))
     return dsn

apps/cic-cache/cic_cache/db/list.py

@@ -2,9 +2,8 @@
 import logging
 import datetime
 
-# external imports
+# third-party imports
 from cic_cache.db.models.base import SessionBase
-from sqlalchemy import text
 
 logg = logging.getLogger()
 
@@ -13,9 +12,6 @@ def list_transactions_mined(
         session,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Executes db query to return all confirmed transactions according to the specified offset and limit.
 
@@ -26,154 +22,7 @@ def list_transactions_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data(
-        session,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
-
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_mined_with_data_index(
-        session,
-        offset,
-        end,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
-
-    r = session.execute(s)
-    return r
-
-
-def list_transactions_account_mined_with_data_index(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
-
-    :param offset: Offset in data set to return transactions from
-    :type offset: int
-    :param limit: Max number of transactions to retrieve
-    :type limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
-    r = session.execute(s)
-    return r
-
-def list_transactions_account_mined_with_data(
-        session,
-        address,
-        offset,
-        limit,
-        block_offset,
-        block_limit,
-        oldest=False,
-        ):
-    """Executes db query to return all confirmed transactions according to the specified offset and limit.
-
-    :param block_offset: First block to include in search
-    :type block_offset: int
-    :param block_limit: Last block to include in search
-    :type block_limit: int
-    :result: Result set
-    :rtype: SQLAlchemy.ResultProxy
-    """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-    else:
-        s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
+    s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
     r = session.execute(s)
     return r
 
@@ -183,9 +32,6 @@ def list_transactions_account_mined(
         address,
         offset,
         limit,
-        block_offset,
-        block_limit,
-        oldest=False,
         ):
     """Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
 
@@ -198,27 +44,13 @@ def list_transactions_account_mined(
     :result: Result set
     :rtype: SQLAlchemy.ResultProxy
     """
-
-    order_by = 'DESC'
-    if oldest:
-        order_by = 'ASC'
-
-    if block_offset:
-        if block_limit:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
-        else:
-            s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
-
-    else:
-        s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
-
+    s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
     r = session.execute(s)
     return r
 
 
 def add_transaction(
-        session,
-        tx_hash,
+        session, tx_hash,
         block_number,
         tx_index,
         sender,
@@ -230,33 +62,6 @@ def add_transaction(
         success,
         timestamp,
         ):
-    """Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param block_number: Block number
-    :type block_number: int
-    :param tx_index: Transaction index in block
-    :type tx_index: int
-    :param sender: Ethereum address of effective sender
-    :type sender: str, 0x-hex
-    :param receiver: Ethereum address of effective recipient
-    :type receiver: str, 0x-hex
-    :param source_token: Ethereum address of token used by sender
-    :type source_token: str, 0x-hex
-    :param destination_token: Ethereum address of token received by recipient
-    :type destination_token: str, 0x-hex
-    :param from_value: Source token value spent in transaction
-    :type from_value: int
-    :param to_value: Destination token value received in transaction
-    :type to_value: int
-    :param success: True if code execution on network was successful
-    :type success: bool
-    :param date_block: Block timestamp
-    :type date_block: datetime
-    """
     date_block = datetime.datetime.fromtimestamp(timestamp)
     s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
             tx_hash,
@@ -272,74 +77,3 @@ def add_transaction(
             date_block,
             )
     session.execute(s)
-
-
-
-def tag_transaction(
-        session,
-        tx_hash,
-        name,
-        domain=None,
-        ):
-    """Tag a single transaction with a single tag.
-
-    Tag must already exist in storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param tx_hash: Transaction hash
-    :type tx_hash: str, 0x-hex
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises ValueError: Unknown tag or transaction hash
-
-    """
-
-    s = text("SELECT id from tx where tx_hash = :a")
-    r = session.execute(s, {'a': tx_hash}).fetchall()
-    tx_id = r[0].values()[0]
-
-    if tx_id == None:
-        raise ValueError('unknown tx hash {}'.format(tx_hash))
-
-    #s = text("SELECT id from tag where value = :a and domain = :b")
-    if domain == None:
-        s = text("SELECT id from tag where value = :a")
-    else:
-        s = text("SELECT id from tag where value = :a and domain = :b")
-    r = session.execute(s, {'a': name, 'b': domain}).fetchall()
-    tag_id = r[0].values()[0]
-
-    logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
-
-    if tag_id == None:
-        raise ValueError('unknown tag name {} domain {}'.format(name, domain))
-
-    s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
-    r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
-
-
-def add_tag(
-        session,
-        name,
-        domain=None,
-        ):
-    """Add a single tag to storage.
-
-    :param session: Persistent storage session object
-    :type session: SQLAlchemy session
-    :param name: Tag value
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    :raises sqlalchemy.exc.IntegrityError: Tag already exists
-    """
-
-    s = None
-    if domain == None: 
-        s = text("INSERT INTO tag (value) VALUES (:b)")
-    else:
-        s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
-    session.execute(s, {'a': domain, 'b': name})

apps/cic-cache/cic_cache/db/migrations/default/versions/6604de4203e2_add_chain_syncer.py

@@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
 """
 from alembic import op
 import sqlalchemy as sa
-from chainsyncer.db.migrations.default.export import (
+from chainsyncer.db.migrations.sqlalchemy import (
         chainsyncer_upgrade,
         chainsyncer_downgrade,
         )

apps/cic-cache/cic_cache/db/migrations/default/versions/aaf2bdce7d6e_transaction_tags.py

@@ -1,38 +0,0 @@
-"""Transaction tags
-
-Revision ID: aaf2bdce7d6e
-Revises: 6604de4203e2
-Create Date: 2021-05-01 09:20:20.775082
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'aaf2bdce7d6e'
-down_revision = '6604de4203e2'
-branch_labels = None
-depends_on = None
-
-
-def upgrade():
-    op.create_table(
-            'tag',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('domain', sa.String(), nullable=True),
-            sa.Column('value', sa.String(), nullable=False),
-            )
-    op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
-
-    op.create_table(
-            'tag_tx_link',
-            sa.Column('id', sa.Integer, primary_key=True),
-            sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
-            sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
-                )
-
-def downgrade():
-    op.drop_table('tag_tx_link')
-    op.drop_index('idx_tag_domain_value')
-    op.drop_table('tag')

apps/cic-cache/cic_cache/db/models/base.py

@@ -100,4 +100,3 @@ class SessionBase(Model):
             logg.debug('destroying session {}'.format(session_key))
             session.commit()
             session.close()
-            del SessionBase.localsessions[session_key]

apps/cic-cache/cic_cache/runnable/daemons/filters/__init__.py

@@ -1,2 +1 @@
 from .erc20 import *
-from .faucet import *

apps/cic-cache/cic_cache/runnable/daemons/filters/base.py

@@ -1,27 +1,2 @@
-class TagSyncFilter:
-    """Holds tag name and domain for an implementing filter.
-
-    :param name: Tag value 
-    :type name: str
-    :param domain: Tag domain
-    :type domain: str
-    """
-
-    def __init__(self, name, domain=None):
-        self.tag_name = name
-        self.tag_domain = domain
-
-
-    def tag(self):
-        """Return tag value/domain.
-
-        :rtype: Tuple
-        :returns: tag value/domain.
-        """
-        return (self.tag_name, self.tag_domain)
-
-
-    def __str__(self):
-        if self.tag_domain == None:
-            return self.tag_name
-        return '{}.{}'.format(self.tag_domain, self.tag_name)
+class SyncFilter:
+    pass

apps/cic-cache/cic_cache/runnable/daemons/filters/erc20.py

@@ -2,6 +2,7 @@
 import logging
 
 # external imports
+from chainlib.eth.erc20 import ERC20
 from chainlib.eth.address import (
         to_checksum_address,
         )
@@ -12,19 +13,17 @@ from cic_eth_registry.error import (
         NotAContractError,
         ContractMismatchError,
         )
-from eth_erc20 import ERC20
 
 # local imports
-from .base import TagSyncFilter
+from .base import SyncFilter
 from cic_cache import db as cic_cache_db
 
 logg = logging.getLogger().getChild(__name__)
 
 
-class ERC20TransferFilter(TagSyncFilter):
+class ERC20TransferFilter(SyncFilter):
 
     def __init__(self, chain_spec):
-        super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
         self.chain_spec = chain_spec
 
 
@@ -47,9 +46,6 @@ class ERC20TransferFilter(TagSyncFilter):
         except RequestMismatchException:
             logg.debug('erc20 match but not a transfer, skipping')
             return False
-        except ValueError:
-            logg.debug('erc20 match but bogus data, skipping')
-            return False
 
         token_sender = tx.outputs[0]
         token_recipient = transfer_data[0]
@@ -72,12 +68,5 @@ class ERC20TransferFilter(TagSyncFilter):
                 block.timestamp,
                 )
         db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
 
         return True

apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py

@@ -1,73 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from erc20_faucet import Faucet
-from chainlib.eth.address import to_checksum_address
-from chainlib.eth.constant import ZERO_ADDRESS
-from chainlib.status import Status
-from hexathon import strip_0x
-
-# local imports
-import cic_cache.db as cic_cache_db
-from .base import TagSyncFilter
-
-#logg = logging.getLogger().getChild(__name__)
-logg = logging.getLogger()
-
-
-class FaucetFilter(TagSyncFilter):
-
-    def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
-        super(FaucetFilter, self).__init__('give_to', domain='faucet')
-        self.chain_spec = chain_spec
-        self.sender_address = sender_address
-
-
-    def filter(self, conn, block, tx, db_session=None):
-        try:
-            data = strip_0x(tx.payload)
-        except ValueError:
-            return False
-        logg.debug('data {}'.format(data))
-        if Faucet.method_for(data[:8]) == None:
-            return False
-
-        token_sender = tx.inputs[0]
-        token_recipient = data[64+8-40:]
-        logg.debug('token recipient {}'.format(token_recipient))
- 
-        f = Faucet(self.chain_spec)
-        o = f.token(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token = f.parse_token(r)
-
-        f = Faucet(self.chain_spec)
-        o = f.token_amount(token_sender, sender_address=self.sender_address)
-        r = conn.do(o)
-        token_value = f.parse_token_amount(r)
-
-        cic_cache_db.add_transaction(
-                db_session,
-                tx.hash,
-                block.number,
-                tx.index,
-                to_checksum_address(token_sender),
-                to_checksum_address(token_recipient),
-                token,
-                token,
-                token_value,
-                token_value,
-                tx.status == Status.SUCCESS,
-                block.timestamp,
-                )
-        db_session.flush()
-        cic_cache_db.tag_transaction(
-                db_session,
-                tx.hash,
-                self.tag_name,
-                domain=self.tag_domain,
-                )
-        db_session.commit()
-
-        return True

apps/cic-cache/cic_cache/runnable/daemons/query.py

@@ -1,186 +0,0 @@
-# standard imports
-import logging
-import json
-import re
-import base64
-
-# external imports
-from hexathon import (
-        add_0x,
-        strip_0x,
-        )
-from chainlib.encode import TxHexNormalizer
-
-# local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-    )
-
-logg = logging.getLogger(__name__)
-#logg = logging.getLogger()
-
-re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
-re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
-re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
-re_default_limit = r'/defaultlimit/?'
-
-DEFAULT_LIMIT = 100
-
-tx_normalize = TxHexNormalizer()
-
-def parse_query_account(r):
-    address = strip_0x(r[1])
-    #address = tx_normalize.wallet_address(address)
-    limit = DEFAULT_LIMIT
-    g = r.groups()
-    if len(g) > 3:
-        limit = int(r[4])
-        if limit == 0:
-            limit = DEFAULT_LIMIT
-    offset = 0
-    if len(g) > 4:
-        offset = int(r[6])
-
-    logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
-
-    return (address, offset, limit,)
-
-
-# r is an re.Match
-def parse_query_any(r):
-    limit = DEFAULT_LIMIT
-    offset = 0
-    block_offset = None
-    block_end = None
-    if r.lastindex != None:
-        if r.lastindex > 0:
-            limit = int(r[1])
-        if r.lastindex > 1:
-            offset = int(r[2])
-        if r.lastindex > 2:
-            block_offset = int(r[3])
-        if r.lastindex > 3:
-            block_end = int(r[4])
-            if block_end < block_offset:
-                raise ValueError('cart before the horse, dude')
-
-    logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
-
-    return (offset, limit, block_offset, block_end,)
-
-
-def process_default_limit(session, env):
-    r = re.match(re_default_limit, env.get('PATH_INFO'))
-    if not r:
-        return None
-
-    return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
-
-
-def process_transactions_account_bloom(session, env):
-    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-    logg.debug('match account bloom')
-
-    (address, offset, limit,) = parse_query_account(r)
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_bloom(session, env):
-    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
-    if not r:
-        return None
-    logg.debug('match all bloom')
-
-    (limit, offset, block_offset, block_end,) = parse_query_any(r)
-
-    c = BloomCache(session)
-    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
-
-    o = {
-        'alg': 'sha256',
-        'low': lowest_block,
-        'high': highest_block,
-        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
-        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
-        'filter_rounds': 3,
-            }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_all_data(session, env):
-    r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-    #    return None
-    logg.debug('match all data')
-
-    logg.debug('got data request {}'.format(env))
-
-    (offset, limit, block_offset, block_end) = parse_query_any(r)
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)
-
-
-def process_transactions_account_data(session, env):
-    r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
-    if not r:
-        return None
-    logg.debug('match account data')
-    #if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
-    #    return None
-
-    (address, offset, limit,) = parse_query_account(r)
-
-    c = DataCache(session)
-    (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
-
-    for r in tx_cache:
-        r['date_block'] = r['date_block'].timestamp()
-
-    o = {
-        'low': lowest_block,
-        'high': highest_block,
-        'data': tx_cache,
-    }
-
-    j = json.dumps(o)
-
-    return ('application/json', j.encode('utf-8'),)

apps/cic-cache/cic_cache/runnable/daemons/server.py

@@ -1,42 +1,115 @@
 # standard imports
 import os
+import re
 import logging
 import argparse
+import json
 import base64
 
-# external imports
+# third-party imports
 import confini
 
 # local imports
-import cic_cache.cli
+from cic_cache import BloomCache
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
-from cic_cache.runnable.daemons.query import (
-        process_default_limit,
-        process_transactions_account_bloom,
-        process_transactions_account_data,
-        process_transactions_all_bloom,
-        process_transactions_all_data,
-        )
-import cic_cache.cli
 
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
 
-arg_flags = cic_cache.cli.argflag_std_read
-local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task 
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-cache')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
 args = argparser.parse_args()
 
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
 
-# connect to database
-dsn = dsn_from_config(config, 'cic_cache')
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+dsn = dsn_from_config(config)
 SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
 
+re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
+re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
+
+DEFAULT_LIMIT = 100
+
+
+def process_transactions_account_bloom(session, env):
+    r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    address = r[1]
+    if r[2] == None:
+        address = '0x' + address
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 2:
+        offset = r[3]
+    limit = 0
+    if r.lastindex > 3:
+        limit = r[4]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
+
+def process_transactions_all_bloom(session, env):
+    r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
+    if not r:
+        return None
+
+    offset = DEFAULT_LIMIT
+    if r.lastindex > 0:
+        offset = r[1]
+    limit = 0
+    if r.lastindex > 1:
+        limit = r[2]
+
+    c = BloomCache(session)
+    (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
+
+    o = {
+        'alg': 'sha256',
+        'low': lowest_block,
+        'high': highest_block,
+        'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
+        'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
+        'filter_rounds': 3,
+            }
+
+    j = json.dumps(o)
+
+    return ('application/json', j.encode('utf-8'),)
+
 
 # uwsgi application
 def application(env, start_response):
@@ -46,18 +119,10 @@ def application(env, start_response):
 
     session = SessionBase.create_session()
     for handler in [
-            process_transactions_account_data,
-            process_transactions_account_bloom,
-            process_transactions_all_data,
             process_transactions_all_bloom,
-            process_default_limit,
+            process_transactions_account_bloom,
             ]:
-        r = None
-        try:
-            r = handler(session, env)
-        except ValueError as e:
-            start_response('400 {}'.format(str(e)))
-            return []
+        r = handler(session, env)
         if r != None:
             (mime_type, content) = r
             break

apps/cic-cache/cic_cache/runnable/daemons/tasker.py

@@ -3,14 +3,12 @@ import logging
 import os
 import sys
 import argparse
-import tempfile
 
 # third-party imports
 import celery
 import confini
 
 # local imports
-import cic_cache.cli
 from cic_cache.db import dsn_from_config
 from cic_cache.db.models.base import SessionBase
 from cic_cache.tasks.tx import *
@@ -18,20 +16,35 @@ from cic_cache.tasks.tx import *
 logging.basicConfig(level=logging.WARNING)
 logg = logging.getLogger()
 
-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_task
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
+config_dir = os.path.join('/usr/local/etc/cic-cache')
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+
 args = argparser.parse_args()
 
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
 
 # connect to database
-dsn = dsn_from_config(config, 'cic_cache')
+dsn = dsn_from_config(config)
 SessionBase.connect(dsn)
 
+# verify database connection with minimal sanity query
+#session = SessionBase.create_session()
+#session.execute('select version_num from alembic_version')
+#session.close()
+
 # set up celery
 current_app = celery.Celery(__name__)
 
@@ -74,9 +87,9 @@ def main():
     elif args.v:
         argv.append('--loglevel=INFO')
     argv.append('-Q')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
     argv.append('-n')
-    argv.append(config.get('CELERY_QUEUE'))
+    argv.append(args.q)
 
     current_app.worker_main(argv)
 

apps/cic-cache/cic_cache/runnable/daemons/tracker.py

@@ -7,8 +7,14 @@ import argparse
 import sys
 import re
 
-# external imports
-import sqlalchemy
+# third-party imports
+import confini
+import celery
+import rlp
+import cic_base.config
+import cic_base.log
+import cic_base.argparse
+import cic_base.rpc
 from cic_eth_registry import CICRegistry
 from cic_eth_registry.error import UnknownContractError
 from chainlib.chain import ChainSpec
@@ -20,64 +26,36 @@ from chainlib.eth.block import (
 from hexathon import (
         strip_0x,
         )
-from chainsyncer.backend.sql import SQLBackend
-from chainsyncer.driver.head import HeadSyncer
-from chainsyncer.driver.history import HistorySyncer
+from chainsyncer.backend import SyncerBackend
+from chainsyncer.driver import (
+        HeadSyncer,
+        )
 from chainsyncer.db.models.base import SessionBase
 
 # local imports
-import cic_cache.cli
-from cic_cache.db import (
-        dsn_from_config,
-        add_tag,
-        )
+from cic_cache.db import dsn_from_config
 from cic_cache.runnable.daemons.filters import (
         ERC20TransferFilter,
-        FaucetFilter,
         )
 
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
+script_dir = os.path.realpath(os.path.dirname(__file__))
 
-# process args
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_sync
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-args = argparser.parse_args()
+logg = cic_base.log.create()
+argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
+#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
+args = cic_base.argparse.parse(argparser, logg)
+config = cic_base.config.create(args.c, args, args.env_prefix)
 
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
+cic_base.config.log(config)
+
+dsn = dsn_from_config(config)
 
-# connect to database
-dsn = dsn_from_config(config, 'cic_cache')
 SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
 
-# set up rpc
-rpc = cic_cache.cli.RPC.from_config(config)
-conn = rpc.get_default()
+chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
 
-# set up chain provisions
-chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
-registry = None
-try:
-    registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
-except UnknownContractError as e:
-    logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
-    sys.exit(1)
-logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
-
-
-def register_filter_tags(filters, session):
-    for f in filters:
-        tag = f.tag()
-        try:
-            add_tag(session, tag[0], domain=tag[1])
-            session.commit()
-            logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
-        except sqlalchemy.exc.IntegrityError:
-            session.rollback()
-            logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
+#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
+cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
 
 
 def main():
@@ -88,29 +66,24 @@ def main():
     r = rpc.do(o)
     block_offset = int(strip_0x(r), 16) + 1
 
-    logg.debug('current block height {}'.format(block_offset))
+    logg.debug('starting at block {}'.format(block_offset))
 
     syncers = []
 
-    syncer_backends = SQLBackend.resume(chain_spec, block_offset)
+    #if SyncerBackend.first(chain_spec):
+    #    backend = SyncerBackend.initial(chain_spec, block_offset)
+    syncer_backends = SyncerBackend.resume(chain_spec, block_offset)
 
     if len(syncer_backends) == 0:
-        initial_block_start = int(config.get('SYNCER_OFFSET'))
-        initial_block_offset = int(block_offset)
-        if config.get('SYNCER_NO_HISTORY'):
-            initial_block_start = initial_block_offset
-            initial_block_offset += 1
-        syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
-        logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
+        logg.info('found no backends to resume')
+        syncer_backends.append(SyncerBackend.initial(chain_spec, block_offset))
     else:
         for syncer_backend in syncer_backends:
             logg.info('resuming sync session {}'.format(syncer_backend))
 
-    for syncer_backend in syncer_backends:
-        syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
+    syncer_backends.append(SyncerBackend.live(chain_spec, block_offset+1))
 
-    syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
-    syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
+    syncers.append(HeadSyncer(syncer_backend))
 
     trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
     if trusted_addresses_src == None:
@@ -121,22 +94,11 @@ def main():
         logg.info('using trusted address {}'.format(address))
 
     erc20_transfer_filter = ERC20TransferFilter(chain_spec)
-    faucet_filter = FaucetFilter(chain_spec)
-
-    filters = [
-        erc20_transfer_filter,
-        faucet_filter,
-            ]
-
-    session = SessionBase.create_session()
-    register_filter_tags(filters, session)
-    session.close()
 
     i = 0
     for syncer in syncers:
         logg.debug('running syncer index {}'.format(i))
-        for f in filters:
-            syncer.add_filter(f)
+        syncer.add_filter(erc20_transfer_filter)
         r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
         sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
 

apps/cic-cache/cic_cache/tasks/tx.py

@@ -2,17 +2,14 @@
 import celery
 
 # local imports
-from cic_cache.cache import (
-        BloomCache,
-        DataCache,
-        )
+from cic_cache.cache import BloomCache
 from cic_cache.db.models.base import SessionBase
 
 celery_app = celery.current_app
 
 
 @celery_app.task(bind=True)
-def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
+def tx_filter(self, offset, limit, address=None, encoding='hex'):
     queue = self.request.delivery_info.get('routing_key')
 
     session = SessionBase.create_session()
@@ -20,9 +17,9 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     c = BloomCache(session)
     b = None
     if address == None:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
     else:
-        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
+        (lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
 
     session.close()
 
@@ -38,17 +35,4 @@ def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
     return o
 
 
-@celery_app.task(bind=True)
-def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
-    session = SessionBase.create_session()
 
-    c = DataCache(session)
-    b = None
-    if address == None:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
-    else:
-        (lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
-
-    session.close()
-
-    return (lowest_block, highest_block, tx_cache,)

apps/cic-cache/cic_cache/version.py

@@ -4,8 +4,8 @@ import semver
 version = (
         0,
         2,
-        1,
-        'alpha.3',
+        0,
+        'alpha.2',
         )
 
 version_object = semver.VersionInfo(

apps/cic-cache/config/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir =

apps/cic-cache/config/celery.ini

@@ -0,0 +1,3 @@
+[celery]
+broker_url = redis:///
+result_url = redis:///

apps/cic-cache/config/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+registry_address = 
+chain_spec =
+trust_address = 

apps/cic-cache/config/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=postgres
+PASSWORD=
+HOST=localhost
+PORT=5432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=

apps/cic-cache/config/docker/bancor.ini

@@ -0,0 +1,3 @@
+[bancor]
+registry_address = 
+dir = /usr/local/share/bancor

apps/cic-cache/config/docker/cic.ini

@@ -0,0 +1,4 @@
+[cic]
+chain_spec =
+registry_address =  
+trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

apps/cic-cache/config/docker/database.ini

@@ -0,0 +1,9 @@
+[database]
+NAME=cic_cache
+USER=grassroots
+PASSWORD=
+HOST=localhost
+PORT=63432
+ENGINE=postgresql
+DRIVER=psycopg2
+DEBUG=0

apps/cic-cache/config/docker/eth.ini

@@ -0,0 +1,2 @@
+[eth]
+provider = ws://localhost:63546

apps/cic-cache/config/eth.ini

@@ -0,0 +1,2 @@
+[eth]
+provider = ws://localhost:8545

apps/cic-cache/config/syncer.ini

@@ -0,0 +1,2 @@
+[syncer]
+loop_interval = 5

apps/cic-cache/config/test/bancor.ini

@@ -0,0 +1,2 @@
+[bancor]
+dir =

apps/cic-cache/config/test/cic.ini

@@ -1,3 +1,2 @@
 [cic]
 registry_address = 
-trust_address =

apps/cic-cache/config/test/database.ini

@@ -1,9 +1,9 @@
 [database]
-PREFIX=cic-cache-test
+NAME=cic-cache-test
 USER=postgres
 PASSWORD=
 HOST=localhost
 PORT=5432
 ENGINE=sqlite
 DRIVER=pysqlite
-DEBUG=1
+DEBUG=

apps/cic-cache/config/test/eth.ini

@@ -0,0 +1,5 @@
+[eth]
+#ws_provider = ws://localhost:8546
+#ttp_provider = http://localhost:8545
+provider = http://localhost:8545
+#chain_id = 

apps/cic-cache/doc/openapi/server.yml

@@ -1,4 +1,4 @@
-openapi: "3.0.2"
+openapi: "3.0.3"
 info:
   title: Grassroots Economics CIC Cache
   description: Cache of processed transaction data from Ethereum blockchain and worker queues
@@ -9,34 +9,17 @@ info:
     email: will@grassecon.org
   license:
     name: GPLv3
-  version: 0.2.0
+  version: 0.1.0
 
 paths:
-  /defaultlimit:
-    summary: The default limit value of result sets.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve default limit
-      operationId: limit.default
-      responses:
-        200:
-          description: Limit query successful
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Limit"
-
-  /tx:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+  /tx/{offset}/{limit}:
+    description: Bloom filter for batch of latest transactions
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get.latest
+      operationId: tx.get
       responses:
         200:
           description: Transaction query successful.
@@ -46,153 +29,27 @@ paths:
                 $ref: "#/components/schemas/BlocksBloom"
 
 
-  /tx/{limit}:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.limit
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from newest to oldest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
     parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{limit}/{offset}:
-    summary: Bloom filter for batch of latest transactions
-    description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.range
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from newest to oldest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
     - name: offset
       in: path
-      required: true
       schema:
         type: integer
         format: int32
-
-
-  /tx/{limit}/{offset}/{block_offset}:
-    summary: Bloom filter for batch of transactions since a particular block.
-    description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.latest.range.block.offset
-      responses:
-        200:
-          description: Transaction query successful. Results are ordered from oldest to newest.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
     - name: limit
       in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
       schema:
         type: integer
         format: int32
 
 
-  /tx/{limit}/{offset}/{block_offset}/{block_end}:
-    summary: Bloom filter for batch of transactions within a particular block range.
-    description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
+  /tx/{address}/{offset}/{limit}:
+    description: Bloom filter for batch of latest transactions by account
     get:
       tags:
         - transactions
       description:
         Retrieve transactions
-      operationId: tx.get.latest.range.block.range
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_end
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}:
-    summary: Bloom filter for batch of latest transactions by account.
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user
+      operationId: tx.get
       responses:
         200:
           description: Transaction query successful. 
@@ -201,30 +58,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/BlocksBloom"
 
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-
-
-  /tx/{address}/{limit}:
-    summary: Bloom filter for batch of latest transactions by account.
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user.limit
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
 
     parameters:
     - name: address
@@ -232,317 +65,26 @@ paths:
       required: true
       schema:
         type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /tx/{address}/{limit}/{offset}:
-    summary: Bloom filter for batch of latest transactions by account
-    description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: tx.get.user.range
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/BlocksBloom"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
     - name: offset
       in: path
-      required: true
       schema:
         type: integer
         format: int32
-
-
-  /txa:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    
-  /txa/{limit}:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.limit
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
     - name: limit
       in: path
-      required: true
       schema:
         type: integer
         format: int32
 
-
-  /txa/{limit}/{offset}:
-    summary: Cached data for latest transactions.
-    description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{limit}/{offset}/{block_offset}:
-    summary: Cached data for transactions since a particular block.
-    description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range.block.offset
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-  /txa/{limit}/{offset}/{block_offset}/{block_end}:
-    summary: Cached data for transactions within a particular block range.
-    description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.latest.range.block.range
-      responses:
-        200:
-          description: Transaction query successful.
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: block_end
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{address}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-
-
-  /txa/{address}/{limit}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user.limit
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
-  /txa/{address}/{limit}/{offset}:
-    summary: Cached data for batch of latest transactions by account.
-    description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
-    get:
-      tags:
-        - transactions
-      description:
-        Retrieve transactions
-      operationId: txa.get.user.range
-      responses:
-        200:
-          description: Transaction query successful. 
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TransactionList"
-
-    parameters:
-    - name: address
-      in: path
-      required: true
-      schema:
-        type: string
-    - name: limit
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-    - name: offset
-      in: path
-      required: true
-      schema:
-        type: integer
-        format: int32
-
-
 components:
   schemas:
-    Limit:
-      type: integer
-      format: int32
     BlocksBloom:
       type: object
       properties:
         low:
-          type: integer
+          type: int
           format: int32
           description: The lowest block number included in the filter
-        high:
-          type: integer
-          format: int32
-          description: The highest block number included in the filter
         block_filter:
           type: string
           format: byte
@@ -555,89 +97,6 @@ components:
           type: string
           description: Hashing algorithm (currently only using sha256)
         filter_rounds:
-          type: integer
+          type: int
           format: int32
           description: Number of hash rounds used to create the filter
-    TransactionList:
-      type: object
-      properties:
-        low:
-          type: integer
-          format: int32
-          description: The lowest block number included in the result set
-        high:
-          type: integer
-          format: int32
-          description: The highest block number included in the filter
-        data:
-          type: array
-          description: Cached transaction data
-          items:
-            $ref: "#/components/schemas/Transaction"
-    Transaction:
-      type: object
-      properties:
-        block_number:
-          type: integer
-          format: int64
-          description: Block number transaction was included in.
-        tx_hash:
-          type: string
-          description: Transaction hash, in hex.
-        date_block:
-          type: integer
-          format: int32
-          description: Block timestamp.
-        sender:
-          type: string
-          description: Spender address, in hex.
-        recipient:
-          type: string
-          description: Beneficiary address, in hex.
-        from_value:
-          type: integer
-          format: int64
-          description: Value deducted from spender's balance.
-        to_value:
-          type: integer
-          format: int64
-          description: Value added to beneficiary's balance.
-        source_token:
-          type: string
-          description: Network address of token in which `from_value` is denominated.
-        destination_token:
-          type: string
-          description: Network address of token in which `to_value` is denominated.
-        success:
-          type: boolean
-          description: Network consensus state on whether the transaction was successful or not.
-        tx_type:
-          type: string
-          enum:
-            - erc20.faucet
-            - faucet.give_to
-
-  examples:
-    data_last:
-      summary: Get the latest cached transactions, using the server's default limit. 
-      value: "/txa"
-
-    data_limit:
-      summary: Get the last 42 cached transactions. 
-      value: "/txa/42"
-
-    data_range:
-      summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
-      value: "/txa/42/13"
-
-    data_range_block_offset:
-      summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
-      value: "/txa/42/0/1337"
-
-    data_range_block_offset:
-      summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
-      value: "/txa/42/0/1337/1453"
-
-    data_range_block_range:
-      summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
-      value: "/txa/42/13/1337/1453"

apps/cic-cache/docker/Dockerfile

@@ -1,32 +1,53 @@
-ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
+FROM python:3.8.6-slim-buster
 
-FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
+#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
 
-COPY requirements.txt .
+WORKDIR /usr/src/cic-cache
 
-ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
-ARG EXTRA_PIP_ARGS=""
-ARG PIP_INDEX_URL=https://pypi.org/simple
+ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
+ARG root_requirement_file='requirements.txt'
 
-RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
-    pip install --index-url $PIP_INDEX_URL \
-    --pre \
-    --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-    -r requirements.txt
+#RUN apk update && \
+#	apk add gcc musl-dev gnupg libpq
+#RUN apk add postgresql-dev
+#RUN apk add linux-headers
+#RUN apk add libffi-dev
+RUN apt-get update && \
+	apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
 
+# Copy shared requirements from top of mono-repo
+RUN echo "copying root req file ${root_requirement_file}"
+RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a58
 
-COPY . .
-RUN pip install .  --extra-index-url $EXTRA_PIP_INDEX_URL
+COPY cic-cache/requirements.txt ./
+COPY cic-cache/setup.cfg \
+	 cic-cache/setup.py \
+ 	 ./
+COPY cic-cache/cic_cache/ ./cic_cache/
+COPY cic-cache/scripts/ ./scripts/
+COPY cic-cache/test_requirements.txt ./
+RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
+RUN pip install $pip_extra_index_url_flag .
+RUN pip install .[server]
+
+COPY cic-cache/tests/ ./tests/
+#COPY db/ cic-cache/db
+#RUN apk add postgresql-client
+
+# ini files in config directory defines the configurable parameters for the application
+# they can all be overridden by environment variables
+# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
+COPY cic-cache/config/ /usr/local/etc/cic-cache/
 
 # for db migrations
-COPY ./aux/wait-for-it/wait-for-it.sh ./
-COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
+COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
+
+RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
+	mkdir -p /usr/local/share/cic/solidity && \
+	cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
 
-COPY /docker/start_tracker.sh ./start_tracker.sh
-COPY /docker/db.sh ./db.sh
-RUN chmod 755 ./*.sh
 # Tracker
 # ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
 # Server
 # ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
-ENTRYPOINT []

apps/cic-cache/docker/db.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
->&2 echo executing database migration
-python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
-set +e

apps/cic-cache/docker/run_tests.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-
-set -e
-
-pip install --extra-index-url https://pip.grassrootseconomics.net \
---extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
--r test_requirements.txt 
-
-export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
-

apps/cic-cache/docker/start_tracker.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. ./db.sh
-
-if [ $? -ne "0" ]; then
-	>&2 echo db migrate fail
-	exit 1
-fi
-
-/usr/local/bin/cic-cache-trackerd $@

apps/cic-cache/requirements.txt

@@ -1,15 +1,13 @@
+cic-base~=0.1.2a62
 alembic==1.4.2
-confini~=0.5.3
+confini~=0.3.6rc3
 uwsgi==2.0.19.1
-moolb~=0.2.0
-cic-eth-registry~=0.6.6
+moolb~=0.1.0
+cic-eth-registry~=0.5.4a12
 SQLAlchemy==1.3.20
 semver==2.13.0
 psycopg2==2.8.6
 celery==4.4.7
 redis==3.5.3
-chainsyncer[sql]~=0.0.7
-erc20-faucet~=0.3.2
-chainlib-eth~=0.0.15
-eth-address-index~=0.2.4
-okota~=0.2.5
+chainlib~=0.0.2a5
+chainsyncer~=0.0.1a21

apps/cic-cache/scripts/migrate.py

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+import os
+import argparse
+import logging
+
+import alembic
+from alembic.config import Config as AlembicConfig
+import confini
+
+from cic_cache.db import dsn_from_config
+
+logging.basicConfig(level=logging.WARNING)
+logg = logging.getLogger()
+
+# BUG: the dbdir doesn't work after script install
+rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+dbdir = os.path.join(rootdir, 'cic_cache', 'db')
+migrationsdir = os.path.join(dbdir, 'migrations')
+
+config_dir = os.path.join('/usr/local/etc/cic-cache')
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('-c', type=str, default=config_dir, help='config file')
+argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
+argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
+argparser.add_argument('-v', action='store_true', help='be verbose')
+argparser.add_argument('-vv', action='store_true', help='be more verbose')
+args = argparser.parse_args()
+
+if args.vv:
+    logging.getLogger().setLevel(logging.DEBUG)
+elif args.v:
+    logging.getLogger().setLevel(logging.INFO)
+
+config = confini.Config(args.c, args.env_prefix)
+config.process()
+config.censor('PASSWORD', 'DATABASE')
+config.censor('PASSWORD', 'SSL')
+logg.debug('config:\n{}'.format(config))
+
+migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
+if not os.path.isdir(migrations_dir):
+    logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
+    migrations_dir = os.path.join(args.migrations_dir, 'default')
+
+# connect to database
+dsn = dsn_from_config(config)
+
+
+logg.info('using migrations dir {}'.format(migrations_dir))
+logg.info('using db {}'.format(dsn))
+ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
+ac.set_main_option('sqlalchemy.url', dsn)
+ac.set_main_option('script_location', migrations_dir)
+
+alembic.command.upgrade(ac, 'head')

apps/cic-cache/scripts/migrate_cic_cache.py

@@ -1,67 +0,0 @@
-#!/usr/bin/python3
-
-# standard imports
-import os
-import argparse
-import logging
-import re
-
-# external imports
-import alembic
-from alembic.config import Config as AlembicConfig
-import confini
-
-# local imports
-from cic_cache.db import dsn_from_config
-import cic_cache.cli
-
-logging.basicConfig(level=logging.WARNING)
-logg = logging.getLogger()
-
-# BUG: the dbdir doesn't work after script install
-rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
-dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-default_migrations_dir = os.path.join(dbdir, 'migrations')
-configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
-
-#config_dir = os.path.join('/usr/local/etc/cic-cache')
-
-arg_flags = cic_cache.cli.argflag_std_base
-local_arg_flags = cic_cache.cli.argflag_local_sync
-argparser = cic_cache.cli.ArgumentParser(arg_flags)
-argparser.process_local_flags(local_arg_flags)
-argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
-argparser.add_argument('-f', '--force', action='store_true', help='force action')
-argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
-args = argparser.parse_args()
-
-extra_args = {
-    'reset': None,
-    'force': None,
-    'migrations_dir': None,
-        }
-# process config
-config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
-
-migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
-if not os.path.isdir(migrations_dir):
-    logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
-    migrations_dir = os.path.join(args.migrations_dir, 'default')
-
-# connect to database
-dsn = dsn_from_config(config, 'cic_cache')
-
-
-logg.info('using migrations dir {}'.format(migrations_dir))
-logg.info('using db {}'.format(dsn))
-ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
-ac.set_main_option('sqlalchemy.url', dsn)
-ac.set_main_option('script_location', migrations_dir)
-
-if args.reset:
-    if not args.f:
-        if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
-            logg.error('user chickened out on requested reset, bailing')
-            sys.exit(1)
-    alembic.command.downgrade(ac, 'base')
-alembic.command.upgrade(ac, 'head')

apps/cic-cache/setup.cfg

@@ -1,7 +1,6 @@
 [metadata]
 name = cic-cache
 description = CIC Cache API and server
-version = 0.3.0a2
 author = Louis Holbrook
 author_email = dev@holbrook.no
 url = https://gitlab.com/grassrootseconomics/cic-eth
@@ -24,22 +23,19 @@ licence_files =
 
 [options]
 python_requires = >= 3.6
-include_package_data = True
 packages = 
 	cic_cache
 	cic_cache.tasks
 	cic_cache.db
 	cic_cache.db.models
-	cic_cache.cli
 	cic_cache.runnable
 	cic_cache.runnable.daemons
 	cic_cache.runnable.daemons.filters
 scripts =
-	./scripts/migrate_cic_cache.py
+	./scripts/migrate.py
 
 [options.entry_points]
 console_scripts =
 	cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
 	cic-cache-serverd = cic_cache.runnable.daemons.server:main
 	cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
-	cic-cache-list = cic_cache.runable.list:main

apps/cic-cache/setup.py

@@ -1,39 +1,38 @@
 from setuptools import setup
 
-# import configparser
+import configparser
 import os
+import time
 
-# import time
+from cic_cache.version import (
+        version_object,
+        version_string
+        )
 
-# from cic_cache.version import (
-#        version_object,
-#        version_string
-#        )
-#
-# class PleaseCommitFirstError(Exception):
-#    pass
-#
-# def git_hash():
-#    import subprocess
-#    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
-#    if len(git_diff.stdout) > 0:
-#        raise PleaseCommitFirstError()
-#    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
-#    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
-#    return git_hash_brief
-#
-# version_string = str(version_object)
-#
-# try:
-#    version_git = git_hash()
-#    version_string += '+build.{}'.format(version_git)
-# except FileNotFoundError:
-#    time_string_pair = str(time.time()).split('.')
-#    version_string += '+build.{}{:<09d}'.format(
-#            time_string_pair[0],
-#            int(time_string_pair[1]),
-#            )
-# print('final version string will be {}'.format(version_string))
+class PleaseCommitFirstError(Exception):
+    pass
+
+def git_hash():
+    import subprocess
+    git_diff = subprocess.run(['git', 'diff'], capture_output=True)
+    if len(git_diff.stdout) > 0:
+        raise PleaseCommitFirstError()
+    git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
+    git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
+    return git_hash_brief
+
+version_string = str(version_object)
+
+try:
+    version_git = git_hash()
+    version_string += '+build.{}'.format(version_git)
+except FileNotFoundError:
+    time_string_pair = str(time.time()).split('.')
+    version_string += '+build.{}{:<09d}'.format(
+            time_string_pair[0],
+            int(time_string_pair[1]),
+            )
+print('final version string will be {}'.format(version_string))
 
 requirements = []
 f = open('requirements.txt', 'r')
@@ -53,8 +52,9 @@ while True:
     test_requirements.append(l.rstrip())
 f.close()
 
+
 setup(
-    #    version=version_string,
+    version=version_string,
     install_requires=requirements,
     tests_require=test_requirements,
         )

apps/cic-cache/test_requirements.txt

@@ -4,7 +4,3 @@ pytest-mock==3.3.1
 pysqlite3==0.4.3
 sqlparse==0.4.1
 pytest-celery==0.0.0a1
-eth_tester==0.5.0b3
-py-evm==0.3.0a20
-sarafu-faucet~=0.0.7a1
-erc20-transfer-authorization~=0.3.6

apps/cic-cache/tests/cli/test_cli_args.py

@@ -1,40 +0,0 @@
-# standard imports
-import os
-
-# external imports
-import chainlib.cli 
-
-# local imports
-import cic_cache.cli
-
-script_dir = os.path.dirname(os.path.realpath(__file__))
-config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
-
-
-def test_argumentparserto_config():
-
-    argparser = cic_cache.cli.ArgumentParser()
-    
-    local_flags = 0xffff
-    argparser.process_local_flags(local_flags) 
-    argparser.add_argument('--foo', type=str)
-    args = argparser.parse_args([
-        '-q', 'baz',
-        '--offset', '13',
-        '--no-history',
-        '-r','0xdeadbeef',
-        '-vv',
-        '--foo', 'bar',
-        ])
-
-    extra_args = {
-            'foo': '_BARBARBAR',
-            }
-    config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
-
-    assert config.get('_BARBARBAR') == 'bar'
-    assert config.get('CELERY_QUEUE') == 'baz'
-    assert config.get('SYNCER_NO_HISTORY') == True
-    assert config.get('SYNCER_OFFSET') == 13
-    assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
-

apps/cic-cache/tests/cli/test_cli_celery.py

@@ -1,17 +0,0 @@
-# standard imports
-import tempfile
-
-# local imports
-import cic_cache.cli
-
-
-def test_cli_celery():
-    cf = tempfile.mkdtemp()
-
-    config = {
-            'CELERY_RESULT_URL': 'filesystem://' + cf,
-            }
-    cic_cache.cli.CeleryApp.from_config(config)
-
-    config['CELERY_BROKER_URL'] = 'filesystem://' + cf
-    cic_cache.cli.CeleryApp.from_config(config)

apps/cic-cache/tests/cli/test_cli_chain.py

@@ -1,68 +0,0 @@
-# external imports
-import pytest
-from chainlib.eth.gas import (
-        Gas,
-        RPCGasOracle,
-        )
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_latest,
-        Block,
-        )
-from chainlib.eth.pytest.fixtures_chain import default_chain_spec
-from chainlib.eth.pytest.fixtures_ethtester import *
-from cic_eth_registry.pytest.fixtures_contracts import *
-from hexathon import add_0x
-
-# local imports
-import cic_cache.cli
-
-
-@pytest.mark.xfail()
-def test_cli_rpc(
-        eth_rpc,
-        eth_signer,
-        default_chain_spec,
-        ):
-    config = {
-        'CHAIN_SPEC': str(default_chain_spec),
-        'RPC_HTTP_PROVIDER': 'http://localhost:8545',
-            }
-    rpc = cic_cache.cli.RPC.from_config(config, default_label='foo') 
-    conn = rpc.get_by_label('foo')
-    #o = block_latest()
-    #conn.do(o)
-
-
-def test_cli_chain(
-        default_chain_spec,
-        eth_rpc,
-        eth_signer,
-        contract_roles,
-        ):
-    ifc = cic_cache.cli.EthChainInterface()
-
-    nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
-    gas_oracle = RPCGasOracle(conn=eth_rpc)
-    c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
-    recipient = add_0x(os.urandom(20).hex())
-    (tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
-    r = eth_rpc.do(o)
-
-    o = ifc.tx_receipt(r)
-    r = eth_rpc.do(o)
-    assert r['status'] == 1
-
-    o = ifc.block_by_number(1)
-    block_src = eth_rpc.do(o)
-    block = ifc.block_from_src(block_src)
-    assert block.number == 1
-
-    with pytest.raises(KeyError):
-        assert block_src['gasUsed'] == 21000
-        assert block_src['gas_used'] == 21000
-
-    block_src = ifc.src_normalize(block_src)
-    assert block_src['gasUsed'] == 21000
-    assert block_src['gas_used'] == 21000
-

apps/cic-cache/tests/conftest.py

@@ -3,15 +3,11 @@ import os
 import sys
 import datetime
 
-# external imports
+# third-party imports
 import pytest
-import moolb
-from chainlib.encode import TxHexNormalizer
 
 # local imports
 from cic_cache import db
-from cic_cache import BloomCache
-from cic_cache.cache import DEFAULT_FILTER_SIZE
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.dirname(script_dir)
@@ -43,8 +39,6 @@ def txs(
         list_tokens,
         ):
 
-    tx_normalize = TxHexNormalizer()
-
     session = init_database
 
     tx_number = 13 
@@ -57,16 +51,17 @@ def txs(
         tx_hash_first,
         list_defaults['block'],
         tx_number,
-        tx_normalize.wallet_address(list_actors['alice']),
-        tx_normalize.wallet_address(list_actors['bob']),
-        tx_normalize.executable_address(list_tokens['foo']),
-        tx_normalize.executable_address(list_tokens['foo']),
+        list_actors['alice'],
+        list_actors['bob'],
+        list_tokens['foo'],
+        list_tokens['foo'],
         1024,
         2048,
         True,
         dt.timestamp(),
             )
 
+
     tx_number = 42
     tx_hash_second = '0x' + os.urandom(32).hex()
     tx_signed_second = '0x' + os.urandom(128).hex()
@@ -77,10 +72,10 @@ def txs(
         tx_hash_second,
         list_defaults['block']-1,
         tx_number,
-        tx_normalize.wallet_address(list_actors['diane']),
-        tx_normalize.wallet_address(list_actors['alice']),
-        tx_normalize.executable_address(list_tokens['foo']),
-        tx_normalize.wallet_address(list_tokens['foo']),
+        list_actors['diane'],
+        list_actors['alice'],
+        list_tokens['foo'],
+        list_tokens['foo'],
         1024,
         2048,
         False,
@@ -89,64 +84,3 @@ def txs(
     
     session.commit()
 
-    return [
-            tx_hash_first,
-            tx_hash_second,
-            ]
-
-
-@pytest.fixture(scope='function')
-def more_txs(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        ):
-
-    session = init_database
-
-    tx_normalize = TxHexNormalizer()
-
-    tx_number = 666
-    tx_hash = '0x' + os.urandom(32).hex()
-    tx_signed = '0x' + os.urandom(128).hex()
-    nonce = 3
-
-    dt = datetime.datetime.utcnow()
-    dt += datetime.timedelta(hours=1)
-    db.add_transaction(
-        session,
-        tx_hash,
-        list_defaults['block']+2,
-        tx_number,
-        tx_normalize.wallet_address(list_actors['alice']),
-        tx_normalize.wallet_address(list_actors['diane']),
-        tx_normalize.executable_address(list_tokens['bar']),
-        tx_normalize.executable_address(list_tokens['bar']),
-        2048,
-        4096,
-        False,
-        dt.timestamp(),
-        )
-
-    session.commit()
-
-    return [tx_hash] + txs
-
-
-@pytest.fixture(scope='function')
-def tag_txs(
-        init_database,
-        txs,
-        ):
-
-    db.add_tag(init_database, 'taag', domain='test')
-    init_database.commit()
-
-    db.tag_transaction(init_database, txs[1], 'taag', domain='test')
-
-
-@pytest.fixture(scope='session')
-def zero_filter():
-    return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)

apps/cic-cache/tests/filters/conftest.py

@@ -1,3 +0,0 @@
-from chainlib.eth.pytest import *
-from cic_eth_registry.pytest.fixtures_tokens import *
-

apps/cic-cache/tests/filters/test_erc20.py

@@ -1,173 +0,0 @@
-# standard imports
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-from sqlalchemy import text
-from chainlib.eth.tx import Tx
-from chainlib.eth.block import Block
-from chainlib.chain import ChainSpec
-from chainlib.eth.error import RequestMismatchException
-from hexathon import (
-        strip_0x,
-        add_0x,
-        )
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
-from cic_cache.runnable.daemons.filters.base import TagSyncFilter
-
-logg = logging.getLogger()
-
-
-def test_base_filter_str(
-        init_database,
-        ):
-        f = TagSyncFilter('foo')
-        assert 'foo' == str(f)
-        f = TagSyncFilter('foo', domain='bar')
-        assert 'bar.foo' == str(f)
-
-
-
-def test_erc20_filter(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash
-
-
-def test_erc20_filter_nocontract(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = 'a9059cbb'
-    data += strip_0x(list_actors['alice'])
-    data += '1000'.ljust(64, '0')
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': os.urandom(20).hex(),
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
-
-
-@pytest.mark.parametrize(
-        'contract_method,contract_input,expected_exception',
-        [
-            ('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
-            ('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
-            ('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
-            ],
-        )
-def test_erc20_filter_bogus(
-        eth_rpc,
-        foo_token,
-        init_database,
-        list_defaults,
-        list_actors,
-        tags,
-        contract_method,
-        contract_input,
-        expected_exception,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = ERC20TransferFilter(chain_spec)
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    # incomplete args
-    data = contract_method
-    data += contract_input 
-    block = Block({
-        'hash': os.urandom(32).hex(),
-        'number': 42,
-        'timestamp': datetime.datetime.utcnow().timestamp(),
-        'transactions': [],
-        })
-
-    tx = Tx({
-        'to': foo_token,
-        'from': list_actors['bob'],
-        'data': data,
-        'value': 0,
-        'hash': os.urandom(32).hex(),
-        'nonce': 13,
-        'gasPrice': 10000000,
-        'gas': 123456,
-            })
-    block.txs.append(tx)
-    tx.block = block
-
-    assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)

apps/cic-cache/tests/filters/test_faucet.py

@@ -1,71 +0,0 @@
-# standard imports
-import logging
-
-# external imports
-from chainlib.chain import ChainSpec
-from chainlib.eth.nonce import RPCNonceOracle
-from chainlib.eth.block import (
-        block_by_hash,
-        Block,
-        )
-from chainlib.eth.tx import (
-        receipt,
-        unpack,
-        transaction,
-        Tx,
-        )
-from hexathon import strip_0x
-from erc20_faucet.faucet import SingleShotFaucet
-from sqlalchemy import text
-
-# local imports
-from cic_cache.db import add_tag
-from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
-
-logg = logging.getLogger()
-
-
-def test_filter_faucet(
-        eth_rpc,
-        eth_signer,
-        foo_token,
-        faucet_noregistry,
-        init_database,
-        list_defaults,
-        contract_roles,
-        agent_roles,
-        tags,
-        ):
-
-    chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
-
-    fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
-
-    add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
-
-    nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
-    c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
-    (tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
-    r = eth_rpc.do(o)
-
-    tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
-
-    o = receipt(r)
-    r = eth_rpc.do(o)
-    rcpt = Tx.src_normalize(r)
-
-    assert r['status'] == 1
-
-    o = block_by_hash(r['block_hash'])
-    r = eth_rpc.do(o)
-    block_object = Block(r)
-
-    tx = Tx(tx_src, block_object)
-    tx.apply_receipt(rcpt)
-
-    r = fltr.filter(eth_rpc, block_object, tx, init_database)
-    assert r
-
-    s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
-    r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
-    assert r[0] == tx.hash

apps/cic-cache/tests/fixtures_config.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-# external imports
+# third-party imports
 import pytest
 import confini
 
@@ -13,9 +13,8 @@ logg = logging.getLogger(__file__)
 
 @pytest.fixture(scope='session')
 def load_config():
-    config_dir = os.path.join(root_dir, 'config/test')
-    schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
-    conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
+    config_dir = os.path.join(root_dir, '.config/test')
+    conf = confini.Config(config_dir, 'CICTEST')
     conf.process()
     logg.debug('config {}'.format(conf))
     return conf

apps/cic-cache/tests/fixtures_database.py

@@ -3,16 +3,13 @@ import os
 import logging
 import re
 
-# external imports
+# third-party imports
 import pytest
 import sqlparse
-import alembic
-from alembic.config import Config as AlembicConfig
 
 # local imports
 from cic_cache.db.models.base import SessionBase
 from cic_cache.db import dsn_from_config
-from cic_cache.db import add_tag
 
 logg = logging.getLogger(__file__)
 
@@ -24,19 +21,16 @@ def database_engine(
     if load_config.get('DATABASE_ENGINE') == 'sqlite':
         SessionBase.transactional = False
         SessionBase.poolable = False
-        name = 'cic_cache'
-        database_name = name
-        if load_config.get('DATABASE_PREFIX'):
-            database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
         try:
-            os.unlink(database_name)
+            os.unlink(load_config.get('DATABASE_NAME'))
         except FileNotFoundError:
             pass
-    dsn = dsn_from_config(load_config, name)
-    SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
+    dsn = dsn_from_config(load_config)
+    SessionBase.connect(dsn)
     return dsn
 
 
+# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
 @pytest.fixture(scope='function')
 def init_database(
         load_config,
@@ -44,23 +38,52 @@ def init_database(
         ):
 
     rootdir = os.path.dirname(os.path.dirname(__file__))
-    dbdir = os.path.join(rootdir, 'cic_cache', 'db')
-    migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
-    if not os.path.isdir(migrationsdir):
-        migrationsdir = os.path.join(dbdir, 'migrations', 'default')
-    logg.info('using migrations directory {}'.format(migrationsdir))
+    schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
+
+    if load_config.get('DATABASE_ENGINE') == 'sqlite':
+        rconn = SessionBase.engine.raw_connection()
+        f = open(os.path.join(schemadir, 'db.sql'))
+        s = f.read()
+        f.close()
+        rconn.executescript(s)
+
+    else:
+        rconn = SessionBase.engine.raw_connection()
+        rcursor = rconn.cursor()
+
+        #rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
+        #rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
+
+        f = open(os.path.join(schemadir, 'db.sql'))
+        s = f.read()
+        f.close()
+        r = re.compile(r'^[A-Z]', re.MULTILINE)
+        for l in sqlparse.parse(s):
+            strl = str(l)
+            # we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
+            if not re.search(r, strl):
+                logg.warning('skipping parsed query line {}'.format(strl))
+                continue
+            rcursor.execute(strl)
+        rconn.commit()
+
+        rcursor.execute('SET search_path TO public')
+
+# this doesn't work when run separately, no idea why
+# functions have been manually added to original schema from cic-eth 
+#        f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
+#        s = f.read()
+#        f.close()
+#        rcursor.execute(s)
+# 
+#        f = open(os.path.join(schemadir, 'proc_balances.sql'))
+#        s = f.read()
+#        f.close()
+#        rcursor.execute(s)
+
+        rcursor.close()
 
     session = SessionBase.create_session()
-
-    ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
-    ac.set_main_option('sqlalchemy.url', database_engine)
-    ac.set_main_option('script_location', migrationsdir)
-
-    alembic.command.downgrade(ac, 'base')
-    alembic.command.upgrade(ac, 'head')
-
-    session.commit()
-
     yield session
     session.commit()
     session.close()
@@ -93,14 +116,3 @@ def list_defaults(
     return {
         'block': 420000,
     }
-
-
-@pytest.fixture(scope='function')
-def tags(
-        init_database,
-        ):
-
-    add_tag(init_database, 'foo') 
-    add_tag(init_database, 'baz', domain='bar') 
-    add_tag(init_database, 'xyzzy', domain='bar') 
-    init_database.commit()

apps/cic-cache/tests/test_api.py

@@ -1,31 +0,0 @@
-# standard imports
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache.runnable.daemons.query import process_transactions_all_data
-
-
-def test_api_all_data(
-        init_database,
-        txs,
-        ):
-
-    env = {
-        'PATH_INFO': '/txa/100/0/410000/420000',
-        'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    j = process_transactions_all_data(init_database, env)
-    o = json.loads(j[1])
-    
-    assert len(o['data']) == 2
-
-    env = {
-        'PATH_INFO': '/txa/100/0/420000/410000',
-        'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-   
-    with pytest.raises(ValueError):
-        j = process_transactions_all_data(init_database, env)

apps/cic-cache/tests/test_cache.py

@@ -4,14 +4,11 @@ import datetime
 import logging
 import json
 
-# external imports
+# third-party imports
 import pytest
-from chainlib.encode import TxHexNormalizer
 
 # local imports
-from cic_cache import db
 from cic_cache import BloomCache
-from cic_cache.cache import DataCache
 
 logg = logging.getLogger()
 
@@ -20,6 +17,7 @@ def test_cache(
         init_database,
         list_defaults,
         list_actors,
+        list_tokens,
         txs,
         ):
 
@@ -35,237 +33,3 @@ def test_cache(
 
     assert b[0] == list_defaults['block'] - 1
 
-
-def test_cache_data(
-        init_database,
-        txs,
-        tag_txs,
-        ):
-
-    session = init_database
-
-    c = DataCache(session)
-    b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
-
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == txs[0]
-    assert b[2][0]['tx_type'] == 'unknown'
-    assert b[2][1]['tx_type'] == 'test.taag'
-
-
-def test_cache_ranges(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    tx_normalize = TxHexNormalizer()
-
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = BloomCache(session)
-    b = c.load_transactions(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-
-    # now check when supplying account
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account(account, 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    account = tx_normalize.wallet_address(list_actors['bob'])
-    b = c.load_transactions_account(account, 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-
-    account = tx_normalize.wallet_address(list_actors['diane'])
-    b = c.load_transactions_account(account, 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-
-    # add block filter to the mix
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-
-    account = tx_normalize.wallet_address(list_actors['bob'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-
-    account = tx_normalize.wallet_address(list_actors['diane'])
-    b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-
-
-def test_cache_ranges_data( 
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        more_txs,
-        ):
-
-    session = init_database
-       
-    tx_normalize = TxHexNormalizer()
-
-    oldest = list_defaults['block'] - 1
-    mid = list_defaults['block']
-    newest = list_defaults['block'] + 2
-
-    c = DataCache(session)
-
-    b = c.load_transactions_with_data(0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(1, 2)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 2)
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 1)
-    assert b[0] == newest
-    assert b[1] == newest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, oldest=True)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[0]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[1]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
-    assert b[0] == oldest
-    assert b[1] == mid
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[2]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    # now check when supplying account
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account_with_data(account, 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 3
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-    assert b[2][2]['tx_hash'] == more_txs[2]
-
-    account = tx_normalize.wallet_address(list_actors['bob'])
-    b = c.load_transactions_account_with_data(account, 0, 100)
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    account = tx_normalize.wallet_address(list_actors['diane'])
-    b = c.load_transactions_account_with_data(account, 0, 100)
-    assert b[0] == oldest
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[2]
-
-    # add block filter to the mix
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    account = tx_normalize.wallet_address(list_actors['alice'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == newest
-    assert len(b[2]) == 2
-    assert b[2][0]['tx_hash'] == more_txs[0]
-    assert b[2][1]['tx_hash'] == more_txs[1]
-
-    account = tx_normalize.wallet_address(list_actors['bob'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == mid
-    assert b[1] == mid
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[1]
-
-    account = tx_normalize.wallet_address(list_actors['diane'])
-    b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
-    assert b[0] == oldest
-    assert b[1] == oldest
-    assert len(b[2]) == 1
-    assert b[2][0]['tx_hash'] == more_txs[2]

apps/cic-cache/tests/test_query.py

@@ -1,232 +0,0 @@
-# standard imports
-import logging
-import json
-import base64
-import copy
-import re
-
-# external imports
-import pytest
-from hexathon import strip_0x
-
-# local imports
-from cic_cache.runnable.daemons.query import *
-
-logg = logging.getLogger()
-
-
-
-@pytest.mark.parametrize(
-        'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
-        [
-            ('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
-            ('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
-            ('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
-            ('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
-            ('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
-            ('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
-            ],
-        )
-def test_query_regex(
-        list_actors,
-        query_path_prefix,
-        query_role,
-        query_address_index,
-        query_offset,
-        query_offset_index,
-        query_limit,
-        query_limit_index,
-        match_re,
-        ):
-
-        paths = []
-        path = query_path_prefix
-        query_address = None
-        if query_role != None:
-            query_address = strip_0x(list_actors[query_role])
-            paths.append(path + '0x' + query_address)
-            paths.append(path + query_address)
-        if query_offset != None:
-            if query_limit != None:
-                for i in range(len(paths)-1):
-                    paths[i] += '/{}/{}'.format(query_offset, query_limit)
-            else:
-                for i in range(len(paths)-1):
-                    paths[i] += '/' + str(query_offset)
-
-        for i in range(len(paths)):
-            paths.append(paths[i] + '/')
-
-        for p in paths:
-            logg.debug('testing path {} against {}'.format(p, match_re))
-            m = re.match(match_re, p)
-            l = len(m.groups())
-            logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
-            for i in range(l+1):
-                logg.debug('group {} {}'.format(i, m[i]))
-            if m.lastindex >= query_offset_index:
-                assert query_offset == int(m[query_offset_index + 1])
-            if m.lastindex >= query_limit_index:
-                assert query_limit == int(m[query_limit_index + 1])
-            if query_address_index != None:
-                match_address = strip_0x(m[query_address_index + 1])
-                assert query_address == match_address
-
-
-
-@pytest.mark.parametrize(
-        'role_name, query_offset, query_limit, query_match',
-        [ 
-            ('alice', None, None, [(420000, 13), (419999, 42)]),
-            ('alice', None, 1, [(420000, 13)]),
-            ('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
-            ('alice', 2, None, []), # 420000 == list_defaults['block']
-            ],
-        )
-def test_query_process_txs_account(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        role_name,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    actor = None
-    try:
-        actor = list_actors[role_name]
-    except KeyError:
-        actor = os.urandom(20).hex()
-    path_info = '/tx/user/0x' + strip_0x(actor)
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit == None:
-        query_limit = 100
-    path_info += '/' + str(query_limit)
-    if query_offset == None:
-        path_info += '/0'
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_account_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_offset, query_limit, query_match',
-        [ 
-            (None, 2, [(420000, 13), (419999, 42)]),
-            (0, 1, [(420000, 13)]),
-            (1, 1, [(419999, 42)]),
-            (2, 0, []),
-            ],
-        )
-def test_query_process_txs_bloom(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_offset,
-        query_limit,
-        query_match,
-        ):
-
-    path_info = '/tx'
-    if query_offset != None:
-        path_info += '/' + str(query_offset)
-    if query_limit != None:
-        if query_offset == None:
-            path_info += '/0'
-        path_info += '/' + str(query_limit)
-    env = {
-            'PATH_INFO': path_info,
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_bloom(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
-    zero_filter_data = zero_filter.to_bytes()
-    if len(query_match) == 0:
-        assert block_filter_data == zero_filter_data
-        return
-
-    assert block_filter_data != zero_filter_data
-    block_filter = copy.copy(zero_filter)
-    block_filter.merge(block_filter_data)
-    block_filter_data = block_filter.to_bytes()
-    assert block_filter_data != zero_filter_data
-
-    for (block, tx) in query_match:
-        block = block.to_bytes(4, byteorder='big')
-        assert block_filter.check(block)
-
-
-@pytest.mark.parametrize(
-        'query_block_start, query_block_end, query_match_count',
-        [ 
-            (1, 42, 0),
-            (420000, 420001, 1),
-            (419999, 419999, 1), # matches are inclusive
-            (419999, 420000, 2),
-            (419999, 420001, 2),
-            ],
-        )
-def test_query_process_txs_data(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        zero_filter,
-        query_block_start,
-        query_block_end,
-        query_match_count,
-        ):
-
-    path_info = '/txa/100/0'
-    if query_block_start != None:
-        path_info += '/' + str(query_block_start)
-    if query_block_end != None:
-        if query_block_start == None:
-            path_info += '/0'
-        path_info += '/' + str(query_block_end)
-    env = {
-            'PATH_INFO': path_info,
-            'HTTP_X_CIC_CACHE_MODE': 'all',
-            }
-    logg.debug('using path {}'.format(path_info))
-    r = process_transactions_all_data(init_database, env)
-    assert r != None
-
-    o = json.loads(r[1])
-    logg.debug('oo {}'.format(o))
-    assert len(o['data']) == query_match_count

apps/cic-cache/tests/test_tag.py

@@ -1,37 +0,0 @@
-import os
-import datetime
-import logging
-import json
-
-# external imports
-import pytest
-
-# local imports
-from cic_cache.db import tag_transaction
-
-logg = logging.getLogger()
-
-
-def test_cache(
-        init_database,
-        list_defaults,
-        list_actors,
-        list_tokens,
-        txs,
-        tags,
-        ):
-
-    tag_transaction(init_database, txs[0], 'foo')
-    tag_transaction(init_database, txs[0], 'baz', domain='bar')
-    tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
-    assert r[0][0] == txs[0]
-
-
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
-    assert r[0][0] == txs[0]
-
-    
-    r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
-    assert r[0][0] == txs[1]

apps/cic-cache/tests/testdata/config/test.ini

@@ -1,2 +0,0 @@
-[foo]
-bar_baz = xyzzy