Compare commits
408 Commits
lash/healt
...
master
Author | SHA1 | Date | |
---|---|---|---|
3dcfbe59fe | |||
0dd21f3970 | |||
6ca8632cde | |||
b7dc290992 | |||
e69801ea08 | |||
50a596e707 | |||
cb61e45e4c | |||
e5b06b18d7 | |||
|
5d1a30021a | ||
|
ade3f4e917 | ||
c3e924ae8f | |||
ff4c42dc24 | |||
b8cd7eec56 | |||
837e8da650 | |||
fe3f2c2549 | |||
46f25e5678 | |||
03c7c1ddbc | |||
|
104ff8a76a | ||
b5653a704c | |||
|
3b6ea6db77 | ||
7d3ff89fe5 | |||
|
dceae4e5d8 | ||
46a4ccfd9c | |||
|
d7c4cb71eb | ||
b15cfee1c9 | |||
efb1967f46 | |||
019824d1f4 | |||
d4c7fd3d7e | |||
4b87a40cc2 | |||
9f9d557c73 | |||
9814da78b8 | |||
cd102807a6 | |||
c3c43c28a5 | |||
e132e534d3 | |||
e426f7b451 | |||
b368b022c1 | |||
bd266ac8dd | |||
69dbbcb6a9 | |||
dd709d7d47 | |||
a9c84bb2b1 | |||
|
85aa1f3066 | ||
28936a58fe | |||
4b8096ff49 | |||
c9eb3e32da | |||
1be5a92f44 | |||
|
8855ccd3d2 | ||
|
09dfdbb38a | ||
1abb642361 | |||
93bcbd7d51 | |||
|
818899670a | ||
|
1882910a8e | ||
3cc909c936 | |||
60b6e1abdb | |||
|
9c7e72f71c | ||
|
e3acc1757a | ||
|
8250b15d32 | ||
|
31d7cf5789 | ||
|
2544c159c2 | ||
|
7691d9a127 | ||
|
a2a3634683 | ||
|
fe0835a4e7 | ||
d8f51c5bdd | |||
|
13fb67d2d8 | ||
|
8f1afa094d | ||
|
1d9f134125 | ||
|
b6a4bab1c8 | ||
805fc56c7b | |||
|
e3d39a2144 | ||
|
90176f2806 | ||
f42f223ce9 | |||
|
045a279ac2 | ||
|
6b59c87f68 | ||
|
9ee42b908d | ||
3343c4163e | |||
|
f5bfc8ace2 | ||
8a9d2ee0be | |||
3608fd1fc7 | |||
0d275f358b | |||
3aef2aa65f | |||
5644baefb2 | |||
1a7c4deab6 | |||
0389d8623d | |||
cf64387d81 | |||
79bcc8a9f1 | |||
7b57f1b4c2 | |||
76b8519637 | |||
e89aec76fa | |||
a138a0ec75 | |||
5128c7828c | |||
2f005195e5 | |||
fb8db3ffd2 | |||
b5f647c4aa | |||
6019143ba1 | |||
610440b722 | |||
|
d65455fc29 | ||
43f8d1c30c | |||
b855211eed | |||
1e0c475f39 | |||
|
3e6cf594e3 | ||
b8f79a2dd1 | |||
540c2fd950 | |||
b9b06eced8 | |||
949bb29379 | |||
|
0468906601 | ||
|
471243488e | ||
3c4acd82ff | |||
e07f992c5a | |||
17e95cb19c | |||
3c3a97ce15 | |||
a492be4927 | |||
|
1f555748b0 | ||
8aa4d20eea | |||
|
90cf24dcee | ||
|
75b711dbd5 | ||
c21c1eb2ef | |||
eb5e612105 | |||
e017d11770 | |||
e327af68e1 | |||
92cc6a3f27 | |||
f42bf7754a | |||
|
7342927e91 | ||
17333af88f | |||
6a68d2ed32 | |||
|
ef77f4c99a | ||
|
56dbe8a502 | ||
|
2dc8ac6a12 | ||
|
0ced68e224 | ||
2afb20e715 | |||
3b0113d0e4 | |||
|
ebf4743a84 | ||
|
3bf92e7a8a | ||
f0b4c42c68 | |||
|
b62d00180c | ||
|
a49978cc36 | ||
1b0ee269d0 | |||
aa2f363b27 | |||
2a24ce6938 | |||
938a10b5c3 | |||
|
76e33e578b | ||
|
2ec4262734 | ||
|
7684fe3883 | ||
|
995a148c6a | ||
|
511e099689 | ||
|
f877218c55 | ||
8ac9a1e99a | |||
c4cb095a29 | |||
05b8bbbbca | |||
1ce32fbbe0 | |||
3fd5e77e2c | |||
e27a49ef33 | |||
|
fffb2bc3f4 | ||
|
8910fb0759 | ||
|
c84239c820 | ||
|
452047b900 | ||
|
b8be457c41 | ||
|
0ec9813e5f | ||
|
defa7797dc | ||
|
bb3d38a1f9 | ||
3be1c1b33d | |||
|
d6c763f2d7 | ||
b7942ddcfa | |||
8de5dc1540 | |||
fad0a4b580 | |||
0672a17d2e | |||
f764b73f66 | |||
806b82504f | |||
ac76e14129 | |||
1c78f4d6d6 | |||
0d6e228f8a | |||
7a3cb7ab75 | |||
992c7b4022 | |||
f19173001e | |||
|
f82bb4515d | ||
24e6db7d87 | |||
ecdfb9bc5a | |||
30415ac997 | |||
d5a8b77349 | |||
|
ed2521b582 | ||
|
395930106a | ||
|
ee1452e530 | ||
|
8cdaf9f28a | ||
|
402b968b6d | ||
|
aa13517534 | ||
|
884b18f2f1 | ||
|
494a8f3e88 | ||
|
1214f605a7 | ||
|
0783a6001c | ||
|
f9594b766a | ||
|
561ae62d5e | ||
|
d6782abbcc | ||
|
8f173fa30b | ||
|
3741cb3283 | ||
|
54dd5acb62 | ||
c52885a016 | |||
f0dd257e05 | |||
e8c870d230 | |||
4bb36a448d | |||
231163e2fc | |||
e599933ef8 | |||
266bc3362d | |||
8350381754 | |||
6ddeacf036 | |||
fe017d2b0f | |||
d7973436e6 | |||
5025c31af6 | |||
|
021c736707 | ||
|
e3028a5060 | ||
|
5f6c57647f | ||
|
ed029a936c | ||
|
c559bb2fee | ||
|
9b79034ed3 | ||
b7d5c6799f | |||
eef8bb2cf7 | |||
cf96fee430 | |||
9740963431 | |||
|
a3c4932488 | ||
|
aa667951be | ||
|
c2459cfd65 | ||
|
e7102ff02d | ||
a942c785f6 | |||
70704b09ec | |||
|
a075c55957 | ||
|
6464f651ec | ||
|
5145282946 | ||
|
1e87f2ed31 | ||
|
c852f41d76 | ||
|
f8e68cff96 | ||
7027d77836 | |||
d356f8167d | |||
753d21fe95 | |||
3b6e031746 | |||
b1d5d45eef | |||
53317cb912 | |||
18382a1f35 | |||
29e91fafab | |||
5b20a9a24a | |||
a252195bdc | |||
|
f1be3b633c | ||
|
e59a71188c | ||
1d0eb06f2f | |||
57127132b5 | |||
0bf2c35fcd | |||
d046595764 | |||
9dd7ec88fd | |||
282fd2ff52 | |||
8f85598861 | |||
8529c349ca | |||
4368d2bf59 | |||
da3c812bf5 | |||
82b1e87462 | |||
e13c423daf | |||
56b3bd751d | |||
4f41c5bacf | |||
07583f0c3b | |||
0ae912082c | |||
094f4d4298 | |||
|
9471b1d8ab | ||
|
57100366d8 | ||
71e0973020 | |||
|
12ab5c2f66 | ||
a804552620 | |||
0319fa6076 | |||
91dfc51d54 | |||
4fd861f080 | |||
|
28de7a4eac | ||
|
a31e79b0f7 | ||
eb2f71aee0 | |||
|
e5b1352970 | ||
|
89b90da5d2 | ||
9607994c31 | |||
0da617d29e | |||
56bcad16a5 | |||
77d9936e39 | |||
|
72aeefc78b | ||
|
fab9b0c520 | ||
9566f8c8e2 | |||
007d7a5121 | |||
fc20849aff | |||
1605e53216 | |||
200fdf0e3c | |||
022db04198 | |||
1c17048981 | |||
|
04c0963f33 | ||
|
096ed9bc27 | ||
1a931eced4 | |||
ed9e032890 | |||
|
69ae9b7c07 | ||
|
634d3fb401 | ||
|
65f722b291 | ||
|
0ad0f9981c | ||
|
5fb0f4a2e9 | ||
|
41a96b5584 | ||
|
d0f2bc0120 | ||
|
e2946052e0 | ||
|
546d69f1e9 | ||
|
fbf7351238 | ||
|
b886384fa8 | ||
|
277033f3b5 | ||
4ae094fd30 | |||
cb239f112a | |||
|
d971a6eded | ||
|
b0a6df0177 | ||
|
92c9df4e19 | ||
|
9c49d568e0 | ||
|
d7113f3923 | ||
|
c569fe4b17 | ||
1c650df27d | |||
a31b7bc9cd | |||
|
78ff58c1a2 | ||
1676addbeb | |||
1efc25ac15 | |||
|
db2ec0dcfa | ||
5148e6428b | |||
|
0c186ed968 | ||
|
c44439bd90 | ||
|
0411603078 | ||
eee895ea71 | |||
|
a5ca898532 | ||
|
6d8508aebf | ||
|
f8f66984d2 | ||
|
0f02dd1b7c | ||
63a4a82ab0 | |||
949c1070a9 | |||
5d9fbe9b64 | |||
873a3f082a | |||
7b408cf564 | |||
|
9dfbd7034c | ||
|
235f5cede8 | ||
|
0a59539f9a | ||
|
60b36945df | ||
dae6526677 | |||
1e94a516c2 | |||
e8512ebbae | |||
f2c955c60b | |||
17b3b27d81 | |||
1cb172b8bf | |||
|
9d47e4c764 | ||
|
c68cc318ab | ||
|
af99ac823a | ||
|
06652eb30f | ||
|
f66f913307 | ||
|
8bf1364864 | ||
0d6d7179eb | |||
e7f48f3ce0 | |||
|
b252fab018 | ||
|
4667916d80 | ||
1f668384cc | |||
123dc55687 | |||
|
0b4d8d5937 | ||
|
ed6bef4052 | ||
|
6a8a356f09 | ||
5ec0b67496 | |||
7d935bcbc3 | |||
fd69a3c6bb | |||
|
298bcf89e5 | ||
|
5d3d773f41 | ||
|
e71b2411d0 | ||
|
b4bfb76634 | ||
aab5c8bf85 | |||
e1564574f7 | |||
13253a2dcc | |||
9020fe1000 | |||
a2e7d2973c | |||
82f650e81d | |||
e77940d0de | |||
1df62717ef | |||
c4919d56b1 | |||
6d44863a49 | |||
|
b02cdee1bd | ||
|
75bf8f15be | ||
8db76dc0a8 | |||
a3261f2f0e | |||
850dd15451 | |||
0c56e84704 | |||
63cd8a4aab | |||
|
2c326f62ae | ||
9ed62c58ae | |||
|
04e9f45feb | ||
|
9126a75c4a | ||
|
1bc29588a1 | ||
e6d57d3bbb | |||
f64ff1290c | |||
|
d5cbe9d113 | ||
|
5663741ed4 | ||
|
0f6615a925 | ||
|
aa15353d68 | ||
|
f7a69830ba | ||
|
7428420cda | ||
|
7504a899a1 | ||
|
c20c5af27c | ||
|
32b72274f5 | ||
|
f50da54274 | ||
|
dd94b8a190 | ||
|
16dd210965 | ||
|
cd0e702e3a | ||
|
cfab16f4a9 | ||
|
60fdb06034 | ||
|
3129a78e06 | ||
|
6b6ec8659b | ||
|
96e755b54d | ||
|
f38458ff4c | ||
|
660d524401 | ||
|
1bc7cde1f0 | ||
|
9c22ffca38 | ||
|
39fe4a14ec | ||
|
65250196cc | ||
|
0123ce13ea | ||
|
03b3e8cd3f |
11
.gitignore
vendored
11
.gitignore
vendored
@ -4,3 +4,14 @@ service-configs/*
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.o
|
||||
gmon.out
|
||||
*.egg-info
|
||||
dist/
|
||||
build/
|
||||
**/*sqlite
|
||||
**/.nyc_output
|
||||
**/coverage
|
||||
**/.venv
|
||||
.idea
|
||||
**/.vim
|
||||
**/*secret.yaml
|
||||
|
@ -1,13 +1,81 @@
|
||||
include:
|
||||
- local: 'ci_templates/.cic-template.yml'
|
||||
- local: 'apps/contract-migration/.gitlab-ci.yml'
|
||||
#- local: 'ci_templates/.cic-template.yml' #kaniko build templates
|
||||
# these includes are app specific unit tests
|
||||
- local: 'apps/cic-eth/.gitlab-ci.yml'
|
||||
- local: 'apps/cic-ussd/.gitlab-ci.yml'
|
||||
- local: 'apps/cic-notify/.gitlab-ci.yml'
|
||||
- local: 'apps/cic-meta/.gitlab-ci.yml'
|
||||
- local: 'apps/cic-cache/.gitlab-ci.yml'
|
||||
#- local: 'apps/contract-migration/.gitlab-ci.yml'
|
||||
#- local: 'apps/data-seeding/.gitlab-ci.yml'
|
||||
|
||||
stages:
|
||||
- version
|
||||
- build
|
||||
- test
|
||||
- release
|
||||
- deploy
|
||||
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
|
||||
|
||||
variables:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
COMPOSE_DOCKER_CLI_BUILD: "1"
|
||||
CI_DEBUG_TRACE: "true"
|
||||
SEMVERBOT_VERSION: "0.2.0"
|
||||
|
||||
#before_script:
|
||||
# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
|
||||
version:
|
||||
#image: python:3.7-stretch
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae
|
||||
stage: version
|
||||
tags:
|
||||
- integration
|
||||
script:
|
||||
- mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
|
||||
- eval $(ssh-agent -s)
|
||||
- ssh-add <(echo "$SSH_PRIVATE_KEY")
|
||||
- git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
|
||||
- export TAG=$(sbot predict version -m auto)
|
||||
- |
|
||||
if [[ -z $TAG ]]
|
||||
then
|
||||
echo "tag could not be set $@"
|
||||
exit 1
|
||||
fi
|
||||
- echo $TAG > version
|
||||
- git tag -a v$TAG -m "ci tagged"
|
||||
- git push origin v$TAG
|
||||
artifacts:
|
||||
paths:
|
||||
- version
|
||||
rules:
|
||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
||||
when: always
|
||||
- if: $CI_COMMIT_REF_NAME == "master"
|
||||
when: always
|
||||
|
||||
# runs on protected branches and pushes to repo
|
||||
build-push:
|
||||
stage: build
|
||||
tags:
|
||||
- integration
|
||||
#script:
|
||||
# - TAG=$CI_Cbefore_script:
|
||||
before_script:
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
script:
|
||||
- TAG=latest ./scripts/build-push.sh
|
||||
- TAG=$(cat ./version) ./scripts/build-push.sh
|
||||
rules:
|
||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
||||
when: always
|
||||
- if: $CI_COMMIT_REF_NAME == "master"
|
||||
when: always
|
||||
|
||||
deploy-dev:
|
||||
stage: deploy
|
||||
trigger: grassrootseconomics/devops
|
||||
when: manual
|
||||
|
44
.gitlab/issue_templates/bug.md
Normal file
44
.gitlab/issue_templates/bug.md
Normal file
@ -0,0 +1,44 @@
|
||||
<!---
|
||||
Please read this!
|
||||
|
||||
Before opening a new issue, make sure to search for keywords in the issues
|
||||
filtered by the "bug" label:
|
||||
|
||||
- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
|
||||
|
||||
and verify the issue you're about to submit isn't a duplicate.
|
||||
--->
|
||||
|
||||
### Summary
|
||||
|
||||
<!-- Summarize the bug encountered concisely. -->
|
||||
|
||||
### Steps to reproduce
|
||||
|
||||
<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
|
||||
|
||||
### Example Project
|
||||
|
||||
<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic
|
||||
behavior, and link to it here in the bug report. If you are using an older version of GitLab, this
|
||||
will also determine whether the bug is fixed in a more recent version. -->
|
||||
|
||||
### What is the current *bug* behavior?
|
||||
|
||||
<!-- Describe what actually happens. -->
|
||||
|
||||
### What is the expected *correct* behavior?
|
||||
|
||||
<!-- Describe what you should see instead. -->
|
||||
|
||||
### Relevant logs and/or screenshots
|
||||
|
||||
<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
|
||||
as it's tough to read otherwise. -->
|
||||
|
||||
|
||||
### Possible fixes
|
||||
|
||||
<!-- If you can, link to the line of code that might be responsible for the problem. -->
|
||||
|
||||
/label ~"bug"
|
16
.semverbot.toml
Normal file
16
.semverbot.toml
Normal file
@ -0,0 +1,16 @@
|
||||
[git]
|
||||
|
||||
[git.config]
|
||||
email = "semverbot@grassroots.org"
|
||||
name = "semvervot"
|
||||
|
||||
[git.tags]
|
||||
prefix = "v"
|
||||
|
||||
[semver]
|
||||
mode = "git-commit"
|
||||
|
||||
[semver.detection]
|
||||
patch = ["fix", "[fix]", "patch", "[patch]"]
|
||||
minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
|
||||
major = ["BREAKING CHANGE"]
|
83
CODE_OF_CONDUCT.md
Normal file
83
CODE_OF_CONDUCT.md
Normal file
@ -0,0 +1,83 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as
|
||||
contributors and maintainers pledge to make participation in our project and
|
||||
our community a harassment-free experience for everyone, regardless of age, body
|
||||
size, disability, ethnicity, sex characteristics, gender identity and expression,
|
||||
level of experience, education, socio-economic status, nationality, personal
|
||||
appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment
|
||||
include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
||||
advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic
|
||||
address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable
|
||||
behavior and are expected to take appropriate and fair corrective action in
|
||||
response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or
|
||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
||||
permanently any contributor for other behaviors that they deem inappropriate,
|
||||
threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all project spaces, and it also applies when
|
||||
an individual is representing the project or its community in public spaces.
|
||||
Examples of representing a project or community include using an official
|
||||
project e-mail address, posting via an official social media account, or acting
|
||||
as an appointed representative at an online or offline event. Representation of
|
||||
a project may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
||||
faith may face temporary or permanent repercussions as determined by other
|
||||
members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see
|
||||
https://www.contributor-covenant.org/faq
|
||||
|
||||
|
||||
|
||||
|
||||
...Try to keep in mind the immortal
|
||||
words of Bill and Ted, "Be excellent to each other."
|
||||
|
16
CONTRIBUTING.md
Normal file
16
CONTRIBUTING.md
Normal file
@ -0,0 +1,16 @@
|
||||
Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider.
|
||||
|
||||
__To request a change to the code please fork this repository and sumbit a merge request.__
|
||||
|
||||
__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
|
||||
|
||||
__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
|
||||
|
||||
__Ask a question in our dev chat:__
|
||||
|
||||
[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
|
||||
|
||||
[Discord](https://discord.gg/XWunwAsX)
|
||||
|
||||
[Matrix, IRC soon?]
|
||||
|
117
CONTRIBUTING_CORE.md
Normal file
117
CONTRIBUTING_CORE.md
Normal file
@ -0,0 +1,117 @@
|
||||
# CORE TEAM CONTRIBUTION GUIDE
|
||||
|
||||
# 1. Transparency
|
||||
|
||||
1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
|
||||
|
||||
1.2 A work log SHOULD be submitted after a "unit of work" is complete.
|
||||
|
||||
1.2.1 A "unit of work" should not span more than one full day's worth of work.
|
||||
|
||||
1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
|
||||
|
||||
1.3 Individual logs are reviewed in weekly meetings
|
||||
|
||||
<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
|
||||
|
||||
```
|
||||
- Core
|
||||
* fixed foo
|
||||
* fixed bar
|
||||
- Frontend
|
||||
* connected bar to baz
|
||||
|
||||
```-->
|
||||
|
||||
1.4 Work log format is defined in []()
|
||||
|
||||
1.5 Link to issue/MR in bullet point where appropriate
|
||||
|
||||
1.6
|
||||
|
||||
|
||||
# 2. Code hygiene
|
||||
|
||||
2.1 Keep function names and variable names short
|
||||
|
||||
2.2 Keep code files, functions and test fixtures short
|
||||
|
||||
2.3 The less magic the better. Recombinable and replaceable is king
|
||||
|
||||
2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
|
||||
|
||||
2.5 Only auto-import when necessary, and always with a minimum of side-effects
|
||||
|
||||
2.6 Use custom errors. Let them bubble up
|
||||
|
||||
2.7 No logs in tight loops
|
||||
|
||||
2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
|
||||
|
||||
2.9 Test coverage MUST be kept higher than 90% after changes
|
||||
|
||||
2.10 Docstrings. Always. Always!
|
||||
|
||||
|
||||
# 3. Versioning
|
||||
|
||||
3.1 Use [Semantic Versioning](https://semver.org/)
|
||||
|
||||
3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
|
||||
|
||||
|
||||
# 4. Issues
|
||||
|
||||
4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
|
||||
|
||||
4.2 Issues need proper problem statement
|
||||
|
||||
4.2.1. What is the current state
|
||||
|
||||
4.2.2. If current state is not behaving as expected, what was the expected state
|
||||
|
||||
4.2.3. What is the desired new state.
|
||||
|
||||
4.3 Issues need proper resolution statement
|
||||
|
||||
4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
|
||||
|
||||
4.3.2. Builet point list of external resources informing the issue and resolution
|
||||
|
||||
4.4 Tasks needs to be appropriately labelled using GROUP labels.
|
||||
|
||||
|
||||
# 5. Code submission
|
||||
|
||||
5.1 A branch and new MR is always created BEFORE THE WORK STARTS
|
||||
|
||||
5.2 An MR should solve ONE SINGLE PART of a problem
|
||||
|
||||
5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
|
||||
|
||||
5.4 MRs should not be open for more than one week (during normal operation periods)
|
||||
|
||||
5.5 MR should ideally not be longer than 400 lines of changes of logic
|
||||
|
||||
5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
|
||||
|
||||
|
||||
# 6. Code reviews
|
||||
|
||||
6.1 At least one peer review before merge
|
||||
|
||||
6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
|
||||
|
||||
6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
|
||||
|
||||
6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
|
||||
|
||||
6.5 If CI is not working, reviewer MUST make sure code builds and runs
|
||||
|
||||
6.6 Behave!
|
||||
|
||||
6.6.1 Don't be a jerk
|
||||
|
||||
6.6.2 Don't block needlessly
|
||||
|
||||
6.6.3 Say please
|
661
LICENSE
Normal file
661
LICENSE
Normal file
@ -0,0 +1,661 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
cic-internal-integration
|
||||
Copyright (C) 2021 Grassroots Economics
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published
|
||||
by the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
41
README.md
41
README.md
@ -1,42 +1,19 @@
|
||||
# cic-internal-integration
|
||||
# Community Inclusion Currency Stack (CIC Stack)
|
||||
|
||||
A custodial evm wallet for executing transactions via USSD
|
||||
|
||||
## Getting started
|
||||
|
||||
## Make some keys
|
||||
This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
|
||||
|
||||
```
|
||||
docker build -t bloxie . && docker run -v "$(pwd)/keys:/root/keys" --rm -it -t bloxie account new --chain /root/bloxberg.json --keys-path /root/keys
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
export DOCKER_BUILDKIT=1
|
||||
```
|
||||
|
||||
To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
|
||||
|
||||
### Prepare the repo
|
||||
## Documentation
|
||||
|
||||
This is stuff we need to put in makefile but for now...
|
||||
|
||||
File mounts and permisssions need to be set
|
||||
```
|
||||
chmod -R 755 scripts/initdb apps/cic-meta/scripts/initdb
|
||||
````
|
||||
|
||||
start cluster
|
||||
```
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
stop cluster
|
||||
```
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
delete data
|
||||
```
|
||||
docker-compose down -v
|
||||
```
|
||||
|
||||
rebuild an images
|
||||
```
|
||||
docker-compose up --build <service_name>
|
||||
```
|
||||
|
||||
Deployment variables are writtend to service-configs/.env after everthing is up.
|
||||
[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
|
||||
|
||||
|
@ -1,34 +0,0 @@
|
||||
# The solc image messes up the alpine environment, so we have to go all over again
|
||||
FROM python:3.8.6-slim-buster
|
||||
|
||||
LABEL authors="Louis Holbrook <dev@holbrook.no> 0826EDA1702D1E87C6E2875121D2E7BB88C2A746"
|
||||
LABEL spdx-license-identifier="GPL-3.0-or-later"
|
||||
LABEL description="Base layer for buiding development images for the cic component suite"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git gcc g++ libpq-dev && \
|
||||
apt-get install -y vim gawk jq telnet openssl iputils-ping curl wget gnupg socat bash procps make python2 postgresql-client
|
||||
|
||||
|
||||
RUN echo installing nodejs tooling
|
||||
|
||||
COPY ./dev/nvm.sh /root/
|
||||
|
||||
# Install nvm with node and npm
|
||||
# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker
|
||||
ENV NVM_DIR /root/.nvm
|
||||
ENV NODE_VERSION 15.3.0
|
||||
ENV BANCOR_NODE_VERSION 10.16.0
|
||||
|
||||
RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh | bash \
|
||||
&& . $NVM_DIR/nvm.sh \
|
||||
&& nvm install $NODE_VERSION \
|
||||
&& nvm alias default $NODE_VERSION \
|
||||
&& nvm use $NODE_VERSION \
|
||||
# So many ridiculously stupid issues with node in docker that take oceans of absolutely wasted time to resolve
|
||||
# owner of these files is "1001" by default - wtf
|
||||
&& chown -R root:root "$NVM_DIR/versions/node/v$NODE_VERSION"
|
||||
|
||||
ENV NODE_PATH $NVM_DIR/versions/node//v$NODE_VERSION/lib/node_modules
|
||||
ENV PATH $NVM_DIR/versions/node//v$NODE_VERSION/bin:$PATH
|
||||
|
@ -1 +0,0 @@
|
||||
## this is an example base image if we wanted one for all the other apps. Its just OS level things
|
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
Normal file
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
**/*.pyc
|
||||
.pydevproject
|
||||
/vendor/
|
7
apps/cic-base-os/aux/wait-for-it/.travis.yml
Normal file
7
apps/cic-base-os/aux/wait-for-it/.travis.yml
Normal file
@ -0,0 +1,7 @@
|
||||
language: python
|
||||
python:
|
||||
- "2.7"
|
||||
|
||||
script:
|
||||
- python test/wait-for-it.py
|
||||
|
20
apps/cic-base-os/aux/wait-for-it/LICENSE
Normal file
20
apps/cic-base-os/aux/wait-for-it/LICENSE
Normal file
@ -0,0 +1,20 @@
|
||||
The MIT License (MIT)
|
||||
Copyright (c) 2016 Giles Hall
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
75
apps/cic-base-os/aux/wait-for-it/README.md
Normal file
75
apps/cic-base-os/aux/wait-for-it/README.md
Normal file
@ -0,0 +1,75 @@
|
||||
# wait-for-it
|
||||
|
||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
||||
host and TCP port. It is useful for synchronizing the spin-up of
|
||||
interdependent services, such as linked docker containers. Since it is a pure
|
||||
bash script, it does not have any external dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
```text
|
||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
||||
and if it is available, echo the message `google is up`.
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
||||
the timeout value to 0 will disable the timeout:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
The subcommand will be executed regardless if the service is up or not. If you
|
||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
||||
fail:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
||||
```
|
||||
|
||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
||||
and determine how to proceed:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
$ echo $?
|
||||
0
|
||||
$ ./wait-for-it.sh www.google.com:81
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
||||
$ echo $?
|
||||
124
|
||||
```
|
||||
|
||||
## Community
|
||||
|
||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
182
apps/cic-base-os/aux/wait-for-it/wait-for-it.sh
Executable file
182
apps/cic-base-os/aux/wait-for-it/wait-for-it.sh
Executable file
@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# Use this script to test if a given TCP host/port are available
|
||||
|
||||
WAITFORIT_cmdname=${0##*/}
|
||||
|
||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
||||
|
||||
usage()
|
||||
{
|
||||
cat << USAGE >&2
|
||||
Usage:
|
||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for()
|
||||
{
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
else
|
||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
||||
fi
|
||||
WAITFORIT_start_ts=$(date +%s)
|
||||
while :
|
||||
do
|
||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
||||
WAITFORIT_result=$?
|
||||
else
|
||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
||||
WAITFORIT_result=$?
|
||||
fi
|
||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
||||
WAITFORIT_end_ts=$(date +%s)
|
||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return $WAITFORIT_result
|
||||
}
|
||||
|
||||
wait_for_wrapper()
|
||||
{
|
||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
else
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
fi
|
||||
WAITFORIT_PID=$!
|
||||
trap "kill -INT -$WAITFORIT_PID" INT
|
||||
wait $WAITFORIT_PID
|
||||
WAITFORIT_RESULT=$?
|
||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
fi
|
||||
return $WAITFORIT_RESULT
|
||||
}
|
||||
|
||||
# process arguments
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
case "$1" in
|
||||
*:* )
|
||||
WAITFORIT_hostport=(${1//:/ })
|
||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
||||
shift 1
|
||||
;;
|
||||
--child)
|
||||
WAITFORIT_CHILD=1
|
||||
shift 1
|
||||
;;
|
||||
-q | --quiet)
|
||||
WAITFORIT_QUIET=1
|
||||
shift 1
|
||||
;;
|
||||
-s | --strict)
|
||||
WAITFORIT_STRICT=1
|
||||
shift 1
|
||||
;;
|
||||
-h)
|
||||
WAITFORIT_HOST="$2"
|
||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--host=*)
|
||||
WAITFORIT_HOST="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-p)
|
||||
WAITFORIT_PORT="$2"
|
||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--port=*)
|
||||
WAITFORIT_PORT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-t)
|
||||
WAITFORIT_TIMEOUT="$2"
|
||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--timeout=*)
|
||||
WAITFORIT_TIMEOUT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
WAITFORIT_CLI=("$@")
|
||||
break
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
echoerr "Unknown argument: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
||||
echoerr "Error: you need to provide a host and port to test."
|
||||
usage
|
||||
fi
|
||||
|
||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
||||
|
||||
# Check to see if timeout is from busybox?
|
||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
||||
|
||||
WAITFORIT_BUSYTIMEFLAG=""
|
||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
||||
WAITFORIT_ISBUSY=1
|
||||
# Check if busybox timeout uses -t flag
|
||||
# (recent Alpine versions don't support -t anymore)
|
||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
||||
fi
|
||||
else
|
||||
WAITFORIT_ISBUSY=0
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
exit $WAITFORIT_RESULT
|
||||
else
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
wait_for_wrapper
|
||||
WAITFORIT_RESULT=$?
|
||||
else
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
||||
exec "${WAITFORIT_CLI[@]}"
|
||||
else
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
@ -6,3 +6,4 @@ HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=
|
||||
|
@ -6,3 +6,4 @@ HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=sqlite
|
||||
DRIVER=pysqlite
|
||||
DEBUG=
|
||||
|
@ -2,4 +2,6 @@
|
||||
omit =
|
||||
.venv/*
|
||||
scripts/*
|
||||
cic_cache/db/postgres/*
|
||||
cic_cache/db/migrations/*
|
||||
cic_cache/version.py
|
||||
cic_cache/cli
|
||||
|
4
apps/cic-cache/.dockerignore
Normal file
4
apps/cic-cache/.dockerignore
Normal file
@ -0,0 +1,4 @@
|
||||
.git
|
||||
.cache
|
||||
.dot
|
||||
**/doc
|
@ -1,22 +1,17 @@
|
||||
.cic_cache_variables:
|
||||
build-test-cic-cache:
|
||||
stage: test
|
||||
tags:
|
||||
- integration
|
||||
variables:
|
||||
APP_NAME: cic-cache
|
||||
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
|
||||
|
||||
.cic_cache_changes_target:
|
||||
MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
|
||||
script:
|
||||
- cd apps/cic-cache
|
||||
- docker build -t $MR_IMAGE_TAG -f docker/Dockerfile .
|
||||
- docker run $MR_IMAGE_TAG sh docker/run_tests.sh
|
||||
allow_failure: true
|
||||
rules:
|
||||
- changes:
|
||||
- $CONTEXT/$APP_NAME/*
|
||||
|
||||
build-mr-cic-cache:
|
||||
extends:
|
||||
- .cic_cache_changes_target
|
||||
- .py_build_merge_request
|
||||
- .cic_cache_variables
|
||||
|
||||
build-push-cic-cache:
|
||||
extends:
|
||||
- .py_build_push
|
||||
- .cic_cache_variables
|
||||
|
||||
|
||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||
changes:
|
||||
- apps/$APP_NAME/**/*
|
||||
when: always
|
||||
|
1
apps/cic-cache/MANIFEST.in
Normal file
1
apps/cic-cache/MANIFEST.in
Normal file
@ -0,0 +1 @@
|
||||
include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*
|
@ -0,0 +1 @@
|
||||
# CIC-CACHE
|
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
Normal file
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
**/*.pyc
|
||||
.pydevproject
|
||||
/vendor/
|
7
apps/cic-cache/aux/wait-for-it/.travis.yml
Normal file
7
apps/cic-cache/aux/wait-for-it/.travis.yml
Normal file
@ -0,0 +1,7 @@
|
||||
language: python
|
||||
python:
|
||||
- "2.7"
|
||||
|
||||
script:
|
||||
- python test/wait-for-it.py
|
||||
|
20
apps/cic-cache/aux/wait-for-it/LICENSE
Normal file
20
apps/cic-cache/aux/wait-for-it/LICENSE
Normal file
@ -0,0 +1,20 @@
|
||||
The MIT License (MIT)
|
||||
Copyright (c) 2016 Giles Hall
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
75
apps/cic-cache/aux/wait-for-it/README.md
Normal file
75
apps/cic-cache/aux/wait-for-it/README.md
Normal file
@ -0,0 +1,75 @@
|
||||
# wait-for-it
|
||||
|
||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
||||
host and TCP port. It is useful for synchronizing the spin-up of
|
||||
interdependent services, such as linked docker containers. Since it is a pure
|
||||
bash script, it does not have any external dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
```text
|
||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
||||
and if it is available, echo the message `google is up`.
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
||||
the timeout value to 0 will disable the timeout:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
The subcommand will be executed regardless if the service is up or not. If you
|
||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
||||
fail:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
||||
```
|
||||
|
||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
||||
and determine how to proceed:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
$ echo $?
|
||||
0
|
||||
$ ./wait-for-it.sh www.google.com:81
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
||||
$ echo $?
|
||||
124
|
||||
```
|
||||
|
||||
## Community
|
||||
|
||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
182
apps/cic-cache/aux/wait-for-it/wait-for-it.sh
Executable file
182
apps/cic-cache/aux/wait-for-it/wait-for-it.sh
Executable file
@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# Use this script to test if a given TCP host/port are available
|
||||
|
||||
WAITFORIT_cmdname=${0##*/}
|
||||
|
||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
||||
|
||||
usage()
|
||||
{
|
||||
cat << USAGE >&2
|
||||
Usage:
|
||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for()
|
||||
{
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
else
|
||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
||||
fi
|
||||
WAITFORIT_start_ts=$(date +%s)
|
||||
while :
|
||||
do
|
||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
||||
WAITFORIT_result=$?
|
||||
else
|
||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
||||
WAITFORIT_result=$?
|
||||
fi
|
||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
||||
WAITFORIT_end_ts=$(date +%s)
|
||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return $WAITFORIT_result
|
||||
}
|
||||
|
||||
wait_for_wrapper()
|
||||
{
|
||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
else
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
fi
|
||||
WAITFORIT_PID=$!
|
||||
trap "kill -INT -$WAITFORIT_PID" INT
|
||||
wait $WAITFORIT_PID
|
||||
WAITFORIT_RESULT=$?
|
||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
fi
|
||||
return $WAITFORIT_RESULT
|
||||
}
|
||||
|
||||
# process arguments
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
case "$1" in
|
||||
*:* )
|
||||
WAITFORIT_hostport=(${1//:/ })
|
||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
||||
shift 1
|
||||
;;
|
||||
--child)
|
||||
WAITFORIT_CHILD=1
|
||||
shift 1
|
||||
;;
|
||||
-q | --quiet)
|
||||
WAITFORIT_QUIET=1
|
||||
shift 1
|
||||
;;
|
||||
-s | --strict)
|
||||
WAITFORIT_STRICT=1
|
||||
shift 1
|
||||
;;
|
||||
-h)
|
||||
WAITFORIT_HOST="$2"
|
||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--host=*)
|
||||
WAITFORIT_HOST="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-p)
|
||||
WAITFORIT_PORT="$2"
|
||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--port=*)
|
||||
WAITFORIT_PORT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-t)
|
||||
WAITFORIT_TIMEOUT="$2"
|
||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--timeout=*)
|
||||
WAITFORIT_TIMEOUT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
WAITFORIT_CLI=("$@")
|
||||
break
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
echoerr "Unknown argument: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
||||
echoerr "Error: you need to provide a host and port to test."
|
||||
usage
|
||||
fi
|
||||
|
||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
||||
|
||||
# Check to see if timeout is from busybox?
|
||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
||||
|
||||
WAITFORIT_BUSYTIMEFLAG=""
|
||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
||||
WAITFORIT_ISBUSY=1
|
||||
# Check if busybox timeout uses -t flag
|
||||
# (recent Alpine versions don't support -t anymore)
|
||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
||||
fi
|
||||
else
|
||||
WAITFORIT_ISBUSY=0
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
exit $WAITFORIT_RESULT
|
||||
else
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
wait_for_wrapper
|
||||
WAITFORIT_RESULT=$?
|
||||
else
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
||||
exec "${WAITFORIT_CLI[@]}"
|
||||
else
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
@ -55,15 +55,37 @@ class Api:
|
||||
queue=callback_queue,
|
||||
)
|
||||
|
||||
def list(self, offset, limit, address=None):
|
||||
def list(self, offset=0, limit=100, address=None, oldest=False):
|
||||
s = celery.signature(
|
||||
'cic_cache.tasks.tx.tx_filter',
|
||||
[
|
||||
0,
|
||||
100,
|
||||
offset,
|
||||
limit,
|
||||
address,
|
||||
oldest,
|
||||
],
|
||||
queue=None
|
||||
queue=self.queue,
|
||||
)
|
||||
if self.callback_param != None:
|
||||
s.link(self.callback_success).on_error(self.callback_error)
|
||||
|
||||
t = s.apply_async()
|
||||
|
||||
return t
|
||||
|
||||
|
||||
def list_content(self, offset=0, limit=100, address=None, block_offset=None, block_limit=None, oldest=False):
|
||||
s = celery.signature(
|
||||
'cic_cache.tasks.tx.tx_filter_content',
|
||||
[
|
||||
offset,
|
||||
limit,
|
||||
address,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest,
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
if self.callback_param != None:
|
||||
s.link(self.callback_success).on_error(self.callback_error)
|
||||
|
@ -1,30 +1,42 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import datetime
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import moolb
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import list_transactions_mined
|
||||
from cic_cache.db import list_transactions_account_mined
|
||||
from cic_cache.db.list import (
|
||||
list_transactions_mined,
|
||||
list_transactions_account_mined,
|
||||
list_transactions_mined_with_data,
|
||||
list_transactions_mined_with_data_index,
|
||||
list_transactions_account_mined_with_data_index,
|
||||
list_transactions_account_mined_with_data,
|
||||
)
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
class BloomCache:
|
||||
DEFAULT_FILTER_SIZE = 8192 * 8
|
||||
DEFAULT_LIMIT = 100
|
||||
|
||||
class Cache:
|
||||
|
||||
def __init__(self, session):
|
||||
self.session = session
|
||||
|
||||
|
||||
class BloomCache(Cache):
|
||||
|
||||
@staticmethod
|
||||
def __get_filter_size(n):
|
||||
n = 8192 * 8
|
||||
n = DEFAULT_FILTER_SIZE
|
||||
logg.warning('filter size hardcoded to {}'.format(n))
|
||||
return n
|
||||
|
||||
|
||||
def load_transactions(self, offset, limit):
|
||||
def load_transactions(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
|
||||
"""Retrieves a list of transactions from cache and creates a bloom filter pointing to blocks and transactions.
|
||||
|
||||
Block and transaction numbers are serialized as 32-bit big-endian numbers. The input to the second bloom filter is the concatenation of the serialized block number and transaction index.
|
||||
@ -41,7 +53,7 @@ class BloomCache:
|
||||
:return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
|
||||
:rtype: tuple
|
||||
"""
|
||||
rows = list_transactions_mined(self.session, offset, limit)
|
||||
rows = list_transactions_mined(self.session, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
|
||||
|
||||
f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
|
||||
f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
|
||||
@ -51,6 +63,11 @@ class BloomCache:
|
||||
if highest_block == -1:
|
||||
highest_block = r[0]
|
||||
lowest_block = r[0]
|
||||
else:
|
||||
if oldest:
|
||||
highest_block = r[0]
|
||||
else:
|
||||
lowest_block = r[0]
|
||||
block = r[0].to_bytes(4, byteorder='big')
|
||||
tx = r[1].to_bytes(4, byteorder='big')
|
||||
f_block.add(block)
|
||||
@ -59,7 +76,7 @@ class BloomCache:
|
||||
return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
|
||||
|
||||
|
||||
def load_transactions_account(self, address, offset, limit):
|
||||
def load_transactions_account(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
|
||||
"""Same as load_transactions(...), but only retrieves transactions where the specified account address is sender or recipient.
|
||||
|
||||
:param address: Address to retrieve transactions for.
|
||||
@ -71,7 +88,7 @@ class BloomCache:
|
||||
:return: Lowest block, bloom filter for blocks, bloom filter for blocks|tx
|
||||
:rtype: tuple
|
||||
"""
|
||||
rows = list_transactions_account_mined(self.session, address, offset, limit)
|
||||
rows = list_transactions_account_mined(self.session, address, offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
|
||||
|
||||
f_block = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
|
||||
f_blocktx = moolb.Bloom(BloomCache.__get_filter_size(limit), 3)
|
||||
@ -81,9 +98,73 @@ class BloomCache:
|
||||
if highest_block == -1:
|
||||
highest_block = r[0]
|
||||
lowest_block = r[0]
|
||||
else:
|
||||
if oldest:
|
||||
highest_block = r[0]
|
||||
else:
|
||||
lowest_block = r[0]
|
||||
block = r[0].to_bytes(4, byteorder='big')
|
||||
tx = r[1].to_bytes(4, byteorder='big')
|
||||
f_block.add(block)
|
||||
f_blocktx.add(block + tx)
|
||||
logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
|
||||
return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
|
||||
|
||||
|
||||
class DataCache(Cache):
|
||||
|
||||
def load_transactions_with_data(self, offset, limit, block_offset=None, block_limit=None, oldest=False):
|
||||
if limit == 0:
|
||||
limit = DEFAULT_LIMIT
|
||||
rows = list_transactions_mined_with_data(self.session, offset, limit, block_offset, block_limit, oldest=oldest)
|
||||
return self.__process_rows(rows, oldest)
|
||||
|
||||
|
||||
def load_transactions_account_with_data(self, address, offset, limit, block_offset=None, block_limit=None, oldest=False):
|
||||
if limit == 0:
|
||||
limit = DEFAULT_LIMIT
|
||||
rows = list_transactions_account_mined_with_data(self.session, address, offset, limit, block_offset, block_limit, oldest=oldest)
|
||||
return self.__process_rows(rows, oldest)
|
||||
|
||||
|
||||
def __process_rows(self, rows, oldest):
|
||||
tx_cache = []
|
||||
highest_block = -1;
|
||||
lowest_block = -1;
|
||||
date_is_str = None # stick this in startup
|
||||
for r in rows:
|
||||
if highest_block == -1:
|
||||
highest_block = r['block_number']
|
||||
lowest_block = r['block_number']
|
||||
else:
|
||||
if oldest:
|
||||
highest_block = r['block_number']
|
||||
else:
|
||||
lowest_block = r['block_number']
|
||||
tx_type = 'unknown'
|
||||
|
||||
if r['value'] != None:
|
||||
tx_type = '{}.{}'.format(r['domain'], r['value'])
|
||||
|
||||
if date_is_str == None:
|
||||
date_is_str = type(r['date_block']).__name__ == 'str'
|
||||
|
||||
o = {
|
||||
'block_number': r['block_number'],
|
||||
'tx_hash': r['tx_hash'],
|
||||
'date_block': r['date_block'],
|
||||
'sender': r['sender'],
|
||||
'recipient': r['recipient'],
|
||||
'from_value': int(r['from_value']),
|
||||
'to_value': int(r['to_value']),
|
||||
'source_token': r['source_token'],
|
||||
'destination_token': r['destination_token'],
|
||||
'success': r['success'],
|
||||
'tx_type': tx_type,
|
||||
}
|
||||
|
||||
if date_is_str:
|
||||
o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
|
||||
|
||||
tx_cache.append(o)
|
||||
return (lowest_block, highest_block, tx_cache)
|
||||
|
15
apps/cic-cache/cic_cache/cli/__init__.py
Normal file
15
apps/cic-cache/cic_cache/cli/__init__.py
Normal file
@ -0,0 +1,15 @@
|
||||
# local imports
|
||||
from .base import *
|
||||
from .chain import (
|
||||
EthChainInterface,
|
||||
chain_interface,
|
||||
)
|
||||
from .rpc import RPC
|
||||
from .arg import ArgumentParser
|
||||
from .config import Config
|
||||
from .celery import CeleryApp
|
||||
from .registry import (
|
||||
connect_registry,
|
||||
connect_token_registry,
|
||||
connect_declarator,
|
||||
)
|
20
apps/cic-cache/cic_cache/cli/arg.py
Normal file
20
apps/cic-cache/cic_cache/cli/arg.py
Normal file
@ -0,0 +1,20 @@
|
||||
# external imports
|
||||
from chainlib.eth.cli import ArgumentParser as BaseArgumentParser
|
||||
|
||||
# local imports
|
||||
from .base import (
|
||||
CICFlag,
|
||||
Flag,
|
||||
)
|
||||
|
||||
|
||||
class ArgumentParser(BaseArgumentParser):
|
||||
|
||||
def process_local_flags(self, local_arg_flags):
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
31
apps/cic-cache/cic_cache/cli/base.py
Normal file
31
apps/cic-cache/cic_cache/cli/base.py
Normal file
@ -0,0 +1,31 @@
|
||||
# standard imports
|
||||
import enum
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.cli import (
|
||||
argflag_std_read,
|
||||
argflag_std_write,
|
||||
argflag_std_base,
|
||||
Flag,
|
||||
)
|
||||
|
||||
class CICFlag(enum.IntEnum):
|
||||
|
||||
# celery - nibble 1
|
||||
CELERY = 1
|
||||
|
||||
# redis - nibble 2
|
||||
# REDIS = 16
|
||||
# REDIS_CALLBACK = 32
|
||||
|
||||
# chain - nibble 3
|
||||
CHAIN = 256
|
||||
|
||||
# sync - nibble 4
|
||||
SYNCER = 4096
|
||||
|
||||
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
#argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
||||
argflag_local_chain = CICFlag.CHAIN
|
||||
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
|
24
apps/cic-cache/cic_cache/cli/celery.py
Normal file
24
apps/cic-cache/cic_cache/cli/celery.py
Normal file
@ -0,0 +1,24 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CeleryApp:
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config):
|
||||
backend_url = config.get('CELERY_RESULT_URL')
|
||||
broker_url = config.get('CELERY_BROKER_URL')
|
||||
celery_app = None
|
||||
if backend_url != None:
|
||||
celery_app = celery.Celery(broker=broker_url, backend=backend_url)
|
||||
logg.info('creating celery app on {} with backend on {}'.format(broker_url, backend_url))
|
||||
else:
|
||||
celery_app = celery.Celery(broker=broker_url)
|
||||
logg.info('creating celery app without results backend on {}'.format(broker_url))
|
||||
|
||||
return celery_app
|
21
apps/cic-cache/cic_cache/cli/chain.py
Normal file
21
apps/cic-cache/cic_cache/cli/chain.py
Normal file
@ -0,0 +1,21 @@
|
||||
# external imports
|
||||
from chainlib.eth.block import (
|
||||
block_by_number,
|
||||
Block,
|
||||
)
|
||||
from chainlib.eth.tx import (
|
||||
receipt,
|
||||
Tx,
|
||||
)
|
||||
from chainlib.interface import ChainInterface
|
||||
|
||||
|
||||
class EthChainInterface(ChainInterface):
|
||||
|
||||
def __init__(self):
|
||||
self._tx_receipt = receipt
|
||||
self._block_by_number = block_by_number
|
||||
self._block_from_src = Block.from_src
|
||||
self._src_normalize = Tx.src_normalize
|
||||
|
||||
chain_interface = EthChainInterface()
|
63
apps/cic-cache/cic_cache/cli/config.py
Normal file
63
apps/cic-cache/cic_cache/cli/config.py
Normal file
@ -0,0 +1,63 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.cli import (
|
||||
Config as BaseConfig,
|
||||
Flag,
|
||||
)
|
||||
|
||||
# local imports
|
||||
from .base import CICFlag
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Config(BaseConfig):
|
||||
|
||||
local_base_config_dir = os.path.join(script_dir, '..', 'data', 'config')
|
||||
|
||||
@classmethod
|
||||
def from_args(cls, args, arg_flags, local_arg_flags, extra_args={}, default_config_dir=None, base_config_dir=None, default_fee_limit=None):
|
||||
expanded_base_config_dir = [cls.local_base_config_dir]
|
||||
if base_config_dir != None:
|
||||
if isinstance(base_config_dir, str):
|
||||
base_config_dir = [base_config_dir]
|
||||
for d in base_config_dir:
|
||||
expanded_base_config_dir.append(d)
|
||||
config = BaseConfig.from_args(args, arg_flags, extra_args=extra_args, default_config_dir=default_config_dir, base_config_dir=expanded_base_config_dir, load_callback=None)
|
||||
|
||||
local_args_override = {}
|
||||
# if local_arg_flags & CICFlag.REDIS:
|
||||
# local_args_override['REDIS_HOST'] = getattr(args, 'redis_host')
|
||||
# local_args_override['REDIS_PORT'] = getattr(args, 'redis_port')
|
||||
# local_args_override['REDIS_DB'] = getattr(args, 'redis_db')
|
||||
# local_args_override['REDIS_TIMEOUT'] = getattr(args, 'redis_timeout')
|
||||
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
||||
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
local_args_override['SYNCER_OFFSET'] = getattr(args, 'offset')
|
||||
local_args_override['SYNCER_NO_HISTORY'] = getattr(args, 'no_history')
|
||||
|
||||
config.dict_override(local_args_override, 'local cli args')
|
||||
|
||||
# if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||
# config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
|
||||
# config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||
|
||||
logg.debug('config loaded:\n{}'.format(config))
|
||||
|
||||
return config
|
||||
|
||||
|
33
apps/cic-cache/cic_cache/cli/registry.py
Normal file
33
apps/cic-cache/cic_cache/cli/registry.py
Normal file
@ -0,0 +1,33 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from cic_eth_registry import CICRegistry
|
||||
from cic_eth_registry.lookup.declarator import AddressDeclaratorLookup
|
||||
from cic_eth_registry.lookup.tokenindex import TokenIndexLookup
|
||||
from chainlib.eth.constant import ZERO_ADDRESS
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def connect_token_registry(self, conn, chain_spec, sender_address=ZERO_ADDRESS):
|
||||
registry = CICRegistry(chain_spec, conn)
|
||||
token_registry_address = registry.by_name('TokenRegistry', sender_address=sender_address)
|
||||
logg.debug('using token registry address {}'.format(token_registry_address))
|
||||
lookup = TokenIndexLookup(chain_spec, token_registry_address)
|
||||
CICRegistry.add_lookup(lookup)
|
||||
|
||||
|
||||
def connect_declarator(self, conn, chain_spec, trusted_addresses, sender_address=ZERO_ADDRESS):
|
||||
registry = CICRegistry(chain_spec, conn)
|
||||
declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
|
||||
logg.debug('using declarator address {}'.format(declarator_address))
|
||||
lookup = AddressDeclaratorLookup(chain_spec, declarator_address, trusted_addresses)
|
||||
CICRegistry.add_lookup(lookup)
|
||||
|
||||
|
||||
def connect_registry(conn, chain_spec, registry_address, sender_address=ZERO_ADDRESS):
|
||||
CICRegistry.address = registry_address
|
||||
registry = CICRegistry(chain_spec, conn)
|
||||
registry_address = registry.by_name('ContractRegistry', sender_address=sender_address)
|
||||
return registry
|
43
apps/cic-cache/cic_cache/cli/rpc.py
Normal file
43
apps/cic-cache/cic_cache/cli/rpc.py
Normal file
@ -0,0 +1,43 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.connection import (
|
||||
RPCConnection,
|
||||
ConnType,
|
||||
)
|
||||
from chainlib.eth.connection import EthUnixSignerConnection
|
||||
from chainlib.chain import ChainSpec
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RPC:
|
||||
|
||||
def __init__(self, chain_spec, rpc_provider, signer_provider=None):
|
||||
self.chain_spec = chain_spec
|
||||
self.rpc_provider = rpc_provider
|
||||
self.signer_provider = signer_provider
|
||||
|
||||
|
||||
def get_default(self):
|
||||
return RPCConnection.connect(self.chain_spec, 'default')
|
||||
|
||||
|
||||
@staticmethod
|
||||
def from_config(config):
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
|
||||
if config.get('SIGNER_PROVIDER'):
|
||||
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
|
||||
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
|
||||
rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
||||
logg.info('set up rpc: {}'.format(rpc))
|
||||
return rpc
|
||||
|
||||
|
||||
def __str__(self):
|
||||
return 'RPC factory, chain {}, rpc {}, signer {}'.format(self.chain_spec, self.rpc_provider, self.signer_provider)
|
||||
|
||||
|
||||
|
5
apps/cic-cache/cic_cache/data/config/celery.ini
Normal file
5
apps/cic-cache/cic_cache/data/config/celery.ini
Normal file
@ -0,0 +1,5 @@
|
||||
[celery]
|
||||
broker_url = redis://localhost:6379
|
||||
result_url =
|
||||
queue = cic-cache
|
||||
debug = 0
|
@ -1,4 +1,4 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
chain_spec =
|
||||
trust_address =
|
||||
health_modules =
|
11
apps/cic-cache/cic_cache/data/config/database.ini
Normal file
11
apps/cic-cache/cic_cache/data/config/database.ini
Normal file
@ -0,0 +1,11 @@
|
||||
[database]
|
||||
engine =
|
||||
driver =
|
||||
host =
|
||||
port =
|
||||
#name = cic-cache
|
||||
prefix =
|
||||
user =
|
||||
password =
|
||||
debug = 0
|
||||
pool_size = 0
|
2
apps/cic-cache/cic_cache/data/config/signer.ini
Normal file
2
apps/cic-cache/cic_cache/data/config/signer.ini
Normal file
@ -0,0 +1,2 @@
|
||||
[signer]
|
||||
provider =
|
4
apps/cic-cache/cic_cache/data/config/syncer.ini
Normal file
4
apps/cic-cache/cic_cache/data/config/syncer.ini
Normal file
@ -0,0 +1,4 @@
|
||||
[syncer]
|
||||
loop_interval = 1
|
||||
offset = 0
|
||||
no_history = 0
|
@ -2,23 +2,33 @@
|
||||
import logging
|
||||
|
||||
# local imports
|
||||
from .list import list_transactions_mined
|
||||
from .list import list_transactions_account_mined
|
||||
from .list import add_transaction
|
||||
from .list import (
|
||||
list_transactions_mined,
|
||||
list_transactions_account_mined,
|
||||
add_transaction,
|
||||
tag_transaction,
|
||||
add_tag,
|
||||
)
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def dsn_from_config(config):
|
||||
def dsn_from_config(config, name):
|
||||
scheme = config.get('DATABASE_ENGINE')
|
||||
if config.get('DATABASE_DRIVER') != None:
|
||||
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
|
||||
|
||||
database_name = name
|
||||
if config.get('DATABASE_PREFIX'):
|
||||
database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
|
||||
dsn = ''
|
||||
if config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
SessionBase.poolable = False
|
||||
dsn = '{}:///{}'.format(
|
||||
scheme,
|
||||
config.get('DATABASE_NAME'),
|
||||
database_name,
|
||||
)
|
||||
|
||||
else:
|
||||
@ -28,7 +38,7 @@ def dsn_from_config(config):
|
||||
config.get('DATABASE_PASSWORD'),
|
||||
config.get('DATABASE_HOST'),
|
||||
config.get('DATABASE_PORT'),
|
||||
config.get('DATABASE_NAME'),
|
||||
database_name,
|
||||
)
|
||||
logg.debug('parsed dsn from config: {}'.format(dsn))
|
||||
return dsn
|
||||
|
@ -2,8 +2,9 @@
|
||||
import logging
|
||||
import datetime
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from sqlalchemy import text
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
@ -12,6 +13,9 @@ def list_transactions_mined(
|
||||
session,
|
||||
offset,
|
||||
limit,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
|
||||
|
||||
@ -22,7 +26,154 @@ def list_transactions_mined(
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
s = "SELECT block_number, tx_index FROM tx ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(limit, offset)
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT block_number, tx_index FROM tx ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
|
||||
def list_transactions_mined_with_data(
|
||||
session,
|
||||
offset,
|
||||
limit,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
|
||||
|
||||
:param block_offset: First block to include in search
|
||||
:type block_offset: int
|
||||
:param block_limit: Last block to include in search
|
||||
:type block_limit: int
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, limit, offset)
|
||||
|
||||
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
|
||||
def list_transactions_mined_with_data_index(
|
||||
session,
|
||||
offset,
|
||||
end,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
|
||||
|
||||
:param offset: Offset in data set to return transactions from
|
||||
:type offset: int
|
||||
:param limit: Max number of transactions to retrieve
|
||||
:type limit: int
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} and block_number <= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, order_by, order_by, offset, end)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, order_by, order_by, offset, end)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(order_by, order_by, offset, end)
|
||||
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
|
||||
def list_transactions_account_mined_with_data_index(
|
||||
session,
|
||||
address,
|
||||
offset,
|
||||
limit,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Executes db query to return all confirmed transactions according to the specified offset and limit, filtered by address
|
||||
|
||||
:param offset: Offset in data set to return transactions from
|
||||
:type offset: int
|
||||
:param limit: Max number of transactions to retrieve
|
||||
:type limit: int
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
|
||||
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
def list_transactions_account_mined_with_data(
|
||||
session,
|
||||
address,
|
||||
offset,
|
||||
limit,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
|
||||
|
||||
:param block_offset: First block to include in search
|
||||
:type block_offset: int
|
||||
:param block_limit: Last block to include in search
|
||||
:type block_limit: int
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, success, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
|
||||
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
@ -32,6 +183,9 @@ def list_transactions_account_mined(
|
||||
address,
|
||||
offset,
|
||||
limit,
|
||||
block_offset,
|
||||
block_limit,
|
||||
oldest=False,
|
||||
):
|
||||
"""Same as list_transactions_mined(...), but only retrieves transaction where the specified account address is sender or recipient.
|
||||
|
||||
@ -44,13 +198,27 @@ def list_transactions_account_mined(
|
||||
:result: Result set
|
||||
:rtype: SQLAlchemy.ResultProxy
|
||||
"""
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number DESC, tx_index DESC LIMIT {} OFFSET {}".format(address, address, limit, offset)
|
||||
|
||||
order_by = 'DESC'
|
||||
if oldest:
|
||||
order_by = 'ASC'
|
||||
|
||||
if block_offset:
|
||||
if block_limit:
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND block_number <= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, block_limit, address, address, order_by, order_by, limit, offset)
|
||||
else:
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE block_number >= {} AND (sender = '{}' OR recipient = '{}') ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(block_offset, address, address, order_by, order_by, limit, offset)
|
||||
|
||||
else:
|
||||
s = "SELECT block_number, tx_index FROM tx WHERE sender = '{}' OR recipient = '{}' ORDER BY block_number {}, tx_index {} LIMIT {} OFFSET {}".format(address, address, order_by, order_by, limit, offset)
|
||||
|
||||
r = session.execute(s)
|
||||
return r
|
||||
|
||||
|
||||
def add_transaction(
|
||||
session, tx_hash,
|
||||
session,
|
||||
tx_hash,
|
||||
block_number,
|
||||
tx_index,
|
||||
sender,
|
||||
@ -62,6 +230,33 @@ def add_transaction(
|
||||
success,
|
||||
timestamp,
|
||||
):
|
||||
"""Adds a single transaction to the cache persistent storage. Sensible interpretation of all fields is the responsibility of the caller.
|
||||
|
||||
:param session: Persistent storage session object
|
||||
:type session: SQLAlchemy session
|
||||
:param tx_hash: Transaction hash
|
||||
:type tx_hash: str, 0x-hex
|
||||
:param block_number: Block number
|
||||
:type block_number: int
|
||||
:param tx_index: Transaction index in block
|
||||
:type tx_index: int
|
||||
:param sender: Ethereum address of effective sender
|
||||
:type sender: str, 0x-hex
|
||||
:param receiver: Ethereum address of effective recipient
|
||||
:type receiver: str, 0x-hex
|
||||
:param source_token: Ethereum address of token used by sender
|
||||
:type source_token: str, 0x-hex
|
||||
:param destination_token: Ethereum address of token received by recipient
|
||||
:type destination_token: str, 0x-hex
|
||||
:param from_value: Source token value spent in transaction
|
||||
:type from_value: int
|
||||
:param to_value: Destination token value received in transaction
|
||||
:type to_value: int
|
||||
:param success: True if code execution on network was successful
|
||||
:type success: bool
|
||||
:param date_block: Block timestamp
|
||||
:type date_block: datetime
|
||||
"""
|
||||
date_block = datetime.datetime.fromtimestamp(timestamp)
|
||||
s = "INSERT INTO tx (tx_hash, block_number, tx_index, sender, recipient, source_token, destination_token, from_value, to_value, success, date_block) VALUES ('{}', {}, {}, '{}', '{}', '{}', '{}', {}, {}, {}, '{}')".format(
|
||||
tx_hash,
|
||||
@ -77,3 +272,74 @@ def add_transaction(
|
||||
date_block,
|
||||
)
|
||||
session.execute(s)
|
||||
|
||||
|
||||
|
||||
def tag_transaction(
|
||||
session,
|
||||
tx_hash,
|
||||
name,
|
||||
domain=None,
|
||||
):
|
||||
"""Tag a single transaction with a single tag.
|
||||
|
||||
Tag must already exist in storage.
|
||||
|
||||
:param session: Persistent storage session object
|
||||
:type session: SQLAlchemy session
|
||||
:param tx_hash: Transaction hash
|
||||
:type tx_hash: str, 0x-hex
|
||||
:param name: Tag value
|
||||
:type name: str
|
||||
:param domain: Tag domain
|
||||
:type domain: str
|
||||
:raises ValueError: Unknown tag or transaction hash
|
||||
|
||||
"""
|
||||
|
||||
s = text("SELECT id from tx where tx_hash = :a")
|
||||
r = session.execute(s, {'a': tx_hash}).fetchall()
|
||||
tx_id = r[0].values()[0]
|
||||
|
||||
if tx_id == None:
|
||||
raise ValueError('unknown tx hash {}'.format(tx_hash))
|
||||
|
||||
#s = text("SELECT id from tag where value = :a and domain = :b")
|
||||
if domain == None:
|
||||
s = text("SELECT id from tag where value = :a")
|
||||
else:
|
||||
s = text("SELECT id from tag where value = :a and domain = :b")
|
||||
r = session.execute(s, {'a': name, 'b': domain}).fetchall()
|
||||
tag_id = r[0].values()[0]
|
||||
|
||||
logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
|
||||
|
||||
if tag_id == None:
|
||||
raise ValueError('unknown tag name {} domain {}'.format(name, domain))
|
||||
|
||||
s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
|
||||
r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
|
||||
|
||||
|
||||
def add_tag(
|
||||
session,
|
||||
name,
|
||||
domain=None,
|
||||
):
|
||||
"""Add a single tag to storage.
|
||||
|
||||
:param session: Persistent storage session object
|
||||
:type session: SQLAlchemy session
|
||||
:param name: Tag value
|
||||
:type name: str
|
||||
:param domain: Tag domain
|
||||
:type domain: str
|
||||
:raises sqlalchemy.exc.IntegrityError: Tag already exists
|
||||
"""
|
||||
|
||||
s = None
|
||||
if domain == None:
|
||||
s = text("INSERT INTO tag (value) VALUES (:b)")
|
||||
else:
|
||||
s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
|
||||
session.execute(s, {'a': domain, 'b': name})
|
||||
|
@ -7,7 +7,7 @@ Create Date: 2021-04-01 08:10:29.156243
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from chainsyncer.db.migrations.sqlalchemy import (
|
||||
from chainsyncer.db.migrations.default.export import (
|
||||
chainsyncer_upgrade,
|
||||
chainsyncer_downgrade,
|
||||
)
|
||||
|
@ -0,0 +1,38 @@
|
||||
"""Transaction tags
|
||||
|
||||
Revision ID: aaf2bdce7d6e
|
||||
Revises: 6604de4203e2
|
||||
Create Date: 2021-05-01 09:20:20.775082
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'aaf2bdce7d6e'
|
||||
down_revision = '6604de4203e2'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'tag',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('domain', sa.String(), nullable=True),
|
||||
sa.Column('value', sa.String(), nullable=False),
|
||||
)
|
||||
op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
|
||||
|
||||
op.create_table(
|
||||
'tag_tx_link',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
|
||||
sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
|
||||
)
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('tag_tx_link')
|
||||
op.drop_index('idx_tag_domain_value')
|
||||
op.drop_table('tag')
|
@ -100,3 +100,4 @@ class SessionBase(Model):
|
||||
logg.debug('destroying session {}'.format(session_key))
|
||||
session.commit()
|
||||
session.close()
|
||||
del SessionBase.localsessions[session_key]
|
||||
|
@ -1 +1,2 @@
|
||||
from .erc20 import *
|
||||
from .faucet import *
|
||||
|
@ -1,2 +1,27 @@
|
||||
class SyncFilter:
|
||||
pass
|
||||
class TagSyncFilter:
|
||||
"""Holds tag name and domain for an implementing filter.
|
||||
|
||||
:param name: Tag value
|
||||
:type name: str
|
||||
:param domain: Tag domain
|
||||
:type domain: str
|
||||
"""
|
||||
|
||||
def __init__(self, name, domain=None):
|
||||
self.tag_name = name
|
||||
self.tag_domain = domain
|
||||
|
||||
|
||||
def tag(self):
|
||||
"""Return tag value/domain.
|
||||
|
||||
:rtype: Tuple
|
||||
:returns: tag value/domain.
|
||||
"""
|
||||
return (self.tag_name, self.tag_domain)
|
||||
|
||||
|
||||
def __str__(self):
|
||||
if self.tag_domain == None:
|
||||
return self.tag_name
|
||||
return '{}.{}'.format(self.tag_domain, self.tag_name)
|
||||
|
@ -2,7 +2,6 @@
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.erc20 import ERC20
|
||||
from chainlib.eth.address import (
|
||||
to_checksum_address,
|
||||
)
|
||||
@ -13,17 +12,19 @@ from cic_eth_registry.error import (
|
||||
NotAContractError,
|
||||
ContractMismatchError,
|
||||
)
|
||||
from eth_erc20 import ERC20
|
||||
|
||||
# local imports
|
||||
from .base import SyncFilter
|
||||
from .base import TagSyncFilter
|
||||
from cic_cache import db as cic_cache_db
|
||||
|
||||
logg = logging.getLogger().getChild(__name__)
|
||||
|
||||
|
||||
class ERC20TransferFilter(SyncFilter):
|
||||
class ERC20TransferFilter(TagSyncFilter):
|
||||
|
||||
def __init__(self, chain_spec):
|
||||
super(ERC20TransferFilter, self).__init__('transfer', domain='erc20')
|
||||
self.chain_spec = chain_spec
|
||||
|
||||
|
||||
@ -46,6 +47,9 @@ class ERC20TransferFilter(SyncFilter):
|
||||
except RequestMismatchException:
|
||||
logg.debug('erc20 match but not a transfer, skipping')
|
||||
return False
|
||||
except ValueError:
|
||||
logg.debug('erc20 match but bogus data, skipping')
|
||||
return False
|
||||
|
||||
token_sender = tx.outputs[0]
|
||||
token_recipient = transfer_data[0]
|
||||
@ -67,7 +71,13 @@ class ERC20TransferFilter(SyncFilter):
|
||||
tx.status == Status.SUCCESS,
|
||||
block.timestamp,
|
||||
)
|
||||
#db_session.flush()
|
||||
db_session.flush()
|
||||
cic_cache_db.tag_transaction(
|
||||
db_session,
|
||||
tx.hash,
|
||||
self.tag_name,
|
||||
domain=self.tag_domain,
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
return True
|
||||
|
73
apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py
Normal file
73
apps/cic-cache/cic_cache/runnable/daemons/filters/faucet.py
Normal file
@ -0,0 +1,73 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from erc20_faucet import Faucet
|
||||
from chainlib.eth.address import to_checksum_address
|
||||
from chainlib.eth.constant import ZERO_ADDRESS
|
||||
from chainlib.status import Status
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
import cic_cache.db as cic_cache_db
|
||||
from .base import TagSyncFilter
|
||||
|
||||
#logg = logging.getLogger().getChild(__name__)
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
class FaucetFilter(TagSyncFilter):
|
||||
|
||||
def __init__(self, chain_spec, sender_address=ZERO_ADDRESS):
|
||||
super(FaucetFilter, self).__init__('give_to', domain='faucet')
|
||||
self.chain_spec = chain_spec
|
||||
self.sender_address = sender_address
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, db_session=None):
|
||||
try:
|
||||
data = strip_0x(tx.payload)
|
||||
except ValueError:
|
||||
return False
|
||||
logg.debug('data {}'.format(data))
|
||||
if Faucet.method_for(data[:8]) == None:
|
||||
return False
|
||||
|
||||
token_sender = tx.inputs[0]
|
||||
token_recipient = data[64+8-40:]
|
||||
logg.debug('token recipient {}'.format(token_recipient))
|
||||
|
||||
f = Faucet(self.chain_spec)
|
||||
o = f.token(token_sender, sender_address=self.sender_address)
|
||||
r = conn.do(o)
|
||||
token = f.parse_token(r)
|
||||
|
||||
f = Faucet(self.chain_spec)
|
||||
o = f.token_amount(token_sender, sender_address=self.sender_address)
|
||||
r = conn.do(o)
|
||||
token_value = f.parse_token_amount(r)
|
||||
|
||||
cic_cache_db.add_transaction(
|
||||
db_session,
|
||||
tx.hash,
|
||||
block.number,
|
||||
tx.index,
|
||||
to_checksum_address(token_sender),
|
||||
to_checksum_address(token_recipient),
|
||||
token,
|
||||
token,
|
||||
token_value,
|
||||
token_value,
|
||||
tx.status == Status.SUCCESS,
|
||||
block.timestamp,
|
||||
)
|
||||
db_session.flush()
|
||||
cic_cache_db.tag_transaction(
|
||||
db_session,
|
||||
tx.hash,
|
||||
self.tag_name,
|
||||
domain=self.tag_domain,
|
||||
)
|
||||
db_session.commit()
|
||||
|
||||
return True
|
186
apps/cic-cache/cic_cache/runnable/daemons/query.py
Normal file
186
apps/cic-cache/cic_cache/runnable/daemons/query.py
Normal file
@ -0,0 +1,186 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import json
|
||||
import re
|
||||
import base64
|
||||
|
||||
# external imports
|
||||
from hexathon import (
|
||||
add_0x,
|
||||
strip_0x,
|
||||
)
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
|
||||
# local imports
|
||||
from cic_cache.cache import (
|
||||
BloomCache,
|
||||
DataCache,
|
||||
)
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
#logg = logging.getLogger()
|
||||
|
||||
re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
||||
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
||||
re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
||||
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
||||
re_default_limit = r'/defaultlimit/?'
|
||||
|
||||
DEFAULT_LIMIT = 100
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
def parse_query_account(r):
|
||||
address = strip_0x(r[1])
|
||||
#address = tx_normalize.wallet_address(address)
|
||||
limit = DEFAULT_LIMIT
|
||||
g = r.groups()
|
||||
if len(g) > 3:
|
||||
limit = int(r[4])
|
||||
if limit == 0:
|
||||
limit = DEFAULT_LIMIT
|
||||
offset = 0
|
||||
if len(g) > 4:
|
||||
offset = int(r[6])
|
||||
|
||||
logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
|
||||
|
||||
return (address, offset, limit,)
|
||||
|
||||
|
||||
# r is an re.Match
|
||||
def parse_query_any(r):
|
||||
limit = DEFAULT_LIMIT
|
||||
offset = 0
|
||||
block_offset = None
|
||||
block_end = None
|
||||
if r.lastindex != None:
|
||||
if r.lastindex > 0:
|
||||
limit = int(r[1])
|
||||
if r.lastindex > 1:
|
||||
offset = int(r[2])
|
||||
if r.lastindex > 2:
|
||||
block_offset = int(r[3])
|
||||
if r.lastindex > 3:
|
||||
block_end = int(r[4])
|
||||
if block_end < block_offset:
|
||||
raise ValueError('cart before the horse, dude')
|
||||
|
||||
logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
|
||||
|
||||
return (offset, limit, block_offset, block_end,)
|
||||
|
||||
|
||||
def process_default_limit(session, env):
|
||||
r = re.match(re_default_limit, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
|
||||
return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_account_bloom(session, env):
|
||||
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match account bloom')
|
||||
|
||||
(address, offset, limit,) = parse_query_account(r)
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
||||
|
||||
o = {
|
||||
'alg': 'sha256',
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
|
||||
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
|
||||
'filter_rounds': 3,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_all_bloom(session, env):
|
||||
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match all bloom')
|
||||
|
||||
(limit, offset, block_offset, block_end,) = parse_query_any(r)
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
||||
|
||||
o = {
|
||||
'alg': 'sha256',
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
|
||||
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
|
||||
'filter_rounds': 3,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_all_data(session, env):
|
||||
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||
# return None
|
||||
logg.debug('match all data')
|
||||
|
||||
logg.debug('got data request {}'.format(env))
|
||||
|
||||
(offset, limit, block_offset, block_end) = parse_query_any(r)
|
||||
|
||||
c = DataCache(session)
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
|
||||
|
||||
for r in tx_cache:
|
||||
r['date_block'] = r['date_block'].timestamp()
|
||||
|
||||
o = {
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'data': tx_cache,
|
||||
}
|
||||
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_account_data(session, env):
|
||||
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match account data')
|
||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||
# return None
|
||||
|
||||
(address, offset, limit,) = parse_query_account(r)
|
||||
|
||||
c = DataCache(session)
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
|
||||
|
||||
for r in tx_cache:
|
||||
r['date_block'] = r['date_block'].timestamp()
|
||||
|
||||
o = {
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'data': tx_cache,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
@ -1,115 +1,42 @@
|
||||
# standard imports
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import argparse
|
||||
import json
|
||||
import base64
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
from cic_cache import BloomCache
|
||||
import cic_cache.cli
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.runnable.daemons.query import (
|
||||
process_default_limit,
|
||||
process_transactions_account_bloom,
|
||||
process_transactions_account_data,
|
||||
process_transactions_all_bloom,
|
||||
process_transactions_all_data,
|
||||
)
|
||||
import cic_cache.cli
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
arg_flags = cic_cache.cli.argflag_std_read
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config:\n{}'.format(config))
|
||||
|
||||
dsn = dsn_from_config(config)
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
|
||||
|
||||
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
|
||||
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
|
||||
|
||||
DEFAULT_LIMIT = 100
|
||||
|
||||
|
||||
def process_transactions_account_bloom(session, env):
|
||||
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
|
||||
address = r[1]
|
||||
if r[2] == None:
|
||||
address = '0x' + address
|
||||
offset = DEFAULT_LIMIT
|
||||
if r.lastindex > 2:
|
||||
offset = r[3]
|
||||
limit = 0
|
||||
if r.lastindex > 3:
|
||||
limit = r[4]
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
||||
|
||||
o = {
|
||||
'alg': 'sha256',
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
|
||||
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
|
||||
'filter_rounds': 3,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_all_bloom(session, env):
|
||||
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
|
||||
offset = DEFAULT_LIMIT
|
||||
if r.lastindex > 0:
|
||||
offset = r[1]
|
||||
limit = 0
|
||||
if r.lastindex > 1:
|
||||
limit = r[2]
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
||||
|
||||
o = {
|
||||
'alg': 'sha256',
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
|
||||
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
|
||||
'filter_rounds': 3,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
# uwsgi application
|
||||
def application(env, start_response):
|
||||
@ -119,10 +46,18 @@ def application(env, start_response):
|
||||
|
||||
session = SessionBase.create_session()
|
||||
for handler in [
|
||||
process_transactions_all_bloom,
|
||||
process_transactions_account_data,
|
||||
process_transactions_account_bloom,
|
||||
process_transactions_all_data,
|
||||
process_transactions_all_bloom,
|
||||
process_default_limit,
|
||||
]:
|
||||
r = None
|
||||
try:
|
||||
r = handler(session, env)
|
||||
except ValueError as e:
|
||||
start_response('400 {}'.format(str(e)))
|
||||
return []
|
||||
if r != None:
|
||||
(mime_type, content) = r
|
||||
break
|
||||
|
@ -3,12 +3,14 @@ import logging
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import tempfile
|
||||
|
||||
# third-party imports
|
||||
import celery
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.tasks.tx import *
|
||||
@ -16,35 +18,20 @@ from cic_cache.tasks.tx import *
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
|
||||
# process args
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
local_arg_flags = cic_cache.cli.argflag_local_task
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config)
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
SessionBase.connect(dsn)
|
||||
|
||||
# verify database connection with minimal sanity query
|
||||
#session = SessionBase.create_session()
|
||||
#session.execute('select version_num from alembic_version')
|
||||
#session.close()
|
||||
|
||||
# set up celery
|
||||
current_app = celery.Celery(__name__)
|
||||
|
||||
@ -87,9 +74,9 @@ def main():
|
||||
elif args.v:
|
||||
argv.append('--loglevel=INFO')
|
||||
argv.append('-Q')
|
||||
argv.append(args.q)
|
||||
argv.append(config.get('CELERY_QUEUE'))
|
||||
argv.append('-n')
|
||||
argv.append(args.q)
|
||||
argv.append(config.get('CELERY_QUEUE'))
|
||||
|
||||
current_app.worker_main(argv)
|
||||
|
||||
|
@ -7,14 +7,8 @@ import argparse
|
||||
import sys
|
||||
import re
|
||||
|
||||
# third-party imports
|
||||
import confini
|
||||
import celery
|
||||
import rlp
|
||||
import cic_base.config
|
||||
import cic_base.log
|
||||
import cic_base.argparse
|
||||
import cic_base.rpc
|
||||
# external imports
|
||||
import sqlalchemy
|
||||
from cic_eth_registry import CICRegistry
|
||||
from cic_eth_registry.error import UnknownContractError
|
||||
from chainlib.chain import ChainSpec
|
||||
@ -27,36 +21,63 @@ from hexathon import (
|
||||
strip_0x,
|
||||
)
|
||||
from chainsyncer.backend.sql import SQLBackend
|
||||
from chainsyncer.driver import (
|
||||
HeadSyncer,
|
||||
HistorySyncer,
|
||||
)
|
||||
from chainsyncer.driver.head import HeadSyncer
|
||||
from chainsyncer.driver.history import HistorySyncer
|
||||
from chainsyncer.db.models.base import SessionBase
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import dsn_from_config
|
||||
import cic_cache.cli
|
||||
from cic_cache.db import (
|
||||
dsn_from_config,
|
||||
add_tag,
|
||||
)
|
||||
from cic_cache.runnable.daemons.filters import (
|
||||
ERC20TransferFilter,
|
||||
FaucetFilter,
|
||||
)
|
||||
|
||||
script_dir = os.path.realpath(os.path.dirname(__file__))
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
logg = cic_base.log.create()
|
||||
argparser = cic_base.argparse.create(script_dir, cic_base.argparse.full_template)
|
||||
#argparser = cic_base.argparse.add(argparser, add_traffic_args, 'traffic')
|
||||
args = cic_base.argparse.parse(argparser, logg)
|
||||
config = cic_base.config.create(args.c, args, args.env_prefix)
|
||||
# process args
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
cic_base.config.log(config)
|
||||
|
||||
dsn = dsn_from_config(config)
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
# set up rpc
|
||||
rpc = cic_cache.cli.RPC.from_config(config)
|
||||
conn = rpc.get_default()
|
||||
|
||||
#RPCConnection.register_location(config.get('ETH_PROVIDER'), chain_spec, 'default')
|
||||
cic_base.rpc.setup(chain_spec, config.get('ETH_PROVIDER'))
|
||||
# set up chain provisions
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
registry = None
|
||||
try:
|
||||
registry = cic_cache.cli.connect_registry(conn, chain_spec, config.get('CIC_REGISTRY_ADDRESS'))
|
||||
except UnknownContractError as e:
|
||||
logg.exception('Registry contract connection failed for {}: {}'.format(config.get('CIC_REGISTRY_ADDRESS'), e))
|
||||
sys.exit(1)
|
||||
logg.info('connected contract registry {}'.format(config.get('CIC_REGISTRY_ADDRESS')))
|
||||
|
||||
|
||||
def register_filter_tags(filters, session):
|
||||
for f in filters:
|
||||
tag = f.tag()
|
||||
try:
|
||||
add_tag(session, tag[0], domain=tag[1])
|
||||
session.commit()
|
||||
logg.info('added tag name "{}" domain "{}"'.format(tag[0], tag[1]))
|
||||
except sqlalchemy.exc.IntegrityError:
|
||||
session.rollback()
|
||||
logg.debug('already have tag name "{}" domain "{}"'.format(tag[0], tag[1]))
|
||||
|
||||
|
||||
def main():
|
||||
@ -67,26 +88,29 @@ def main():
|
||||
r = rpc.do(o)
|
||||
block_offset = int(strip_0x(r), 16) + 1
|
||||
|
||||
logg.debug('starting at block {}'.format(block_offset))
|
||||
logg.debug('current block height {}'.format(block_offset))
|
||||
|
||||
syncers = []
|
||||
|
||||
#if SQLBackend.first(chain_spec):
|
||||
# backend = SQLBackend.initial(chain_spec, block_offset)
|
||||
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
||||
|
||||
if len(syncer_backends) == 0:
|
||||
logg.info('found no backends to resume')
|
||||
syncer_backends.append(SQLBackend.initial(chain_spec, block_offset))
|
||||
initial_block_start = int(config.get('SYNCER_OFFSET'))
|
||||
initial_block_offset = int(block_offset)
|
||||
if config.get('SYNCER_NO_HISTORY'):
|
||||
initial_block_start = initial_block_offset
|
||||
initial_block_offset += 1
|
||||
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
||||
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
||||
else:
|
||||
for syncer_backend in syncer_backends:
|
||||
logg.info('resuming sync session {}'.format(syncer_backend))
|
||||
|
||||
for syncer_backend in syncer_backends:
|
||||
syncers.append(HistorySyncer(syncer_backend))
|
||||
syncers.append(HistorySyncer(syncer_backend, cic_cache.cli.chain_interface))
|
||||
|
||||
syncer_backend = SQLBackend.live(chain_spec, block_offset+1)
|
||||
syncers.append(HeadSyncer(syncer_backend))
|
||||
syncers.append(HeadSyncer(syncer_backend, cic_cache.cli.chain_interface))
|
||||
|
||||
trusted_addresses_src = config.get('CIC_TRUST_ADDRESS')
|
||||
if trusted_addresses_src == None:
|
||||
@ -97,11 +121,22 @@ def main():
|
||||
logg.info('using trusted address {}'.format(address))
|
||||
|
||||
erc20_transfer_filter = ERC20TransferFilter(chain_spec)
|
||||
faucet_filter = FaucetFilter(chain_spec)
|
||||
|
||||
filters = [
|
||||
erc20_transfer_filter,
|
||||
faucet_filter,
|
||||
]
|
||||
|
||||
session = SessionBase.create_session()
|
||||
register_filter_tags(filters, session)
|
||||
session.close()
|
||||
|
||||
i = 0
|
||||
for syncer in syncers:
|
||||
logg.debug('running syncer index {}'.format(i))
|
||||
syncer.add_filter(erc20_transfer_filter)
|
||||
for f in filters:
|
||||
syncer.add_filter(f)
|
||||
r = syncer.loop(int(config.get('SYNCER_LOOP_INTERVAL')), rpc)
|
||||
sys.stderr.write("sync {} done at block {}\n".format(syncer, r))
|
||||
|
||||
|
@ -2,14 +2,17 @@
|
||||
import celery
|
||||
|
||||
# local imports
|
||||
from cic_cache.cache import BloomCache
|
||||
from cic_cache.cache import (
|
||||
BloomCache,
|
||||
DataCache,
|
||||
)
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
|
||||
celery_app = celery.current_app
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def tx_filter(self, offset, limit, address=None, encoding='hex'):
|
||||
def tx_filter(self, offset, limit, address=None, oldest=False, encoding='hex'):
|
||||
queue = self.request.delivery_info.get('routing_key')
|
||||
|
||||
session = SessionBase.create_session()
|
||||
@ -17,9 +20,9 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
|
||||
c = BloomCache(session)
|
||||
b = None
|
||||
if address == None:
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit, oldest=oldest)
|
||||
else:
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit, oldest=oldest)
|
||||
|
||||
session.close()
|
||||
|
||||
@ -35,4 +38,17 @@ def tx_filter(self, offset, limit, address=None, encoding='hex'):
|
||||
return o
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
def tx_filter_content(self, offset, limit, address=None, block_offset=None, block_limit=None, oldest=False, encoding='hex'):
|
||||
session = SessionBase.create_session()
|
||||
|
||||
c = DataCache(session)
|
||||
b = None
|
||||
if address == None:
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset=block_offset, block_limit=block_limit, oldest=oldest)
|
||||
else:
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data_index(address, offset, limit, block_offset=block_offset, block_limit=block_limit)
|
||||
|
||||
session.close()
|
||||
|
||||
return (lowest_block, highest_block, tx_cache,)
|
||||
|
@ -4,8 +4,8 @@ import semver
|
||||
version = (
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
'alpha.2',
|
||||
1,
|
||||
'alpha.3',
|
||||
)
|
||||
|
||||
version_object = semver.VersionInfo(
|
||||
|
@ -1,2 +0,0 @@
|
||||
[bancor]
|
||||
dir =
|
@ -1,3 +0,0 @@
|
||||
[celery]
|
||||
broker_url = redis:///
|
||||
result_url = redis:///
|
@ -1,4 +0,0 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
chain_spec =
|
||||
trust_address =
|
@ -1,9 +0,0 @@
|
||||
[database]
|
||||
NAME=cic_cache
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=
|
@ -1,3 +0,0 @@
|
||||
[bancor]
|
||||
registry_address =
|
||||
dir = /usr/local/share/bancor
|
@ -1,4 +0,0 @@
|
||||
[cic]
|
||||
chain_spec =
|
||||
registry_address =
|
||||
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
|
@ -1,9 +0,0 @@
|
||||
[database]
|
||||
NAME=cic_cache
|
||||
USER=grassroots
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=63432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=0
|
@ -1,2 +0,0 @@
|
||||
[eth]
|
||||
provider = ws://localhost:63546
|
@ -1,2 +0,0 @@
|
||||
[eth]
|
||||
provider = ws://localhost:8545
|
@ -1,2 +0,0 @@
|
||||
[syncer]
|
||||
loop_interval = 5
|
@ -1,2 +0,0 @@
|
||||
[bancor]
|
||||
dir =
|
@ -1,2 +1,3 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
trust_address =
|
||||
|
@ -1,9 +1,9 @@
|
||||
[database]
|
||||
NAME=cic-cache-test
|
||||
PREFIX=cic-cache-test
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=sqlite
|
||||
DRIVER=pysqlite
|
||||
DEBUG=
|
||||
DEBUG=1
|
||||
|
@ -1,5 +0,0 @@
|
||||
[eth]
|
||||
#ws_provider = ws://localhost:8546
|
||||
#ttp_provider = http://localhost:8545
|
||||
provider = http://localhost:8545
|
||||
#chain_id =
|
@ -1,4 +1,4 @@
|
||||
openapi: "3.0.3"
|
||||
openapi: "3.0.2"
|
||||
info:
|
||||
title: Grassroots Economics CIC Cache
|
||||
description: Cache of processed transaction data from Ethereum blockchain and worker queues
|
||||
@ -9,17 +9,34 @@ info:
|
||||
email: will@grassecon.org
|
||||
license:
|
||||
name: GPLv3
|
||||
version: 0.1.0
|
||||
version: 0.2.0
|
||||
|
||||
paths:
|
||||
/tx/{offset}/{limit}:
|
||||
description: Bloom filter for batch of latest transactions
|
||||
/defaultlimit:
|
||||
summary: The default limit value of result sets.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve default limit
|
||||
operationId: limit.default
|
||||
responses:
|
||||
200:
|
||||
description: Limit query successful
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Limit"
|
||||
|
||||
/tx:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get
|
||||
operationId: tx.get.latest
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
@ -29,27 +46,109 @@ paths:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
|
||||
parameters:
|
||||
- name: offset
|
||||
in: path
|
||||
/tx/{limit}:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{address}/{offset}/{limit}:
|
||||
description: Bloom filter for batch of latest transactions by account
|
||||
/tx/{limit}/{offset}:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get
|
||||
operationId: tx.get.latest.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{limit}/{offset}/{block_offset}:
|
||||
summary: Bloom filter for batch of transactions since a particular block.
|
||||
description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.range.block.offset
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from oldest to newest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{limit}/{offset}/{block_offset}/{block_end}:
|
||||
summary: Bloom filter for batch of transactions within a particular block range.
|
||||
description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.range.block.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
@ -58,6 +157,49 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_end
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{address}:
|
||||
summary: Bloom filter for batch of latest transactions by account.
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
@ -65,26 +207,342 @@ paths:
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: offset
|
||||
in: path
|
||||
|
||||
|
||||
/tx/{address}/{limit}:
|
||||
summary: Bloom filter for batch of latest transactions by account.
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{address}/{limit}/{offset}:
|
||||
summary: Bloom filter for batch of latest transactions by account
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
|
||||
/txa/{limit}:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{limit}/{offset}:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{limit}/{offset}/{block_offset}:
|
||||
summary: Cached data for transactions since a particular block.
|
||||
description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range.block.offset
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
/txa/{limit}/{offset}/{block_offset}/{block_end}:
|
||||
summary: Cached data for transactions within a particular block range.
|
||||
description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range.block.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_end
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{address}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
|
||||
|
||||
/txa/{address}/{limit}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{address}/{limit}/{offset}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
components:
|
||||
schemas:
|
||||
Limit:
|
||||
type: integer
|
||||
format: int32
|
||||
BlocksBloom:
|
||||
type: object
|
||||
properties:
|
||||
low:
|
||||
type: int
|
||||
type: integer
|
||||
format: int32
|
||||
description: The lowest block number included in the filter
|
||||
high:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The highest block number included in the filter
|
||||
block_filter:
|
||||
type: string
|
||||
format: byte
|
||||
@ -97,6 +555,89 @@ components:
|
||||
type: string
|
||||
description: Hashing algorithm (currently only using sha256)
|
||||
filter_rounds:
|
||||
type: int
|
||||
type: integer
|
||||
format: int32
|
||||
description: Number of hash rounds used to create the filter
|
||||
TransactionList:
|
||||
type: object
|
||||
properties:
|
||||
low:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The lowest block number included in the result set
|
||||
high:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The highest block number included in the filter
|
||||
data:
|
||||
type: array
|
||||
description: Cached transaction data
|
||||
items:
|
||||
$ref: "#/components/schemas/Transaction"
|
||||
Transaction:
|
||||
type: object
|
||||
properties:
|
||||
block_number:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Block number transaction was included in.
|
||||
tx_hash:
|
||||
type: string
|
||||
description: Transaction hash, in hex.
|
||||
date_block:
|
||||
type: integer
|
||||
format: int32
|
||||
description: Block timestamp.
|
||||
sender:
|
||||
type: string
|
||||
description: Spender address, in hex.
|
||||
recipient:
|
||||
type: string
|
||||
description: Beneficiary address, in hex.
|
||||
from_value:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Value deducted from spender's balance.
|
||||
to_value:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Value added to beneficiary's balance.
|
||||
source_token:
|
||||
type: string
|
||||
description: Network address of token in which `from_value` is denominated.
|
||||
destination_token:
|
||||
type: string
|
||||
description: Network address of token in which `to_value` is denominated.
|
||||
success:
|
||||
type: boolean
|
||||
description: Network consensus state on whether the transaction was successful or not.
|
||||
tx_type:
|
||||
type: string
|
||||
enum:
|
||||
- erc20.faucet
|
||||
- faucet.give_to
|
||||
|
||||
examples:
|
||||
data_last:
|
||||
summary: Get the latest cached transactions, using the server's default limit.
|
||||
value: "/txa"
|
||||
|
||||
data_limit:
|
||||
summary: Get the last 42 cached transactions.
|
||||
value: "/txa/42"
|
||||
|
||||
data_range:
|
||||
summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
|
||||
value: "/txa/42/13"
|
||||
|
||||
data_range_block_offset:
|
||||
summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
|
||||
value: "/txa/42/0/1337"
|
||||
|
||||
data_range_block_offset:
|
||||
summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
|
||||
value: "/txa/42/0/1337/1453"
|
||||
|
||||
data_range_block_range:
|
||||
summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
|
||||
value: "/txa/42/13/1337/1453"
|
||||
|
@ -1,56 +1,32 @@
|
||||
FROM python:3.8.6-slim-buster
|
||||
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
|
||||
|
||||
#COPY --from=0 /usr/local/share/cic/solidity/ /usr/local/share/cic/solidity/
|
||||
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
|
||||
|
||||
WORKDIR /usr/src/cic-cache
|
||||
COPY requirements.txt .
|
||||
|
||||
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
|
||||
ARG root_requirement_file='requirements.txt'
|
||||
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
|
||||
ARG EXTRA_PIP_ARGS=""
|
||||
ARG PIP_INDEX_URL=https://pypi.org/simple
|
||||
|
||||
#RUN apk update && \
|
||||
# apk add gcc musl-dev gnupg libpq
|
||||
#RUN apk add postgresql-dev
|
||||
#RUN apk add linux-headers
|
||||
#RUN apk add libffi-dev
|
||||
RUN apt-get update && \
|
||||
apt install -y gcc gnupg libpq-dev wget make g++ gnupg bash procps git
|
||||
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
|
||||
pip install --index-url $PIP_INDEX_URL \
|
||||
--pre \
|
||||
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
|
||||
-r requirements.txt
|
||||
|
||||
# Copy shared requirements from top of mono-repo
|
||||
RUN echo "copying root req file ${root_requirement_file}"
|
||||
RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2a76
|
||||
|
||||
COPY cic-cache/requirements.txt ./
|
||||
COPY cic-cache/setup.cfg \
|
||||
cic-cache/setup.py \
|
||||
./
|
||||
COPY cic-cache/cic_cache/ ./cic_cache/
|
||||
COPY cic-cache/scripts/ ./scripts/
|
||||
COPY cic-cache/test_requirements.txt ./
|
||||
RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
|
||||
RUN pip install $pip_extra_index_url_flag .
|
||||
RUN pip install .[server]
|
||||
|
||||
COPY cic-cache/tests/ ./tests/
|
||||
#COPY db/ cic-cache/db
|
||||
#RUN apk add postgresql-client
|
||||
|
||||
# ini files in config directory defines the configurable parameters for the application
|
||||
# they can all be overridden by environment variables
|
||||
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
|
||||
COPY cic-cache/config/ /usr/local/etc/cic-cache/
|
||||
COPY . .
|
||||
RUN pip install . --extra-index-url $EXTRA_PIP_INDEX_URL
|
||||
|
||||
# for db migrations
|
||||
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
|
||||
COPY cic-cache/cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
|
||||
COPY ./aux/wait-for-it/wait-for-it.sh ./
|
||||
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
|
||||
|
||||
RUN git clone https://gitlab.com/grassrootseconomics/cic-contracts.git && \
|
||||
mkdir -p /usr/local/share/cic/solidity && \
|
||||
cp -R cic-contracts/abis /usr/local/share/cic/solidity/abi
|
||||
|
||||
COPY cic-cache/docker/start_tracker.sh ./start_tracker.sh
|
||||
COPY cic-cache/docker/db.sh ./db.sh
|
||||
COPY /docker/start_tracker.sh ./start_tracker.sh
|
||||
COPY /docker/db.sh ./db.sh
|
||||
RUN chmod 755 ./*.sh
|
||||
# Tracker
|
||||
# ENTRYPOINT ["/usr/local/bin/cic-cache-tracker", "-vv"]
|
||||
# Server
|
||||
# ENTRYPOINT [ "/usr/local/bin/uwsgi", "--wsgi-file", "/usr/local/lib/python3.8/site-packages/cic_cache/runnable/server.py", "--http", ":80", "--pyargv", "-vv" ]
|
||||
ENTRYPOINT []
|
||||
|
@ -2,5 +2,5 @@
|
||||
|
||||
set -e
|
||||
>&2 echo executing database migration
|
||||
python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
||||
python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
||||
set +e
|
||||
|
10
apps/cic-cache/docker/run_tests.sh
Normal file
10
apps/cic-cache/docker/run_tests.sh
Normal file
@ -0,0 +1,10 @@
|
||||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
pip install --extra-index-url https://pip.grassrootseconomics.net \
|
||||
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
|
||||
-r test_requirements.txt
|
||||
|
||||
export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
|
||||
|
@ -2,4 +2,9 @@
|
||||
|
||||
. ./db.sh
|
||||
|
||||
if [ $? -ne "0" ]; then
|
||||
>&2 echo db migrate fail
|
||||
exit 1
|
||||
fi
|
||||
|
||||
/usr/local/bin/cic-cache-trackerd $@
|
||||
|
@ -1,12 +1,15 @@
|
||||
cic-base~=0.1.2a77
|
||||
alembic==1.4.2
|
||||
confini~=0.3.6rc3
|
||||
confini~=0.5.3
|
||||
uwsgi==2.0.19.1
|
||||
moolb~=0.1.0
|
||||
cic-eth-registry~=0.5.4a16
|
||||
moolb~=0.2.0
|
||||
cic-eth-registry~=0.6.6
|
||||
SQLAlchemy==1.3.20
|
||||
semver==2.13.0
|
||||
psycopg2==2.8.6
|
||||
celery==4.4.7
|
||||
redis==3.5.3
|
||||
chainsyncer[sql]~=0.0.2a2
|
||||
chainsyncer[sql]~=0.0.7
|
||||
erc20-faucet~=0.3.2
|
||||
chainlib-eth~=0.0.15
|
||||
eth-address-index~=0.2.4
|
||||
okota~=0.2.5
|
||||
|
@ -1,56 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
import confini
|
||||
|
||||
from cic_cache.db import dsn_from_config
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
# BUG: the dbdir doesn't work after script install
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
args = argparser.parse_args()
|
||||
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config:\n{}'.format(config))
|
||||
|
||||
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
|
||||
if not os.path.isdir(migrations_dir):
|
||||
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
|
||||
migrations_dir = os.path.join(args.migrations_dir, 'default')
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config)
|
||||
|
||||
|
||||
logg.info('using migrations dir {}'.format(migrations_dir))
|
||||
logg.info('using db {}'.format(dsn))
|
||||
ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
|
||||
ac.set_main_option('sqlalchemy.url', dsn)
|
||||
ac.set_main_option('script_location', migrations_dir)
|
||||
|
||||
alembic.command.upgrade(ac, 'head')
|
67
apps/cic-cache/scripts/migrate_cic_cache.py
Normal file
67
apps/cic-cache/scripts/migrate_cic_cache.py
Normal file
@ -0,0 +1,67 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
# standard imports
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
|
||||
# external imports
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import dsn_from_config
|
||||
import cic_cache.cli
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
# BUG: the dbdir doesn't work after script install
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
default_migrations_dir = os.path.join(dbdir, 'migrations')
|
||||
configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
|
||||
|
||||
#config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
|
||||
argparser.add_argument('-f', '--force', action='store_true', help='force action')
|
||||
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
|
||||
args = argparser.parse_args()
|
||||
|
||||
extra_args = {
|
||||
'reset': None,
|
||||
'force': None,
|
||||
'migrations_dir': None,
|
||||
}
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
|
||||
|
||||
migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
|
||||
if not os.path.isdir(migrations_dir):
|
||||
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
|
||||
migrations_dir = os.path.join(args.migrations_dir, 'default')
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
|
||||
|
||||
logg.info('using migrations dir {}'.format(migrations_dir))
|
||||
logg.info('using db {}'.format(dsn))
|
||||
ac = AlembicConfig(os.path.join(migrations_dir, 'alembic.ini'))
|
||||
ac.set_main_option('sqlalchemy.url', dsn)
|
||||
ac.set_main_option('script_location', migrations_dir)
|
||||
|
||||
if args.reset:
|
||||
if not args.f:
|
||||
if not re.match(r'[yY][eE]?[sS]?', input('EEK! this will DELETE the existing db. are you sure??')):
|
||||
logg.error('user chickened out on requested reset, bailing')
|
||||
sys.exit(1)
|
||||
alembic.command.downgrade(ac, 'base')
|
||||
alembic.command.upgrade(ac, 'head')
|
@ -1,6 +1,7 @@
|
||||
[metadata]
|
||||
name = cic-cache
|
||||
description = CIC Cache API and server
|
||||
version = 0.3.0a2
|
||||
author = Louis Holbrook
|
||||
author_email = dev@holbrook.no
|
||||
url = https://gitlab.com/grassrootseconomics/cic-eth
|
||||
@ -23,19 +24,22 @@ licence_files =
|
||||
|
||||
[options]
|
||||
python_requires = >= 3.6
|
||||
include_package_data = True
|
||||
packages =
|
||||
cic_cache
|
||||
cic_cache.tasks
|
||||
cic_cache.db
|
||||
cic_cache.db.models
|
||||
cic_cache.cli
|
||||
cic_cache.runnable
|
||||
cic_cache.runnable.daemons
|
||||
cic_cache.runnable.daemons.filters
|
||||
scripts =
|
||||
./scripts/migrate.py
|
||||
./scripts/migrate_cic_cache.py
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
cic-cache-trackerd = cic_cache.runnable.daemons.tracker:main
|
||||
cic-cache-serverd = cic_cache.runnable.daemons.server:main
|
||||
cic-cache-taskerd = cic_cache.runnable.daemons.tasker:main
|
||||
cic-cache-list = cic_cache.runable.list:main
|
||||
|
@ -1,38 +1,39 @@
|
||||
from setuptools import setup
|
||||
|
||||
import configparser
|
||||
# import configparser
|
||||
import os
|
||||
import time
|
||||
|
||||
from cic_cache.version import (
|
||||
version_object,
|
||||
version_string
|
||||
)
|
||||
# import time
|
||||
|
||||
class PleaseCommitFirstError(Exception):
|
||||
pass
|
||||
|
||||
def git_hash():
|
||||
import subprocess
|
||||
git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
||||
if len(git_diff.stdout) > 0:
|
||||
raise PleaseCommitFirstError()
|
||||
git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
||||
git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
||||
return git_hash_brief
|
||||
|
||||
version_string = str(version_object)
|
||||
|
||||
try:
|
||||
version_git = git_hash()
|
||||
version_string += '+build.{}'.format(version_git)
|
||||
except FileNotFoundError:
|
||||
time_string_pair = str(time.time()).split('.')
|
||||
version_string += '+build.{}{:<09d}'.format(
|
||||
time_string_pair[0],
|
||||
int(time_string_pair[1]),
|
||||
)
|
||||
print('final version string will be {}'.format(version_string))
|
||||
# from cic_cache.version import (
|
||||
# version_object,
|
||||
# version_string
|
||||
# )
|
||||
#
|
||||
# class PleaseCommitFirstError(Exception):
|
||||
# pass
|
||||
#
|
||||
# def git_hash():
|
||||
# import subprocess
|
||||
# git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
||||
# if len(git_diff.stdout) > 0:
|
||||
# raise PleaseCommitFirstError()
|
||||
# git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
||||
# git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
||||
# return git_hash_brief
|
||||
#
|
||||
# version_string = str(version_object)
|
||||
#
|
||||
# try:
|
||||
# version_git = git_hash()
|
||||
# version_string += '+build.{}'.format(version_git)
|
||||
# except FileNotFoundError:
|
||||
# time_string_pair = str(time.time()).split('.')
|
||||
# version_string += '+build.{}{:<09d}'.format(
|
||||
# time_string_pair[0],
|
||||
# int(time_string_pair[1]),
|
||||
# )
|
||||
# print('final version string will be {}'.format(version_string))
|
||||
|
||||
requirements = []
|
||||
f = open('requirements.txt', 'r')
|
||||
@ -52,9 +53,8 @@ while True:
|
||||
test_requirements.append(l.rstrip())
|
||||
f.close()
|
||||
|
||||
|
||||
setup(
|
||||
version=version_string,
|
||||
# version=version_string,
|
||||
install_requires=requirements,
|
||||
tests_require=test_requirements,
|
||||
)
|
||||
|
@ -4,3 +4,7 @@ pytest-mock==3.3.1
|
||||
pysqlite3==0.4.3
|
||||
sqlparse==0.4.1
|
||||
pytest-celery==0.0.0a1
|
||||
eth_tester==0.5.0b3
|
||||
py-evm==0.3.0a20
|
||||
sarafu-faucet~=0.0.7a1
|
||||
erc20-transfer-authorization~=0.3.6
|
||||
|
40
apps/cic-cache/tests/cli/test_cli_args.py
Normal file
40
apps/cic-cache/tests/cli/test_cli_args.py
Normal file
@ -0,0 +1,40 @@
|
||||
# standard imports
|
||||
import os
|
||||
|
||||
# external imports
|
||||
import chainlib.cli
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
config_dir = os.path.join(script_dir, '..', 'testdata', 'config')
|
||||
|
||||
|
||||
def test_argumentparserto_config():
|
||||
|
||||
argparser = cic_cache.cli.ArgumentParser()
|
||||
|
||||
local_flags = 0xffff
|
||||
argparser.process_local_flags(local_flags)
|
||||
argparser.add_argument('--foo', type=str)
|
||||
args = argparser.parse_args([
|
||||
'-q', 'baz',
|
||||
'--offset', '13',
|
||||
'--no-history',
|
||||
'-r','0xdeadbeef',
|
||||
'-vv',
|
||||
'--foo', 'bar',
|
||||
])
|
||||
|
||||
extra_args = {
|
||||
'foo': '_BARBARBAR',
|
||||
}
|
||||
config = cic_cache.cli.Config.from_args(args, chainlib.cli.argflag_std_base, local_flags, extra_args=extra_args, base_config_dir=config_dir)
|
||||
|
||||
assert config.get('_BARBARBAR') == 'bar'
|
||||
assert config.get('CELERY_QUEUE') == 'baz'
|
||||
assert config.get('SYNCER_NO_HISTORY') == True
|
||||
assert config.get('SYNCER_OFFSET') == 13
|
||||
assert config.get('CIC_REGISTRY_ADDRESS') == '0xdeadbeef'
|
||||
|
17
apps/cic-cache/tests/cli/test_cli_celery.py
Normal file
17
apps/cic-cache/tests/cli/test_cli_celery.py
Normal file
@ -0,0 +1,17 @@
|
||||
# standard imports
|
||||
import tempfile
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
|
||||
|
||||
def test_cli_celery():
|
||||
cf = tempfile.mkdtemp()
|
||||
|
||||
config = {
|
||||
'CELERY_RESULT_URL': 'filesystem://' + cf,
|
||||
}
|
||||
cic_cache.cli.CeleryApp.from_config(config)
|
||||
|
||||
config['CELERY_BROKER_URL'] = 'filesystem://' + cf
|
||||
cic_cache.cli.CeleryApp.from_config(config)
|
68
apps/cic-cache/tests/cli/test_cli_chain.py
Normal file
68
apps/cic-cache/tests/cli/test_cli_chain.py
Normal file
@ -0,0 +1,68 @@
|
||||
# external imports
|
||||
import pytest
|
||||
from chainlib.eth.gas import (
|
||||
Gas,
|
||||
RPCGasOracle,
|
||||
)
|
||||
from chainlib.eth.nonce import RPCNonceOracle
|
||||
from chainlib.eth.block import (
|
||||
block_latest,
|
||||
Block,
|
||||
)
|
||||
from chainlib.eth.pytest.fixtures_chain import default_chain_spec
|
||||
from chainlib.eth.pytest.fixtures_ethtester import *
|
||||
from cic_eth_registry.pytest.fixtures_contracts import *
|
||||
from hexathon import add_0x
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
|
||||
|
||||
@pytest.mark.xfail()
|
||||
def test_cli_rpc(
|
||||
eth_rpc,
|
||||
eth_signer,
|
||||
default_chain_spec,
|
||||
):
|
||||
config = {
|
||||
'CHAIN_SPEC': str(default_chain_spec),
|
||||
'RPC_HTTP_PROVIDER': 'http://localhost:8545',
|
||||
}
|
||||
rpc = cic_cache.cli.RPC.from_config(config, default_label='foo')
|
||||
conn = rpc.get_by_label('foo')
|
||||
#o = block_latest()
|
||||
#conn.do(o)
|
||||
|
||||
|
||||
def test_cli_chain(
|
||||
default_chain_spec,
|
||||
eth_rpc,
|
||||
eth_signer,
|
||||
contract_roles,
|
||||
):
|
||||
ifc = cic_cache.cli.EthChainInterface()
|
||||
|
||||
nonce_oracle = RPCNonceOracle(contract_roles['CONTRACT_DEPLOYER'], conn=eth_rpc)
|
||||
gas_oracle = RPCGasOracle(conn=eth_rpc)
|
||||
c = Gas(default_chain_spec, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle, signer=eth_signer)
|
||||
recipient = add_0x(os.urandom(20).hex())
|
||||
(tx_hash, o) = c.create(contract_roles['CONTRACT_DEPLOYER'], recipient, 1024)
|
||||
r = eth_rpc.do(o)
|
||||
|
||||
o = ifc.tx_receipt(r)
|
||||
r = eth_rpc.do(o)
|
||||
assert r['status'] == 1
|
||||
|
||||
o = ifc.block_by_number(1)
|
||||
block_src = eth_rpc.do(o)
|
||||
block = ifc.block_from_src(block_src)
|
||||
assert block.number == 1
|
||||
|
||||
with pytest.raises(KeyError):
|
||||
assert block_src['gasUsed'] == 21000
|
||||
assert block_src['gas_used'] == 21000
|
||||
|
||||
block_src = ifc.src_normalize(block_src)
|
||||
assert block_src['gasUsed'] == 21000
|
||||
assert block_src['gas_used'] == 21000
|
||||
|
@ -3,11 +3,15 @@ import os
|
||||
import sys
|
||||
import datetime
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import moolb
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
|
||||
# local imports
|
||||
from cic_cache import db
|
||||
from cic_cache import BloomCache
|
||||
from cic_cache.cache import DEFAULT_FILTER_SIZE
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.dirname(script_dir)
|
||||
@ -39,6 +43,8 @@ def txs(
|
||||
list_tokens,
|
||||
):
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_number = 13
|
||||
@ -51,17 +57,16 @@ def txs(
|
||||
tx_hash_first,
|
||||
list_defaults['block'],
|
||||
tx_number,
|
||||
list_actors['alice'],
|
||||
list_actors['bob'],
|
||||
list_tokens['foo'],
|
||||
list_tokens['foo'],
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.wallet_address(list_actors['bob']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
1024,
|
||||
2048,
|
||||
True,
|
||||
dt.timestamp(),
|
||||
)
|
||||
|
||||
|
||||
tx_number = 42
|
||||
tx_hash_second = '0x' + os.urandom(32).hex()
|
||||
tx_signed_second = '0x' + os.urandom(128).hex()
|
||||
@ -72,10 +77,10 @@ def txs(
|
||||
tx_hash_second,
|
||||
list_defaults['block']-1,
|
||||
tx_number,
|
||||
list_actors['diane'],
|
||||
list_actors['alice'],
|
||||
list_tokens['foo'],
|
||||
list_tokens['foo'],
|
||||
tx_normalize.wallet_address(list_actors['diane']),
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
tx_normalize.wallet_address(list_tokens['foo']),
|
||||
1024,
|
||||
2048,
|
||||
False,
|
||||
@ -84,3 +89,64 @@ def txs(
|
||||
|
||||
session.commit()
|
||||
|
||||
return [
|
||||
tx_hash_first,
|
||||
tx_hash_second,
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def more_txs(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
):
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
tx_number = 666
|
||||
tx_hash = '0x' + os.urandom(32).hex()
|
||||
tx_signed = '0x' + os.urandom(128).hex()
|
||||
nonce = 3
|
||||
|
||||
dt = datetime.datetime.utcnow()
|
||||
dt += datetime.timedelta(hours=1)
|
||||
db.add_transaction(
|
||||
session,
|
||||
tx_hash,
|
||||
list_defaults['block']+2,
|
||||
tx_number,
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.wallet_address(list_actors['diane']),
|
||||
tx_normalize.executable_address(list_tokens['bar']),
|
||||
tx_normalize.executable_address(list_tokens['bar']),
|
||||
2048,
|
||||
4096,
|
||||
False,
|
||||
dt.timestamp(),
|
||||
)
|
||||
|
||||
session.commit()
|
||||
|
||||
return [tx_hash] + txs
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def tag_txs(
|
||||
init_database,
|
||||
txs,
|
||||
):
|
||||
|
||||
db.add_tag(init_database, 'taag', domain='test')
|
||||
init_database.commit()
|
||||
|
||||
db.tag_transaction(init_database, txs[1], 'taag', domain='test')
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def zero_filter():
|
||||
return moolb.Bloom(DEFAULT_FILTER_SIZE, 3)
|
||||
|
3
apps/cic-cache/tests/filters/conftest.py
Normal file
3
apps/cic-cache/tests/filters/conftest.py
Normal file
@ -0,0 +1,3 @@
|
||||
from chainlib.eth.pytest import *
|
||||
from cic_eth_registry.pytest.fixtures_tokens import *
|
||||
|
173
apps/cic-cache/tests/filters/test_erc20.py
Normal file
173
apps/cic-cache/tests/filters/test_erc20.py
Normal file
@ -0,0 +1,173 @@
|
||||
# standard imports
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
from chainlib.eth.tx import Tx
|
||||
from chainlib.eth.block import Block
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.error import RequestMismatchException
|
||||
from hexathon import (
|
||||
strip_0x,
|
||||
add_0x,
|
||||
)
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import add_tag
|
||||
from cic_cache.runnable.daemons.filters.erc20 import ERC20TransferFilter
|
||||
from cic_cache.runnable.daemons.filters.base import TagSyncFilter
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def test_base_filter_str(
|
||||
init_database,
|
||||
):
|
||||
f = TagSyncFilter('foo')
|
||||
assert 'foo' == str(f)
|
||||
f = TagSyncFilter('foo', domain='bar')
|
||||
assert 'bar.foo' == str(f)
|
||||
|
||||
|
||||
|
||||
def test_erc20_filter(
|
||||
eth_rpc,
|
||||
foo_token,
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
tags,
|
||||
):
|
||||
|
||||
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
|
||||
|
||||
fltr = ERC20TransferFilter(chain_spec)
|
||||
|
||||
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
|
||||
|
||||
data = 'a9059cbb'
|
||||
data += strip_0x(list_actors['alice'])
|
||||
data += '1000'.ljust(64, '0')
|
||||
|
||||
block = Block({
|
||||
'hash': os.urandom(32).hex(),
|
||||
'number': 42,
|
||||
'timestamp': datetime.datetime.utcnow().timestamp(),
|
||||
'transactions': [],
|
||||
})
|
||||
|
||||
tx = Tx({
|
||||
'to': foo_token,
|
||||
'from': list_actors['bob'],
|
||||
'data': data,
|
||||
'value': 0,
|
||||
'hash': os.urandom(32).hex(),
|
||||
'nonce': 13,
|
||||
'gasPrice': 10000000,
|
||||
'gas': 123456,
|
||||
})
|
||||
block.txs.append(tx)
|
||||
tx.block = block
|
||||
|
||||
r = fltr.filter(eth_rpc, block, tx, db_session=init_database)
|
||||
assert r
|
||||
|
||||
s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
|
||||
r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
|
||||
assert r[0] == tx.hash
|
||||
|
||||
|
||||
def test_erc20_filter_nocontract(
|
||||
eth_rpc,
|
||||
foo_token,
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
tags,
|
||||
):
|
||||
|
||||
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
|
||||
|
||||
fltr = ERC20TransferFilter(chain_spec)
|
||||
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
|
||||
|
||||
# incomplete args
|
||||
data = 'a9059cbb'
|
||||
data += strip_0x(list_actors['alice'])
|
||||
data += '1000'.ljust(64, '0')
|
||||
block = Block({
|
||||
'hash': os.urandom(32).hex(),
|
||||
'number': 42,
|
||||
'timestamp': datetime.datetime.utcnow().timestamp(),
|
||||
'transactions': [],
|
||||
})
|
||||
|
||||
tx = Tx({
|
||||
'to': os.urandom(20).hex(),
|
||||
'from': list_actors['bob'],
|
||||
'data': data,
|
||||
'value': 0,
|
||||
'hash': os.urandom(32).hex(),
|
||||
'nonce': 13,
|
||||
'gasPrice': 10000000,
|
||||
'gas': 123456,
|
||||
})
|
||||
block.txs.append(tx)
|
||||
tx.block = block
|
||||
|
||||
assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'contract_method,contract_input,expected_exception',
|
||||
[
|
||||
('a9059cbb', os.urandom(32).hex(), ValueError), # not enough args
|
||||
('a9059cbb', os.urandom(31).hex(), ValueError), # wrong arg boundary
|
||||
('a9059cbc', os.urandom(64).hex(), RequestMismatchException), # wrong method
|
||||
],
|
||||
)
|
||||
def test_erc20_filter_bogus(
|
||||
eth_rpc,
|
||||
foo_token,
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
tags,
|
||||
contract_method,
|
||||
contract_input,
|
||||
expected_exception,
|
||||
):
|
||||
|
||||
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
|
||||
|
||||
fltr = ERC20TransferFilter(chain_spec)
|
||||
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
|
||||
|
||||
# incomplete args
|
||||
data = contract_method
|
||||
data += contract_input
|
||||
block = Block({
|
||||
'hash': os.urandom(32).hex(),
|
||||
'number': 42,
|
||||
'timestamp': datetime.datetime.utcnow().timestamp(),
|
||||
'transactions': [],
|
||||
})
|
||||
|
||||
tx = Tx({
|
||||
'to': foo_token,
|
||||
'from': list_actors['bob'],
|
||||
'data': data,
|
||||
'value': 0,
|
||||
'hash': os.urandom(32).hex(),
|
||||
'nonce': 13,
|
||||
'gasPrice': 10000000,
|
||||
'gas': 123456,
|
||||
})
|
||||
block.txs.append(tx)
|
||||
tx.block = block
|
||||
|
||||
assert not fltr.filter(eth_rpc, block, tx, db_session=init_database)
|
71
apps/cic-cache/tests/filters/test_faucet.py
Normal file
71
apps/cic-cache/tests/filters/test_faucet.py
Normal file
@ -0,0 +1,71 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.nonce import RPCNonceOracle
|
||||
from chainlib.eth.block import (
|
||||
block_by_hash,
|
||||
Block,
|
||||
)
|
||||
from chainlib.eth.tx import (
|
||||
receipt,
|
||||
unpack,
|
||||
transaction,
|
||||
Tx,
|
||||
)
|
||||
from hexathon import strip_0x
|
||||
from erc20_faucet.faucet import SingleShotFaucet
|
||||
from sqlalchemy import text
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import add_tag
|
||||
from cic_cache.runnable.daemons.filters.faucet import FaucetFilter
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def test_filter_faucet(
|
||||
eth_rpc,
|
||||
eth_signer,
|
||||
foo_token,
|
||||
faucet_noregistry,
|
||||
init_database,
|
||||
list_defaults,
|
||||
contract_roles,
|
||||
agent_roles,
|
||||
tags,
|
||||
):
|
||||
|
||||
chain_spec = ChainSpec('foo', 'bar', 42, 'baz')
|
||||
|
||||
fltr = FaucetFilter(chain_spec, contract_roles['CONTRACT_DEPLOYER'])
|
||||
|
||||
add_tag(init_database, fltr.tag_name, domain=fltr.tag_domain)
|
||||
|
||||
nonce_oracle = RPCNonceOracle(agent_roles['ALICE'], eth_rpc)
|
||||
c = SingleShotFaucet(chain_spec, signer=eth_signer, nonce_oracle=nonce_oracle)
|
||||
(tx_hash_hex, o) = c.give_to(faucet_noregistry, agent_roles['ALICE'], agent_roles['ALICE'])
|
||||
r = eth_rpc.do(o)
|
||||
|
||||
tx_src = unpack(bytes.fromhex(strip_0x(o['params'][0])), chain_spec)
|
||||
|
||||
o = receipt(r)
|
||||
r = eth_rpc.do(o)
|
||||
rcpt = Tx.src_normalize(r)
|
||||
|
||||
assert r['status'] == 1
|
||||
|
||||
o = block_by_hash(r['block_hash'])
|
||||
r = eth_rpc.do(o)
|
||||
block_object = Block(r)
|
||||
|
||||
tx = Tx(tx_src, block_object)
|
||||
tx.apply_receipt(rcpt)
|
||||
|
||||
r = fltr.filter(eth_rpc, block_object, tx, init_database)
|
||||
assert r
|
||||
|
||||
s = text("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = :a AND a.value = :b")
|
||||
r = init_database.execute(s, {'a': fltr.tag_domain, 'b': fltr.tag_name}).fetchone()
|
||||
assert r[0] == tx.hash
|
@ -2,7 +2,7 @@
|
||||
import os
|
||||
import logging
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import confini
|
||||
|
||||
@ -13,8 +13,9 @@ logg = logging.getLogger(__file__)
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def load_config():
|
||||
config_dir = os.path.join(root_dir, '.config/test')
|
||||
conf = confini.Config(config_dir, 'CICTEST')
|
||||
config_dir = os.path.join(root_dir, 'config/test')
|
||||
schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
|
||||
conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
|
||||
conf.process()
|
||||
logg.debug('config {}'.format(conf))
|
||||
return conf
|
||||
|
@ -3,13 +3,16 @@ import os
|
||||
import logging
|
||||
import re
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import sqlparse
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
# local imports
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db import add_tag
|
||||
|
||||
logg = logging.getLogger(__file__)
|
||||
|
||||
@ -21,16 +24,19 @@ def database_engine(
|
||||
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
SessionBase.transactional = False
|
||||
SessionBase.poolable = False
|
||||
name = 'cic_cache'
|
||||
database_name = name
|
||||
if load_config.get('DATABASE_PREFIX'):
|
||||
database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
|
||||
try:
|
||||
os.unlink(load_config.get('DATABASE_NAME'))
|
||||
os.unlink(database_name)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
dsn = dsn_from_config(load_config)
|
||||
SessionBase.connect(dsn)
|
||||
dsn = dsn_from_config(load_config, name)
|
||||
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
|
||||
return dsn
|
||||
|
||||
|
||||
# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
|
||||
@pytest.fixture(scope='function')
|
||||
def init_database(
|
||||
load_config,
|
||||
@ -38,52 +44,23 @@ def init_database(
|
||||
):
|
||||
|
||||
rootdir = os.path.dirname(os.path.dirname(__file__))
|
||||
schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
|
||||
|
||||
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
rconn = SessionBase.engine.raw_connection()
|
||||
f = open(os.path.join(schemadir, 'db.sql'))
|
||||
s = f.read()
|
||||
f.close()
|
||||
rconn.executescript(s)
|
||||
|
||||
else:
|
||||
rconn = SessionBase.engine.raw_connection()
|
||||
rcursor = rconn.cursor()
|
||||
|
||||
#rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
|
||||
#rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
|
||||
|
||||
f = open(os.path.join(schemadir, 'db.sql'))
|
||||
s = f.read()
|
||||
f.close()
|
||||
r = re.compile(r'^[A-Z]', re.MULTILINE)
|
||||
for l in sqlparse.parse(s):
|
||||
strl = str(l)
|
||||
# we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
|
||||
if not re.search(r, strl):
|
||||
logg.warning('skipping parsed query line {}'.format(strl))
|
||||
continue
|
||||
rcursor.execute(strl)
|
||||
rconn.commit()
|
||||
|
||||
rcursor.execute('SET search_path TO public')
|
||||
|
||||
# this doesn't work when run separately, no idea why
|
||||
# functions have been manually added to original schema from cic-eth
|
||||
# f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
|
||||
# s = f.read()
|
||||
# f.close()
|
||||
# rcursor.execute(s)
|
||||
#
|
||||
# f = open(os.path.join(schemadir, 'proc_balances.sql'))
|
||||
# s = f.read()
|
||||
# f.close()
|
||||
# rcursor.execute(s)
|
||||
|
||||
rcursor.close()
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
|
||||
if not os.path.isdir(migrationsdir):
|
||||
migrationsdir = os.path.join(dbdir, 'migrations', 'default')
|
||||
logg.info('using migrations directory {}'.format(migrationsdir))
|
||||
|
||||
session = SessionBase.create_session()
|
||||
|
||||
ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
|
||||
ac.set_main_option('sqlalchemy.url', database_engine)
|
||||
ac.set_main_option('script_location', migrationsdir)
|
||||
|
||||
alembic.command.downgrade(ac, 'base')
|
||||
alembic.command.upgrade(ac, 'head')
|
||||
|
||||
session.commit()
|
||||
|
||||
yield session
|
||||
session.commit()
|
||||
session.close()
|
||||
@ -116,3 +93,14 @@ def list_defaults(
|
||||
return {
|
||||
'block': 420000,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def tags(
|
||||
init_database,
|
||||
):
|
||||
|
||||
add_tag(init_database, 'foo')
|
||||
add_tag(init_database, 'baz', domain='bar')
|
||||
add_tag(init_database, 'xyzzy', domain='bar')
|
||||
init_database.commit()
|
||||
|
31
apps/cic-cache/tests/test_api.py
Normal file
31
apps/cic-cache/tests/test_api.py
Normal file
@ -0,0 +1,31 @@
|
||||
# standard imports
|
||||
import json
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
from cic_cache.runnable.daemons.query import process_transactions_all_data
|
||||
|
||||
|
||||
def test_api_all_data(
|
||||
init_database,
|
||||
txs,
|
||||
):
|
||||
|
||||
env = {
|
||||
'PATH_INFO': '/txa/100/0/410000/420000',
|
||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||
}
|
||||
j = process_transactions_all_data(init_database, env)
|
||||
o = json.loads(j[1])
|
||||
|
||||
assert len(o['data']) == 2
|
||||
|
||||
env = {
|
||||
'PATH_INFO': '/txa/100/0/420000/410000',
|
||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
j = process_transactions_all_data(init_database, env)
|
@ -4,11 +4,14 @@ import datetime
|
||||
import logging
|
||||
import json
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
|
||||
# local imports
|
||||
from cic_cache import db
|
||||
from cic_cache import BloomCache
|
||||
from cic_cache.cache import DataCache
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
@ -17,7 +20,6 @@ def test_cache(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
):
|
||||
|
||||
@ -33,3 +35,237 @@ def test_cache(
|
||||
|
||||
assert b[0] == list_defaults['block'] - 1
|
||||
|
||||
|
||||
def test_cache_data(
|
||||
init_database,
|
||||
txs,
|
||||
tag_txs,
|
||||
):
|
||||
|
||||
session = init_database
|
||||
|
||||
c = DataCache(session)
|
||||
b = c.load_transactions_with_data(0, 3) #410000, 420000) #, 100, block_offset=410000, block_limit=420000, oldest=True)
|
||||
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == txs[0]
|
||||
assert b[2][0]['tx_type'] == 'unknown'
|
||||
assert b[2][1]['tx_type'] == 'test.taag'
|
||||
|
||||
|
||||
def test_cache_ranges(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
more_txs,
|
||||
):
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
oldest = list_defaults['block'] - 1
|
||||
mid = list_defaults['block']
|
||||
newest = list_defaults['block'] + 2
|
||||
|
||||
c = BloomCache(session)
|
||||
b = c.load_transactions(0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
b = c.load_transactions(1, 2)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
|
||||
b = c.load_transactions(0, 2)
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
b = c.load_transactions(0, 1)
|
||||
assert b[0] == newest
|
||||
assert b[1] == newest
|
||||
|
||||
b = c.load_transactions(0, 100, oldest=True)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
b = c.load_transactions(0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
|
||||
b = c.load_transactions(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
|
||||
# now check when supplying account
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
# add block filter to the mix
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == oldest
|
||||
|
||||
|
||||
def test_cache_ranges_data(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
more_txs,
|
||||
):
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
oldest = list_defaults['block'] - 1
|
||||
mid = list_defaults['block']
|
||||
newest = list_defaults['block'] + 2
|
||||
|
||||
c = DataCache(session)
|
||||
|
||||
b = c.load_transactions_with_data(0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 3
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][2]['tx_hash'] == more_txs[2]
|
||||
|
||||
b = c.load_transactions_with_data(1, 2)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
assert b[2][1]['tx_hash'] == more_txs[2]
|
||||
|
||||
b = c.load_transactions_with_data(0, 2)
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
b = c.load_transactions_with_data(0, 1)
|
||||
assert b[0] == newest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
|
||||
b = c.load_transactions_with_data(0, 100, oldest=True)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 3
|
||||
assert b[2][0]['tx_hash'] == more_txs[2]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
assert b[2][2]['tx_hash'] == more_txs[0]
|
||||
|
||||
b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
assert b[2][1]['tx_hash'] == more_txs[2]
|
||||
|
||||
b = c.load_transactions_with_data(0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'], oldest=True)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[2]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
# now check when supplying account
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 3
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
assert b[2][2]['tx_hash'] == more_txs[2]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[2]
|
||||
|
||||
# add block filter to the mix
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == oldest
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[2]
|
||||
|
232
apps/cic-cache/tests/test_query.py
Normal file
232
apps/cic-cache/tests/test_query.py
Normal file
@ -0,0 +1,232 @@
|
||||
# standard imports
|
||||
import logging
|
||||
import json
|
||||
import base64
|
||||
import copy
|
||||
import re
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
from cic_cache.runnable.daemons.query import *
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'query_path_prefix, query_role, query_address_index, query_offset, query_offset_index, query_limit, query_limit_index, match_re',
|
||||
[
|
||||
('/tx/user/', 'alice', 0, None, 3, None, 5, re_transactions_account_bloom),
|
||||
('/tx/user/', 'alice', 0, 42, 3, None, 5, re_transactions_account_bloom),
|
||||
('/tx/user/', 'alice', 0, 42, 3, 13, 5, re_transactions_account_bloom),
|
||||
('/tx/', None, 0, None, 3, None, 5, re_transactions_all_bloom),
|
||||
('/tx/', None, 0, 42, 3, None, 5, re_transactions_all_bloom),
|
||||
('/tx/', None, 0, 42, 3, 13, 5, re_transactions_all_bloom),
|
||||
('/txa/', None, 0, None, 3, None, 5, re_transactions_all_data),
|
||||
('/txa/', None, 0, 42, 3, None, 5, re_transactions_all_data),
|
||||
('/txa/', None, 0, 42, 3, 13, 5, re_transactions_all_data),
|
||||
],
|
||||
)
|
||||
def test_query_regex(
|
||||
list_actors,
|
||||
query_path_prefix,
|
||||
query_role,
|
||||
query_address_index,
|
||||
query_offset,
|
||||
query_offset_index,
|
||||
query_limit,
|
||||
query_limit_index,
|
||||
match_re,
|
||||
):
|
||||
|
||||
paths = []
|
||||
path = query_path_prefix
|
||||
query_address = None
|
||||
if query_role != None:
|
||||
query_address = strip_0x(list_actors[query_role])
|
||||
paths.append(path + '0x' + query_address)
|
||||
paths.append(path + query_address)
|
||||
if query_offset != None:
|
||||
if query_limit != None:
|
||||
for i in range(len(paths)-1):
|
||||
paths[i] += '/{}/{}'.format(query_offset, query_limit)
|
||||
else:
|
||||
for i in range(len(paths)-1):
|
||||
paths[i] += '/' + str(query_offset)
|
||||
|
||||
for i in range(len(paths)):
|
||||
paths.append(paths[i] + '/')
|
||||
|
||||
for p in paths:
|
||||
logg.debug('testing path {} against {}'.format(p, match_re))
|
||||
m = re.match(match_re, p)
|
||||
l = len(m.groups())
|
||||
logg.debug('laast index match {} groups {}'.format(m.lastindex, l))
|
||||
for i in range(l+1):
|
||||
logg.debug('group {} {}'.format(i, m[i]))
|
||||
if m.lastindex >= query_offset_index:
|
||||
assert query_offset == int(m[query_offset_index + 1])
|
||||
if m.lastindex >= query_limit_index:
|
||||
assert query_limit == int(m[query_limit_index + 1])
|
||||
if query_address_index != None:
|
||||
match_address = strip_0x(m[query_address_index + 1])
|
||||
assert query_address == match_address
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'role_name, query_offset, query_limit, query_match',
|
||||
[
|
||||
('alice', None, None, [(420000, 13), (419999, 42)]),
|
||||
('alice', None, 1, [(420000, 13)]),
|
||||
('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
|
||||
('alice', 2, None, []), # 420000 == list_defaults['block']
|
||||
],
|
||||
)
|
||||
def test_query_process_txs_account(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
zero_filter,
|
||||
role_name,
|
||||
query_offset,
|
||||
query_limit,
|
||||
query_match,
|
||||
):
|
||||
|
||||
actor = None
|
||||
try:
|
||||
actor = list_actors[role_name]
|
||||
except KeyError:
|
||||
actor = os.urandom(20).hex()
|
||||
path_info = '/tx/user/0x' + strip_0x(actor)
|
||||
if query_offset != None:
|
||||
path_info += '/' + str(query_offset)
|
||||
if query_limit == None:
|
||||
query_limit = 100
|
||||
path_info += '/' + str(query_limit)
|
||||
if query_offset == None:
|
||||
path_info += '/0'
|
||||
env = {
|
||||
'PATH_INFO': path_info,
|
||||
}
|
||||
logg.debug('using path {}'.format(path_info))
|
||||
r = process_transactions_account_bloom(init_database, env)
|
||||
assert r != None
|
||||
|
||||
o = json.loads(r[1])
|
||||
block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
|
||||
zero_filter_data = zero_filter.to_bytes()
|
||||
if len(query_match) == 0:
|
||||
assert block_filter_data == zero_filter_data
|
||||
return
|
||||
|
||||
assert block_filter_data != zero_filter_data
|
||||
block_filter = copy.copy(zero_filter)
|
||||
block_filter.merge(block_filter_data)
|
||||
block_filter_data = block_filter.to_bytes()
|
||||
assert block_filter_data != zero_filter_data
|
||||
|
||||
for (block, tx) in query_match:
|
||||
block = block.to_bytes(4, byteorder='big')
|
||||
assert block_filter.check(block)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'query_offset, query_limit, query_match',
|
||||
[
|
||||
(None, 2, [(420000, 13), (419999, 42)]),
|
||||
(0, 1, [(420000, 13)]),
|
||||
(1, 1, [(419999, 42)]),
|
||||
(2, 0, []),
|
||||
],
|
||||
)
|
||||
def test_query_process_txs_bloom(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
zero_filter,
|
||||
query_offset,
|
||||
query_limit,
|
||||
query_match,
|
||||
):
|
||||
|
||||
path_info = '/tx'
|
||||
if query_offset != None:
|
||||
path_info += '/' + str(query_offset)
|
||||
if query_limit != None:
|
||||
if query_offset == None:
|
||||
path_info += '/0'
|
||||
path_info += '/' + str(query_limit)
|
||||
env = {
|
||||
'PATH_INFO': path_info,
|
||||
}
|
||||
logg.debug('using path {}'.format(path_info))
|
||||
r = process_transactions_all_bloom(init_database, env)
|
||||
assert r != None
|
||||
|
||||
o = json.loads(r[1])
|
||||
block_filter_data = base64.b64decode(o['block_filter'].encode('utf-8'))
|
||||
zero_filter_data = zero_filter.to_bytes()
|
||||
if len(query_match) == 0:
|
||||
assert block_filter_data == zero_filter_data
|
||||
return
|
||||
|
||||
assert block_filter_data != zero_filter_data
|
||||
block_filter = copy.copy(zero_filter)
|
||||
block_filter.merge(block_filter_data)
|
||||
block_filter_data = block_filter.to_bytes()
|
||||
assert block_filter_data != zero_filter_data
|
||||
|
||||
for (block, tx) in query_match:
|
||||
block = block.to_bytes(4, byteorder='big')
|
||||
assert block_filter.check(block)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'query_block_start, query_block_end, query_match_count',
|
||||
[
|
||||
(1, 42, 0),
|
||||
(420000, 420001, 1),
|
||||
(419999, 419999, 1), # matches are inclusive
|
||||
(419999, 420000, 2),
|
||||
(419999, 420001, 2),
|
||||
],
|
||||
)
|
||||
def test_query_process_txs_data(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
zero_filter,
|
||||
query_block_start,
|
||||
query_block_end,
|
||||
query_match_count,
|
||||
):
|
||||
|
||||
path_info = '/txa/100/0'
|
||||
if query_block_start != None:
|
||||
path_info += '/' + str(query_block_start)
|
||||
if query_block_end != None:
|
||||
if query_block_start == None:
|
||||
path_info += '/0'
|
||||
path_info += '/' + str(query_block_end)
|
||||
env = {
|
||||
'PATH_INFO': path_info,
|
||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||
}
|
||||
logg.debug('using path {}'.format(path_info))
|
||||
r = process_transactions_all_data(init_database, env)
|
||||
assert r != None
|
||||
|
||||
o = json.loads(r[1])
|
||||
logg.debug('oo {}'.format(o))
|
||||
assert len(o['data']) == query_match_count
|
37
apps/cic-cache/tests/test_tag.py
Normal file
37
apps/cic-cache/tests/test_tag.py
Normal file
@ -0,0 +1,37 @@
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import tag_transaction
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def test_cache(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
tags,
|
||||
):
|
||||
|
||||
tag_transaction(init_database, txs[0], 'foo')
|
||||
tag_transaction(init_database, txs[0], 'baz', domain='bar')
|
||||
tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
|
||||
assert r[0][0] == txs[0]
|
||||
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
|
||||
assert r[0][0] == txs[0]
|
||||
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
|
||||
assert r[0][0] == txs[1]
|
2
apps/cic-cache/tests/testdata/config/test.ini
vendored
Normal file
2
apps/cic-cache/tests/testdata/config/test.ini
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
[foo]
|
||||
bar_baz = xyzzy
|
1
apps/cic-eth-aux/erc20-demurrage-token/MANIFEST.in
Normal file
1
apps/cic-eth-aux/erc20-demurrage-token/MANIFEST.in
Normal file
@ -0,0 +1 @@
|
||||
include *requirements.txt
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user