Compare commits
1 Commits
master
...
bvander/sw
Author | SHA1 | Date | |
---|---|---|---|
894d8bcd21 |
37
.env
Normal file
37
.env
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
DOMAIN=localhost
|
||||||
|
|
||||||
|
STACK_NAME=cic-net
|
||||||
|
|
||||||
|
TRAEFIK_PUBLIC_NETWORK=traefik-public
|
||||||
|
TRAEFIK_TAG=cic.net
|
||||||
|
TRAEFIK_PUBLIC_TAG=traefik-public
|
||||||
|
|
||||||
|
FRONTEND_ENV=dev
|
||||||
|
|
||||||
|
# Flower
|
||||||
|
FLOWER_BASIC_AUTH=admin:changethis
|
||||||
|
|
||||||
|
# Postgres
|
||||||
|
DATABASE_HOST=postgres
|
||||||
|
DATABASE_PORT=5432
|
||||||
|
DATABASE_ENGINE=postgresql
|
||||||
|
DATABASE_DRIVER=psycopg2
|
||||||
|
DATABASE_USER=postgres
|
||||||
|
|
||||||
|
# Redis
|
||||||
|
REDIS_HOST=redis
|
||||||
|
REDIS_PORT=6379
|
||||||
|
REDIS_DB=0
|
||||||
|
|
||||||
|
# Celery Broker
|
||||||
|
CELERY_BROKER_URL=redis://redis:6379
|
||||||
|
CELERY_RESULT_URL=redis://redis:6379
|
||||||
|
|
||||||
|
# Blockchain node
|
||||||
|
ETH_PROVIDER=http://eth:8545
|
||||||
|
CIC_CHAIN_SPEC=evm:bloxberg:8996
|
||||||
|
|
||||||
|
# PgAdmin
|
||||||
|
PGADMIN_LISTEN_PORT=5050
|
||||||
|
PGADMIN_DEFAULT_EMAIL=admin@cic.net
|
||||||
|
PGADMIN_DEFAULT_PASSWORD=changethis
|
3
.gitignore
vendored
3
.gitignore
vendored
@ -14,4 +14,5 @@ build/
|
|||||||
**/.venv
|
**/.venv
|
||||||
.idea
|
.idea
|
||||||
**/.vim
|
**/.vim
|
||||||
**/*secret.yaml
|
docker-stack.yml
|
||||||
|
.npm/
|
||||||
|
132
.gitlab-ci.yml
132
.gitlab-ci.yml
@ -1,81 +1,99 @@
|
|||||||
include:
|
include:
|
||||||
#- local: 'ci_templates/.cic-template.yml' #kaniko build templates
|
# - local: 'ci_templates/.cic-template.yml'
|
||||||
# these includes are app specific unit tests
|
# - local: 'apps/contract-migration/.gitlab-ci.yml'
|
||||||
- local: 'apps/cic-eth/.gitlab-ci.yml'
|
- local: 'apps/cic-eth/.gitlab-ci.yml'
|
||||||
- local: 'apps/cic-ussd/.gitlab-ci.yml'
|
- local: 'apps/cic-ussd/.gitlab-ci.yml'
|
||||||
- local: 'apps/cic-notify/.gitlab-ci.yml'
|
- local: 'apps/cic-notify/.gitlab-ci.yml'
|
||||||
- local: 'apps/cic-meta/.gitlab-ci.yml'
|
- local: 'apps/cic-meta/.gitlab-ci.yml'
|
||||||
- local: 'apps/cic-cache/.gitlab-ci.yml'
|
- local: 'apps/cic-cache/.gitlab-ci.yml'
|
||||||
#- local: 'apps/contract-migration/.gitlab-ci.yml'
|
# - local: 'apps/data-seeding/.gitlab-ci.yml'
|
||||||
#- local: 'apps/data-seeding/.gitlab-ci.yml'
|
|
||||||
|
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
|
||||||
|
|
||||||
|
|
||||||
stages:
|
stages:
|
||||||
- version
|
|
||||||
- build
|
- build
|
||||||
- test
|
- test
|
||||||
- deploy
|
- deploy
|
||||||
|
|
||||||
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/docker-with-compose:latest
|
|
||||||
|
|
||||||
variables:
|
variables:
|
||||||
DOCKER_BUILDKIT: "1"
|
DOCKER_BUILDKIT: "1"
|
||||||
COMPOSE_DOCKER_CLI_BUILD: "1"
|
COMPOSE_DOCKER_CLI_BUILD: "1"
|
||||||
CI_DEBUG_TRACE: "true"
|
MR_IMAGE_TAG: mr-$CI_COMMIT_SHORT_SHA
|
||||||
SEMVERBOT_VERSION: "0.2.0"
|
|
||||||
|
|
||||||
#before_script:
|
# todo you can probably just build the single image w/o docker-compose
|
||||||
# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
build-merge-request:
|
||||||
|
before_script:
|
||||||
version:
|
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||||
#image: python:3.7-stretch
|
|
||||||
image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae
|
|
||||||
stage: version
|
|
||||||
tags:
|
|
||||||
- integration
|
|
||||||
script:
|
|
||||||
- mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
|
||||||
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
|
|
||||||
- eval $(ssh-agent -s)
|
|
||||||
- ssh-add <(echo "$SSH_PRIVATE_KEY")
|
|
||||||
- git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
|
|
||||||
- export TAG=$(sbot predict version -m auto)
|
|
||||||
- |
|
|
||||||
if [[ -z $TAG ]]
|
|
||||||
then
|
|
||||||
echo "tag could not be set $@"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
- echo $TAG > version
|
|
||||||
- git tag -a v$TAG -m "ci tagged"
|
|
||||||
- git push origin v$TAG
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- version
|
|
||||||
rules:
|
|
||||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
|
||||||
when: always
|
|
||||||
- if: $CI_COMMIT_REF_NAME == "master"
|
|
||||||
when: always
|
|
||||||
|
|
||||||
# runs on protected branches and pushes to repo
|
|
||||||
build-push:
|
|
||||||
stage: build
|
stage: build
|
||||||
tags:
|
tags:
|
||||||
- integration
|
- integration
|
||||||
#script:
|
variables:
|
||||||
# - TAG=$CI_Cbefore_script:
|
CI_DEBUG_TRACE: "true"
|
||||||
before_script:
|
|
||||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
|
||||||
script:
|
script:
|
||||||
- TAG=latest ./scripts/build-push.sh
|
- TAG=$MR_IMAGE_TAG FRONTEND_ENV=dev sh ./scripts/build-push.sh
|
||||||
- TAG=$(cat ./version) ./scripts/build-push.sh
|
|
||||||
rules:
|
rules:
|
||||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
when: always
|
|
||||||
- if: $CI_COMMIT_REF_NAME == "master"
|
|
||||||
when: always
|
when: always
|
||||||
|
|
||||||
deploy-dev:
|
build-staging:
|
||||||
|
before_script:
|
||||||
|
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||||
|
tags:
|
||||||
|
- integration
|
||||||
|
#- blocal
|
||||||
|
variables:
|
||||||
|
CI_DEBUG_TRACE: "true"
|
||||||
|
stage: build
|
||||||
|
script:
|
||||||
|
- TAG=stag FRONTEND_ENV=staging sh ./scripts/build-push.sh
|
||||||
|
only:
|
||||||
|
- staging
|
||||||
|
|
||||||
|
deploy-staging:
|
||||||
|
before_script:
|
||||||
|
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||||
|
- pip install docker-auto-labels
|
||||||
|
tags:
|
||||||
|
- integration
|
||||||
stage: deploy
|
stage: deploy
|
||||||
trigger: grassrootseconomics/devops
|
script:
|
||||||
when: manual
|
- >
|
||||||
|
DOMAIN=stag.grassrootseconomics.net
|
||||||
|
TRAEFIK_TAG=grassrootseconomics.net
|
||||||
|
STACK_NAME=stag-cic-net
|
||||||
|
TAG=stag
|
||||||
|
sh ./scripts/deploy.sh
|
||||||
|
environment:
|
||||||
|
name: staging
|
||||||
|
url: https://stag.grassrootseconomics.net
|
||||||
|
only:
|
||||||
|
- staging
|
||||||
|
|
||||||
|
#build-prod:
|
||||||
|
# stage: build
|
||||||
|
# script:
|
||||||
|
# - TAG=prod FRONTEND_ENV=production sh ./scripts/build-push.sh
|
||||||
|
# only:
|
||||||
|
# - production
|
||||||
|
# tags:
|
||||||
|
# - build
|
||||||
|
# - test
|
||||||
|
#
|
||||||
|
#deploy-prod:
|
||||||
|
# stage: deploy
|
||||||
|
# script:
|
||||||
|
# - >
|
||||||
|
# DOMAIN=demo1.com
|
||||||
|
# TRAEFIK_TAG=demo1.com
|
||||||
|
# STACK_NAME=demo1-com
|
||||||
|
# TAG=prod
|
||||||
|
# sh ./scripts/deploy.sh
|
||||||
|
# environment:
|
||||||
|
# name: production
|
||||||
|
# url: https://demo1.com
|
||||||
|
# only:
|
||||||
|
# - production
|
||||||
|
# tags:
|
||||||
|
# - swarm
|
||||||
|
# - prod
|
||||||
|
@ -1,44 +0,0 @@
|
|||||||
<!---
|
|
||||||
Please read this!
|
|
||||||
|
|
||||||
Before opening a new issue, make sure to search for keywords in the issues
|
|
||||||
filtered by the "bug" label:
|
|
||||||
|
|
||||||
- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
|
|
||||||
|
|
||||||
and verify the issue you're about to submit isn't a duplicate.
|
|
||||||
--->
|
|
||||||
|
|
||||||
### Summary
|
|
||||||
|
|
||||||
<!-- Summarize the bug encountered concisely. -->
|
|
||||||
|
|
||||||
### Steps to reproduce
|
|
||||||
|
|
||||||
<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
|
|
||||||
|
|
||||||
### Example Project
|
|
||||||
|
|
||||||
<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic
|
|
||||||
behavior, and link to it here in the bug report. If you are using an older version of GitLab, this
|
|
||||||
will also determine whether the bug is fixed in a more recent version. -->
|
|
||||||
|
|
||||||
### What is the current *bug* behavior?
|
|
||||||
|
|
||||||
<!-- Describe what actually happens. -->
|
|
||||||
|
|
||||||
### What is the expected *correct* behavior?
|
|
||||||
|
|
||||||
<!-- Describe what you should see instead. -->
|
|
||||||
|
|
||||||
### Relevant logs and/or screenshots
|
|
||||||
|
|
||||||
<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
|
|
||||||
as it's tough to read otherwise. -->
|
|
||||||
|
|
||||||
|
|
||||||
### Possible fixes
|
|
||||||
|
|
||||||
<!-- If you can, link to the line of code that might be responsible for the problem. -->
|
|
||||||
|
|
||||||
/label ~"bug"
|
|
@ -1,16 +0,0 @@
|
|||||||
[git]
|
|
||||||
|
|
||||||
[git.config]
|
|
||||||
email = "semverbot@grassroots.org"
|
|
||||||
name = "semvervot"
|
|
||||||
|
|
||||||
[git.tags]
|
|
||||||
prefix = "v"
|
|
||||||
|
|
||||||
[semver]
|
|
||||||
mode = "git-commit"
|
|
||||||
|
|
||||||
[semver.detection]
|
|
||||||
patch = ["fix", "[fix]", "patch", "[patch]"]
|
|
||||||
minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
|
|
||||||
major = ["BREAKING CHANGE"]
|
|
@ -1,83 +0,0 @@
|
|||||||
# Contributor Covenant Code of Conduct
|
|
||||||
|
|
||||||
## Our Pledge
|
|
||||||
|
|
||||||
In the interest of fostering an open and welcoming environment, we as
|
|
||||||
contributors and maintainers pledge to make participation in our project and
|
|
||||||
our community a harassment-free experience for everyone, regardless of age, body
|
|
||||||
size, disability, ethnicity, sex characteristics, gender identity and expression,
|
|
||||||
level of experience, education, socio-economic status, nationality, personal
|
|
||||||
appearance, race, religion, or sexual identity and orientation.
|
|
||||||
|
|
||||||
## Our Standards
|
|
||||||
|
|
||||||
Examples of behavior that contributes to creating a positive environment
|
|
||||||
include:
|
|
||||||
|
|
||||||
* Using welcoming and inclusive language
|
|
||||||
* Being respectful of differing viewpoints and experiences
|
|
||||||
* Gracefully accepting constructive criticism
|
|
||||||
* Focusing on what is best for the community
|
|
||||||
* Showing empathy towards other community members
|
|
||||||
|
|
||||||
Examples of unacceptable behavior by participants include:
|
|
||||||
|
|
||||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
|
||||||
advances
|
|
||||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
|
||||||
* Public or private harassment
|
|
||||||
* Publishing others' private information, such as a physical or electronic
|
|
||||||
address, without explicit permission
|
|
||||||
* Other conduct which could reasonably be considered inappropriate in a
|
|
||||||
professional setting
|
|
||||||
|
|
||||||
## Our Responsibilities
|
|
||||||
|
|
||||||
Project maintainers are responsible for clarifying the standards of acceptable
|
|
||||||
behavior and are expected to take appropriate and fair corrective action in
|
|
||||||
response to any instances of unacceptable behavior.
|
|
||||||
|
|
||||||
Project maintainers have the right and responsibility to remove, edit, or
|
|
||||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
|
||||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
|
||||||
permanently any contributor for other behaviors that they deem inappropriate,
|
|
||||||
threatening, offensive, or harmful.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
This Code of Conduct applies within all project spaces, and it also applies when
|
|
||||||
an individual is representing the project or its community in public spaces.
|
|
||||||
Examples of representing a project or community include using an official
|
|
||||||
project e-mail address, posting via an official social media account, or acting
|
|
||||||
as an appointed representative at an online or offline event. Representation of
|
|
||||||
a project may be further defined and clarified by project maintainers.
|
|
||||||
|
|
||||||
## Enforcement
|
|
||||||
|
|
||||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
|
||||||
reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
|
|
||||||
complaints will be reviewed and investigated and will result in a response that
|
|
||||||
is deemed necessary and appropriate to the circumstances. The project team is
|
|
||||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
|
||||||
Further details of specific enforcement policies may be posted separately.
|
|
||||||
|
|
||||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
|
||||||
faith may face temporary or permanent repercussions as determined by other
|
|
||||||
members of the project's leadership.
|
|
||||||
|
|
||||||
## Attribution
|
|
||||||
|
|
||||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
|
||||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
|
||||||
|
|
||||||
[homepage]: https://www.contributor-covenant.org
|
|
||||||
|
|
||||||
For answers to common questions about this code of conduct, see
|
|
||||||
https://www.contributor-covenant.org/faq
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
...Try to keep in mind the immortal
|
|
||||||
words of Bill and Ted, "Be excellent to each other."
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
|||||||
Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider.
|
|
||||||
|
|
||||||
__To request a change to the code please fork this repository and sumbit a merge request.__
|
|
||||||
|
|
||||||
__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
|
|
||||||
|
|
||||||
__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
|
|
||||||
|
|
||||||
__Ask a question in our dev chat:__
|
|
||||||
|
|
||||||
[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
|
|
||||||
|
|
||||||
[Discord](https://discord.gg/XWunwAsX)
|
|
||||||
|
|
||||||
[Matrix, IRC soon?]
|
|
||||||
|
|
@ -1,117 +0,0 @@
|
|||||||
# CORE TEAM CONTRIBUTION GUIDE
|
|
||||||
|
|
||||||
# 1. Transparency
|
|
||||||
|
|
||||||
1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
|
|
||||||
|
|
||||||
1.2 A work log SHOULD be submitted after a "unit of work" is complete.
|
|
||||||
|
|
||||||
1.2.1 A "unit of work" should not span more than one full day's worth of work.
|
|
||||||
|
|
||||||
1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
|
|
||||||
|
|
||||||
1.3 Individual logs are reviewed in weekly meetings
|
|
||||||
|
|
||||||
<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
|
|
||||||
|
|
||||||
```
|
|
||||||
- Core
|
|
||||||
* fixed foo
|
|
||||||
* fixed bar
|
|
||||||
- Frontend
|
|
||||||
* connected bar to baz
|
|
||||||
|
|
||||||
```-->
|
|
||||||
|
|
||||||
1.4 Work log format is defined in []()
|
|
||||||
|
|
||||||
1.5 Link to issue/MR in bullet point where appropriate
|
|
||||||
|
|
||||||
1.6
|
|
||||||
|
|
||||||
|
|
||||||
# 2. Code hygiene
|
|
||||||
|
|
||||||
2.1 Keep function names and variable names short
|
|
||||||
|
|
||||||
2.2 Keep code files, functions and test fixtures short
|
|
||||||
|
|
||||||
2.3 The less magic the better. Recombinable and replaceable is king
|
|
||||||
|
|
||||||
2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
|
|
||||||
|
|
||||||
2.5 Only auto-import when necessary, and always with a minimum of side-effects
|
|
||||||
|
|
||||||
2.6 Use custom errors. Let them bubble up
|
|
||||||
|
|
||||||
2.7 No logs in tight loops
|
|
||||||
|
|
||||||
2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
|
|
||||||
|
|
||||||
2.9 Test coverage MUST be kept higher than 90% after changes
|
|
||||||
|
|
||||||
2.10 Docstrings. Always. Always!
|
|
||||||
|
|
||||||
|
|
||||||
# 3. Versioning
|
|
||||||
|
|
||||||
3.1 Use [Semantic Versioning](https://semver.org/)
|
|
||||||
|
|
||||||
3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
|
|
||||||
|
|
||||||
|
|
||||||
# 4. Issues
|
|
||||||
|
|
||||||
4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
|
|
||||||
|
|
||||||
4.2 Issues need proper problem statement
|
|
||||||
|
|
||||||
4.2.1. What is the current state
|
|
||||||
|
|
||||||
4.2.2. If current state is not behaving as expected, what was the expected state
|
|
||||||
|
|
||||||
4.2.3. What is the desired new state.
|
|
||||||
|
|
||||||
4.3 Issues need proper resolution statement
|
|
||||||
|
|
||||||
4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
|
|
||||||
|
|
||||||
4.3.2. Builet point list of external resources informing the issue and resolution
|
|
||||||
|
|
||||||
4.4 Tasks needs to be appropriately labelled using GROUP labels.
|
|
||||||
|
|
||||||
|
|
||||||
# 5. Code submission
|
|
||||||
|
|
||||||
5.1 A branch and new MR is always created BEFORE THE WORK STARTS
|
|
||||||
|
|
||||||
5.2 An MR should solve ONE SINGLE PART of a problem
|
|
||||||
|
|
||||||
5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
|
|
||||||
|
|
||||||
5.4 MRs should not be open for more than one week (during normal operation periods)
|
|
||||||
|
|
||||||
5.5 MR should ideally not be longer than 400 lines of changes of logic
|
|
||||||
|
|
||||||
5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
|
|
||||||
|
|
||||||
|
|
||||||
# 6. Code reviews
|
|
||||||
|
|
||||||
6.1 At least one peer review before merge
|
|
||||||
|
|
||||||
6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
|
|
||||||
|
|
||||||
6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
|
|
||||||
|
|
||||||
6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
|
|
||||||
|
|
||||||
6.5 If CI is not working, reviewer MUST make sure code builds and runs
|
|
||||||
|
|
||||||
6.6 Behave!
|
|
||||||
|
|
||||||
6.6.1 Don't be a jerk
|
|
||||||
|
|
||||||
6.6.2 Don't block needlessly
|
|
||||||
|
|
||||||
6.6.3 Say please
|
|
105
README.md
105
README.md
@ -1,19 +1,104 @@
|
|||||||
# Community Inclusion Currency Stack (CIC Stack)
|
# cic-internal-integration
|
||||||
|
|
||||||
A custodial evm wallet for executing transactions via USSD
|
## Backend Requirements
|
||||||
|
|
||||||
## Getting started
|
* [Docker](https://www.docker.com/).
|
||||||
|
* [Docker Compose](https://docs.docker.com/compose/install/).
|
||||||
|
|
||||||
This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
|
## Backend local development
|
||||||
|
|
||||||
```
|
* Start the stack with Docker Compose:
|
||||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
|
||||||
export DOCKER_BUILDKIT=1
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
```
|
```
|
||||||
|
|
||||||
To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
|
* Now you can open your browser and interact with these URLs:
|
||||||
|
|
||||||
## Documentation
|
Frontend (CICADA), built with Docker, with routes handled based on the path: http://localhost
|
||||||
|
|
||||||
|
PGAdmin, PostgreSQL web administration: http://localhost:5050
|
||||||
|
|
||||||
|
Flower, administration of Celery tasks: http://localhost:5555
|
||||||
|
|
||||||
|
Traefik UI, to see how the routes are being handled by the proxy: http://localhost:8090
|
||||||
|
|
||||||
|
**Note**: The first time you start your stack, it might take a minute for it to be ready. While the backend waits for the database to be ready and configures everything. You can check the logs to monitor it.
|
||||||
|
|
||||||
|
To check the logs, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose logs
|
||||||
|
```
|
||||||
|
|
||||||
|
To check the logs of a specific service, add the name of the service, e.g.:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose logs backend
|
||||||
|
```
|
||||||
|
|
||||||
|
If your Docker is not running in `localhost` (the URLs above wouldn't work) check the sections below on **Development with Docker Toolbox** and **Development with a custom IP**.
|
||||||
|
|
||||||
|
#### Deploy the stack locally
|
||||||
|
|
||||||
|
If you want to run the docker stack locally on swarm
|
||||||
|
|
||||||
|
```
|
||||||
|
docker-compose -f docker-compose.yml -f docker-compose.override.yml config > docker-stack.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
docker node update z1ehkrw1mvqlxc2udwt4xpype --label-add cic-net.app-db-data=true
|
||||||
|
docker stack deploy -c docker-stack.yml cic-net
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## Backend local development, additional details
|
||||||
|
|
||||||
|
**fill me in**
|
||||||
|
|
||||||
|
### Docker Compose Override
|
||||||
|
|
||||||
|
During development, you can change Docker Compose settings that will only affect the local development environment, in the file `docker-compose.override.yml`.
|
||||||
|
|
||||||
|
The changes to that file only affect the local development environment, not the production environment. So, you can add "temporary" changes that help the development workflow.
|
||||||
|
|
||||||
|
For example, the directory with the backend code is mounted as a Docker "host volume", mapping the code you change live to the directory inside the container. That allows you to test your changes right away, without having to build the Docker image again. It should only be done during development, for production, you should build the Docker image with a recent version of the backend code. But during development, it allows you to iterate very fast.
|
||||||
|
|
||||||
|
There is also a command override that runs `/start-reload.sh` (included in the base image) instead of the default `/start.sh` (also included in the base image). It starts a single server process (instead of multiple, as would be for production) and reloads the process whenever the code changes. Have in mind that if you have a syntax error and save the Python file, it will break and exit, and the container will stop. After that, you can restart the container by fixing the error and running again:
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
There is also a commented out `command` override, you can uncomment it and comment the default one. It makes the backend container run a process that does "nothing", but keeps the container alive. That allows you to get inside your running container and execute commands inside, for example a Python interpreter to test installed dependencies, or start the development server that reloads when it detects changes, or start a Jupyter Notebook session.
|
||||||
|
|
||||||
|
To get inside the container with a `bash` session you can start the stack with:
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
and then `exec` inside the running container:
|
||||||
|
|
||||||
|
```console
|
||||||
|
$ docker-compose exec backend bash
|
||||||
|
```
|
||||||
|
|
||||||
|
You should see an output like:
|
||||||
|
|
||||||
|
```console
|
||||||
|
root@7f2607af31c3:/app#
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Test running stack
|
||||||
|
|
||||||
|
If your stack is already up and you just want to run the tests, you can use:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose exec data-seeding /script/run_ussd_user_imports.sh
|
||||||
|
```
|
||||||
|
|
||||||
[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
|
|
||||||
|
|
||||||
|
@ -2,3 +2,4 @@
|
|||||||
.cache
|
.cache
|
||||||
.dot
|
.dot
|
||||||
**/doc
|
**/doc
|
||||||
|
**/node_modules/
|
||||||
|
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
@ -1,3 +0,0 @@
|
|||||||
**/*.pyc
|
|
||||||
.pydevproject
|
|
||||||
/vendor/
|
|
@ -1,7 +0,0 @@
|
|||||||
language: python
|
|
||||||
python:
|
|
||||||
- "2.7"
|
|
||||||
|
|
||||||
script:
|
|
||||||
- python test/wait-for-it.py
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
|||||||
The MIT License (MIT)
|
|
||||||
Copyright (c) 2016 Giles Hall
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
|
||||||
the Software without restriction, including without limitation the rights to
|
|
||||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
||||||
of the Software, and to permit persons to whom the Software is furnished to do
|
|
||||||
so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
@ -1,75 +0,0 @@
|
|||||||
# wait-for-it
|
|
||||||
|
|
||||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
|
||||||
host and TCP port. It is useful for synchronizing the spin-up of
|
|
||||||
interdependent services, such as linked docker containers. Since it is a pure
|
|
||||||
bash script, it does not have any external dependencies.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```text
|
|
||||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
|
||||||
-h HOST | --host=HOST Host or IP under test
|
|
||||||
-p PORT | --port=PORT TCP port under test
|
|
||||||
Alternatively, you specify the host and port as host:port
|
|
||||||
-s | --strict Only execute subcommand if the test succeeds
|
|
||||||
-q | --quiet Don't output any status messages
|
|
||||||
-t TIMEOUT | --timeout=TIMEOUT
|
|
||||||
Timeout in seconds, zero for no timeout
|
|
||||||
-- COMMAND ARGS Execute command with args after the test finishes
|
|
||||||
```
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
|
||||||
and if it is available, echo the message `google is up`.
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
google is up
|
|
||||||
```
|
|
||||||
|
|
||||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
|
||||||
the timeout value to 0 will disable the timeout:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
google is up
|
|
||||||
```
|
|
||||||
|
|
||||||
The subcommand will be executed regardless if the service is up or not. If you
|
|
||||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
|
||||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
|
||||||
fail:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
|
||||||
```
|
|
||||||
|
|
||||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
|
||||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
|
||||||
and determine how to proceed:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:80
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
$ echo $?
|
|
||||||
0
|
|
||||||
$ ./wait-for-it.sh www.google.com:81
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
|
||||||
$ echo $?
|
|
||||||
124
|
|
||||||
```
|
|
||||||
|
|
||||||
## Community
|
|
||||||
|
|
||||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
|
@ -1,182 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Use this script to test if a given TCP host/port are available
|
|
||||||
|
|
||||||
WAITFORIT_cmdname=${0##*/}
|
|
||||||
|
|
||||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
|
||||||
|
|
||||||
usage()
|
|
||||||
{
|
|
||||||
cat << USAGE >&2
|
|
||||||
Usage:
|
|
||||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
|
||||||
-h HOST | --host=HOST Host or IP under test
|
|
||||||
-p PORT | --port=PORT TCP port under test
|
|
||||||
Alternatively, you specify the host and port as host:port
|
|
||||||
-s | --strict Only execute subcommand if the test succeeds
|
|
||||||
-q | --quiet Don't output any status messages
|
|
||||||
-t TIMEOUT | --timeout=TIMEOUT
|
|
||||||
Timeout in seconds, zero for no timeout
|
|
||||||
-- COMMAND ARGS Execute command with args after the test finishes
|
|
||||||
USAGE
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for()
|
|
||||||
{
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
else
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
|
||||||
fi
|
|
||||||
WAITFORIT_start_ts=$(date +%s)
|
|
||||||
while :
|
|
||||||
do
|
|
||||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
|
||||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
else
|
|
||||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
fi
|
|
||||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
|
||||||
WAITFORIT_end_ts=$(date +%s)
|
|
||||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
return $WAITFORIT_result
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_wrapper()
|
|
||||||
{
|
|
||||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
|
||||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
else
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
fi
|
|
||||||
WAITFORIT_PID=$!
|
|
||||||
trap "kill -INT -$WAITFORIT_PID" INT
|
|
||||||
wait $WAITFORIT_PID
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
fi
|
|
||||||
return $WAITFORIT_RESULT
|
|
||||||
}
|
|
||||||
|
|
||||||
# process arguments
|
|
||||||
while [[ $# -gt 0 ]]
|
|
||||||
do
|
|
||||||
case "$1" in
|
|
||||||
*:* )
|
|
||||||
WAITFORIT_hostport=(${1//:/ })
|
|
||||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
|
||||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--child)
|
|
||||||
WAITFORIT_CHILD=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-q | --quiet)
|
|
||||||
WAITFORIT_QUIET=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-s | --strict)
|
|
||||||
WAITFORIT_STRICT=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-h)
|
|
||||||
WAITFORIT_HOST="$2"
|
|
||||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--host=*)
|
|
||||||
WAITFORIT_HOST="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-p)
|
|
||||||
WAITFORIT_PORT="$2"
|
|
||||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--port=*)
|
|
||||||
WAITFORIT_PORT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-t)
|
|
||||||
WAITFORIT_TIMEOUT="$2"
|
|
||||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--timeout=*)
|
|
||||||
WAITFORIT_TIMEOUT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--)
|
|
||||||
shift
|
|
||||||
WAITFORIT_CLI=("$@")
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
--help)
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echoerr "Unknown argument: $1"
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
|
||||||
echoerr "Error: you need to provide a host and port to test."
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
|
|
||||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
|
||||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
|
||||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
|
||||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
|
||||||
|
|
||||||
# Check to see if timeout is from busybox?
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
|
||||||
|
|
||||||
WAITFORIT_BUSYTIMEFLAG=""
|
|
||||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
|
||||||
WAITFORIT_ISBUSY=1
|
|
||||||
# Check if busybox timeout uses -t flag
|
|
||||||
# (recent Alpine versions don't support -t anymore)
|
|
||||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
|
||||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
WAITFORIT_ISBUSY=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
else
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
wait_for_wrapper
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
else
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
||||||
exec "${WAITFORIT_CLI[@]}"
|
|
||||||
else
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
@ -1,17 +1,41 @@
|
|||||||
build-test-cic-cache:
|
.cic_cache_variables:
|
||||||
stage: test
|
variables:
|
||||||
tags:
|
APP_NAME: cic-cache
|
||||||
- integration
|
|
||||||
variables:
|
#build-mr-cic-cache:
|
||||||
APP_NAME: cic-cache
|
# extends:
|
||||||
MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
|
# - .py_build_merge_request
|
||||||
script:
|
# - .cic_cache_variables
|
||||||
- cd apps/cic-cache
|
# rules:
|
||||||
- docker build -t $MR_IMAGE_TAG -f docker/Dockerfile .
|
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
- docker run $MR_IMAGE_TAG sh docker/run_tests.sh
|
# changes:
|
||||||
allow_failure: true
|
# - apps/cic-cache/**/*
|
||||||
rules:
|
# when: always
|
||||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
||||||
changes:
|
test-mr-cic-cache:
|
||||||
- apps/$APP_NAME/**/*
|
stage: test
|
||||||
when: always
|
tags:
|
||||||
|
- integration
|
||||||
|
extends:
|
||||||
|
- .cic_cache_variables
|
||||||
|
cache:
|
||||||
|
key:
|
||||||
|
files:
|
||||||
|
- test_requirements.txt
|
||||||
|
paths:
|
||||||
|
- /root/.cache/pip
|
||||||
|
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/$APP_NAME:$MR_IMAGE_TAG
|
||||||
|
script:
|
||||||
|
- cd apps/$APP_NAME/
|
||||||
|
- >
|
||||||
|
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
|
||||||
|
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
|
||||||
|
-r test_requirements.txt
|
||||||
|
- export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
|
||||||
|
allow_failure: true
|
||||||
|
needs: ["build-merge-request"]
|
||||||
|
rules:
|
||||||
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
|
changes:
|
||||||
|
- apps/$APP_NAME/**/*
|
||||||
|
when: always
|
||||||
|
@ -1 +1 @@
|
|||||||
include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*
|
include *requirements.txt cic_cache/data/config/*
|
||||||
|
@ -1 +0,0 @@
|
|||||||
# CIC-CACHE
|
|
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
@ -1,3 +0,0 @@
|
|||||||
**/*.pyc
|
|
||||||
.pydevproject
|
|
||||||
/vendor/
|
|
@ -1,7 +0,0 @@
|
|||||||
language: python
|
|
||||||
python:
|
|
||||||
- "2.7"
|
|
||||||
|
|
||||||
script:
|
|
||||||
- python test/wait-for-it.py
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
|||||||
The MIT License (MIT)
|
|
||||||
Copyright (c) 2016 Giles Hall
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
|
||||||
the Software without restriction, including without limitation the rights to
|
|
||||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
|
||||||
of the Software, and to permit persons to whom the Software is furnished to do
|
|
||||||
so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
@ -1,75 +0,0 @@
|
|||||||
# wait-for-it
|
|
||||||
|
|
||||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
|
||||||
host and TCP port. It is useful for synchronizing the spin-up of
|
|
||||||
interdependent services, such as linked docker containers. Since it is a pure
|
|
||||||
bash script, it does not have any external dependencies.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
```text
|
|
||||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
|
||||||
-h HOST | --host=HOST Host or IP under test
|
|
||||||
-p PORT | --port=PORT TCP port under test
|
|
||||||
Alternatively, you specify the host and port as host:port
|
|
||||||
-s | --strict Only execute subcommand if the test succeeds
|
|
||||||
-q | --quiet Don't output any status messages
|
|
||||||
-t TIMEOUT | --timeout=TIMEOUT
|
|
||||||
Timeout in seconds, zero for no timeout
|
|
||||||
-- COMMAND ARGS Execute command with args after the test finishes
|
|
||||||
```
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
|
||||||
and if it is available, echo the message `google is up`.
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
google is up
|
|
||||||
```
|
|
||||||
|
|
||||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
|
||||||
the timeout value to 0 will disable the timeout:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
google is up
|
|
||||||
```
|
|
||||||
|
|
||||||
The subcommand will be executed regardless if the service is up or not. If you
|
|
||||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
|
||||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
|
||||||
fail:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
|
||||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
|
||||||
```
|
|
||||||
|
|
||||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
|
||||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
|
||||||
and determine how to proceed:
|
|
||||||
|
|
||||||
```text
|
|
||||||
$ ./wait-for-it.sh www.google.com:80
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
|
||||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
|
||||||
$ echo $?
|
|
||||||
0
|
|
||||||
$ ./wait-for-it.sh www.google.com:81
|
|
||||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
|
||||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
|
||||||
$ echo $?
|
|
||||||
124
|
|
||||||
```
|
|
||||||
|
|
||||||
## Community
|
|
||||||
|
|
||||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
|
@ -1,182 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
# Use this script to test if a given TCP host/port are available
|
|
||||||
|
|
||||||
WAITFORIT_cmdname=${0##*/}
|
|
||||||
|
|
||||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
|
||||||
|
|
||||||
usage()
|
|
||||||
{
|
|
||||||
cat << USAGE >&2
|
|
||||||
Usage:
|
|
||||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
|
||||||
-h HOST | --host=HOST Host or IP under test
|
|
||||||
-p PORT | --port=PORT TCP port under test
|
|
||||||
Alternatively, you specify the host and port as host:port
|
|
||||||
-s | --strict Only execute subcommand if the test succeeds
|
|
||||||
-q | --quiet Don't output any status messages
|
|
||||||
-t TIMEOUT | --timeout=TIMEOUT
|
|
||||||
Timeout in seconds, zero for no timeout
|
|
||||||
-- COMMAND ARGS Execute command with args after the test finishes
|
|
||||||
USAGE
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for()
|
|
||||||
{
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
else
|
|
||||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
|
||||||
fi
|
|
||||||
WAITFORIT_start_ts=$(date +%s)
|
|
||||||
while :
|
|
||||||
do
|
|
||||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
|
||||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
else
|
|
||||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
|
||||||
WAITFORIT_result=$?
|
|
||||||
fi
|
|
||||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
|
||||||
WAITFORIT_end_ts=$(date +%s)
|
|
||||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
|
||||||
break
|
|
||||||
fi
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
return $WAITFORIT_result
|
|
||||||
}
|
|
||||||
|
|
||||||
wait_for_wrapper()
|
|
||||||
{
|
|
||||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
|
||||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
else
|
|
||||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
|
||||||
fi
|
|
||||||
WAITFORIT_PID=$!
|
|
||||||
trap "kill -INT -$WAITFORIT_PID" INT
|
|
||||||
wait $WAITFORIT_PID
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
|
||||||
fi
|
|
||||||
return $WAITFORIT_RESULT
|
|
||||||
}
|
|
||||||
|
|
||||||
# process arguments
|
|
||||||
while [[ $# -gt 0 ]]
|
|
||||||
do
|
|
||||||
case "$1" in
|
|
||||||
*:* )
|
|
||||||
WAITFORIT_hostport=(${1//:/ })
|
|
||||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
|
||||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--child)
|
|
||||||
WAITFORIT_CHILD=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-q | --quiet)
|
|
||||||
WAITFORIT_QUIET=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-s | --strict)
|
|
||||||
WAITFORIT_STRICT=1
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-h)
|
|
||||||
WAITFORIT_HOST="$2"
|
|
||||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--host=*)
|
|
||||||
WAITFORIT_HOST="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-p)
|
|
||||||
WAITFORIT_PORT="$2"
|
|
||||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--port=*)
|
|
||||||
WAITFORIT_PORT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
-t)
|
|
||||||
WAITFORIT_TIMEOUT="$2"
|
|
||||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
--timeout=*)
|
|
||||||
WAITFORIT_TIMEOUT="${1#*=}"
|
|
||||||
shift 1
|
|
||||||
;;
|
|
||||||
--)
|
|
||||||
shift
|
|
||||||
WAITFORIT_CLI=("$@")
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
--help)
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echoerr "Unknown argument: $1"
|
|
||||||
usage
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
|
||||||
echoerr "Error: you need to provide a host and port to test."
|
|
||||||
usage
|
|
||||||
fi
|
|
||||||
|
|
||||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
|
||||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
|
||||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
|
||||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
|
||||||
|
|
||||||
# Check to see if timeout is from busybox?
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
|
||||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
|
||||||
|
|
||||||
WAITFORIT_BUSYTIMEFLAG=""
|
|
||||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
|
||||||
WAITFORIT_ISBUSY=1
|
|
||||||
# Check if busybox timeout uses -t flag
|
|
||||||
# (recent Alpine versions don't support -t anymore)
|
|
||||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
|
||||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
WAITFORIT_ISBUSY=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
else
|
|
||||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
|
||||||
wait_for_wrapper
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
else
|
|
||||||
wait_for
|
|
||||||
WAITFORIT_RESULT=$?
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
|
||||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
|
||||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
||||||
exec "${WAITFORIT_CLI[@]}"
|
|
||||||
else
|
|
||||||
exit $WAITFORIT_RESULT
|
|
||||||
fi
|
|
@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser):
|
|||||||
if local_arg_flags & CICFlag.CELERY:
|
if local_arg_flags & CICFlag.CELERY:
|
||||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
|
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
|
||||||
if local_arg_flags & CICFlag.SYNCER:
|
if local_arg_flags & CICFlag.SYNCER:
|
||||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
|
||||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||||
if local_arg_flags & CICFlag.CHAIN:
|
if local_arg_flags & CICFlag.CHAIN:
|
||||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||||
|
@ -27,11 +27,11 @@ class RPC:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def from_config(config):
|
def from_config(config):
|
||||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||||
RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, 'default')
|
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, 'default')
|
||||||
if config.get('SIGNER_PROVIDER'):
|
if config.get('SIGNER_PROVIDER'):
|
||||||
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
|
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, tag='signer')
|
||||||
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
|
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, 'signer')
|
||||||
rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
||||||
logg.info('set up rpc: {}'.format(rpc))
|
logg.info('set up rpc: {}'.format(rpc))
|
||||||
return rpc
|
return rpc
|
||||||
|
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
[cic]
|
[cic]
|
||||||
registry_address =
|
registry_address =
|
||||||
trust_address =
|
trust_address =
|
||||||
health_modules =
|
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||||
|
@ -3,8 +3,7 @@ engine =
|
|||||||
driver =
|
driver =
|
||||||
host =
|
host =
|
||||||
port =
|
port =
|
||||||
#name = cic-cache
|
name = cic-cache
|
||||||
prefix =
|
|
||||||
user =
|
user =
|
||||||
password =
|
password =
|
||||||
debug = 0
|
debug = 0
|
||||||
|
@ -9,26 +9,21 @@ from .list import (
|
|||||||
tag_transaction,
|
tag_transaction,
|
||||||
add_tag,
|
add_tag,
|
||||||
)
|
)
|
||||||
from cic_cache.db.models.base import SessionBase
|
|
||||||
|
|
||||||
|
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
def dsn_from_config(config, name):
|
def dsn_from_config(config):
|
||||||
scheme = config.get('DATABASE_ENGINE')
|
scheme = config.get('DATABASE_ENGINE')
|
||||||
if config.get('DATABASE_DRIVER') != None:
|
if config.get('DATABASE_DRIVER') != None:
|
||||||
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
|
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
|
||||||
|
|
||||||
database_name = name
|
|
||||||
if config.get('DATABASE_PREFIX'):
|
|
||||||
database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
|
|
||||||
dsn = ''
|
dsn = ''
|
||||||
if config.get('DATABASE_ENGINE') == 'sqlite':
|
if config.get('DATABASE_ENGINE') == 'sqlite':
|
||||||
SessionBase.poolable = False
|
|
||||||
dsn = '{}:///{}'.format(
|
dsn = '{}:///{}'.format(
|
||||||
scheme,
|
scheme,
|
||||||
database_name,
|
config.get('DATABASE_NAME'),
|
||||||
)
|
)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
@ -38,7 +33,7 @@ def dsn_from_config(config, name):
|
|||||||
config.get('DATABASE_PASSWORD'),
|
config.get('DATABASE_PASSWORD'),
|
||||||
config.get('DATABASE_HOST'),
|
config.get('DATABASE_HOST'),
|
||||||
config.get('DATABASE_PORT'),
|
config.get('DATABASE_PORT'),
|
||||||
database_name,
|
config.get('DATABASE_NAME'),
|
||||||
)
|
)
|
||||||
logg.debug('parsed dsn from config: {}'.format(dsn))
|
logg.debug('parsed dsn from config: {}'.format(dsn))
|
||||||
return dsn
|
return dsn
|
||||||
|
@ -5,11 +5,7 @@ import re
|
|||||||
import base64
|
import base64
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
from hexathon import (
|
from hexathon import add_0x
|
||||||
add_0x,
|
|
||||||
strip_0x,
|
|
||||||
)
|
|
||||||
from chainlib.encode import TxHexNormalizer
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_cache.cache import (
|
from cic_cache.cache import (
|
||||||
@ -20,72 +16,27 @@ from cic_cache.cache import (
|
|||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
#logg = logging.getLogger()
|
#logg = logging.getLogger()
|
||||||
|
|
||||||
re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
|
||||||
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
||||||
re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
|
||||||
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
|
||||||
re_default_limit = r'/defaultlimit/?'
|
|
||||||
|
|
||||||
DEFAULT_LIMIT = 100
|
DEFAULT_LIMIT = 100
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
def parse_query_account(r):
|
|
||||||
address = strip_0x(r[1])
|
|
||||||
#address = tx_normalize.wallet_address(address)
|
|
||||||
limit = DEFAULT_LIMIT
|
|
||||||
g = r.groups()
|
|
||||||
if len(g) > 3:
|
|
||||||
limit = int(r[4])
|
|
||||||
if limit == 0:
|
|
||||||
limit = DEFAULT_LIMIT
|
|
||||||
offset = 0
|
|
||||||
if len(g) > 4:
|
|
||||||
offset = int(r[6])
|
|
||||||
|
|
||||||
logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
|
|
||||||
|
|
||||||
return (address, offset, limit,)
|
|
||||||
|
|
||||||
|
|
||||||
# r is an re.Match
|
|
||||||
def parse_query_any(r):
|
|
||||||
limit = DEFAULT_LIMIT
|
|
||||||
offset = 0
|
|
||||||
block_offset = None
|
|
||||||
block_end = None
|
|
||||||
if r.lastindex != None:
|
|
||||||
if r.lastindex > 0:
|
|
||||||
limit = int(r[1])
|
|
||||||
if r.lastindex > 1:
|
|
||||||
offset = int(r[2])
|
|
||||||
if r.lastindex > 2:
|
|
||||||
block_offset = int(r[3])
|
|
||||||
if r.lastindex > 3:
|
|
||||||
block_end = int(r[4])
|
|
||||||
if block_end < block_offset:
|
|
||||||
raise ValueError('cart before the horse, dude')
|
|
||||||
|
|
||||||
logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
|
|
||||||
|
|
||||||
return (offset, limit, block_offset, block_end,)
|
|
||||||
|
|
||||||
|
|
||||||
def process_default_limit(session, env):
|
|
||||||
r = re.match(re_default_limit, env.get('PATH_INFO'))
|
|
||||||
if not r:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
|
|
||||||
|
|
||||||
|
|
||||||
def process_transactions_account_bloom(session, env):
|
def process_transactions_account_bloom(session, env):
|
||||||
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
|
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
|
||||||
if not r:
|
if not r:
|
||||||
return None
|
return None
|
||||||
logg.debug('match account bloom')
|
|
||||||
|
|
||||||
(address, offset, limit,) = parse_query_account(r)
|
address = r[1]
|
||||||
|
if r[2] == None:
|
||||||
|
address = add_0x(address)
|
||||||
|
offset = 0
|
||||||
|
if r.lastindex > 2:
|
||||||
|
offset = r[4]
|
||||||
|
limit = DEFAULT_LIMIT
|
||||||
|
if r.lastindex > 4:
|
||||||
|
limit = r[6]
|
||||||
|
|
||||||
c = BloomCache(session)
|
c = BloomCache(session)
|
||||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
||||||
@ -108,9 +59,13 @@ def process_transactions_all_bloom(session, env):
|
|||||||
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
|
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
|
||||||
if not r:
|
if not r:
|
||||||
return None
|
return None
|
||||||
logg.debug('match all bloom')
|
|
||||||
|
|
||||||
(limit, offset, block_offset, block_end,) = parse_query_any(r)
|
offset = DEFAULT_LIMIT
|
||||||
|
if r.lastindex > 0:
|
||||||
|
offset = r[1]
|
||||||
|
limit = 0
|
||||||
|
if r.lastindex > 1:
|
||||||
|
limit = r[2]
|
||||||
|
|
||||||
c = BloomCache(session)
|
c = BloomCache(session)
|
||||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
||||||
@ -133,16 +88,17 @@ def process_transactions_all_data(session, env):
|
|||||||
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
|
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
|
||||||
if not r:
|
if not r:
|
||||||
return None
|
return None
|
||||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||||
# return None
|
return None
|
||||||
logg.debug('match all data')
|
|
||||||
|
|
||||||
logg.debug('got data request {}'.format(env))
|
logg.debug('got data request {}'.format(env))
|
||||||
|
block_offset = r[1]
|
||||||
(offset, limit, block_offset, block_end) = parse_query_any(r)
|
block_end = r[2]
|
||||||
|
if int(r[2]) < int(r[1]):
|
||||||
|
raise ValueError('cart before the horse, dude')
|
||||||
|
|
||||||
c = DataCache(session)
|
c = DataCache(session)
|
||||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
|
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
|
||||||
|
|
||||||
for r in tx_cache:
|
for r in tx_cache:
|
||||||
r['date_block'] = r['date_block'].timestamp()
|
r['date_block'] = r['date_block'].timestamp()
|
||||||
@ -157,30 +113,3 @@ def process_transactions_all_data(session, env):
|
|||||||
j = json.dumps(o)
|
j = json.dumps(o)
|
||||||
|
|
||||||
return ('application/json', j.encode('utf-8'),)
|
return ('application/json', j.encode('utf-8'),)
|
||||||
|
|
||||||
|
|
||||||
def process_transactions_account_data(session, env):
|
|
||||||
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
|
|
||||||
if not r:
|
|
||||||
return None
|
|
||||||
logg.debug('match account data')
|
|
||||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
|
||||||
# return None
|
|
||||||
|
|
||||||
(address, offset, limit,) = parse_query_account(r)
|
|
||||||
|
|
||||||
c = DataCache(session)
|
|
||||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
|
|
||||||
|
|
||||||
for r in tx_cache:
|
|
||||||
r['date_block'] = r['date_block'].timestamp()
|
|
||||||
|
|
||||||
o = {
|
|
||||||
'low': lowest_block,
|
|
||||||
'high': highest_block,
|
|
||||||
'data': tx_cache,
|
|
||||||
}
|
|
||||||
|
|
||||||
j = json.dumps(o)
|
|
||||||
|
|
||||||
return ('application/json', j.encode('utf-8'),)
|
|
||||||
|
@ -8,33 +8,42 @@ import base64
|
|||||||
import confini
|
import confini
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
import cic_cache.cli
|
|
||||||
from cic_cache.db import dsn_from_config
|
from cic_cache.db import dsn_from_config
|
||||||
from cic_cache.db.models.base import SessionBase
|
from cic_cache.db.models.base import SessionBase
|
||||||
from cic_cache.runnable.daemons.query import (
|
from cic_cache.runnable.daemons.query import (
|
||||||
process_default_limit,
|
|
||||||
process_transactions_account_bloom,
|
process_transactions_account_bloom,
|
||||||
process_transactions_account_data,
|
|
||||||
process_transactions_all_bloom,
|
process_transactions_all_bloom,
|
||||||
process_transactions_all_data,
|
process_transactions_all_data,
|
||||||
)
|
)
|
||||||
import cic_cache.cli
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.WARNING)
|
logging.basicConfig(level=logging.WARNING)
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||||
|
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||||
|
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||||
|
|
||||||
arg_flags = cic_cache.cli.argflag_std_read
|
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||||
local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
|
|
||||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
argparser = argparse.ArgumentParser()
|
||||||
argparser.process_local_flags(local_arg_flags)
|
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||||
|
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||||
|
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||||
|
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||||
args = argparser.parse_args()
|
args = argparser.parse_args()
|
||||||
|
|
||||||
# process config
|
if args.vv:
|
||||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
|
elif args.v:
|
||||||
|
logging.getLogger().setLevel(logging.INFO)
|
||||||
|
|
||||||
# connect to database
|
config = confini.Config(args.c, args.env_prefix)
|
||||||
dsn = dsn_from_config(config, 'cic_cache')
|
config.process()
|
||||||
|
config.censor('PASSWORD', 'DATABASE')
|
||||||
|
config.censor('PASSWORD', 'SSL')
|
||||||
|
logg.debug('config:\n{}'.format(config))
|
||||||
|
|
||||||
|
dsn = dsn_from_config(config)
|
||||||
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
|
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
|
||||||
|
|
||||||
|
|
||||||
@ -46,11 +55,9 @@ def application(env, start_response):
|
|||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
for handler in [
|
for handler in [
|
||||||
process_transactions_account_data,
|
|
||||||
process_transactions_account_bloom,
|
|
||||||
process_transactions_all_data,
|
process_transactions_all_data,
|
||||||
process_transactions_all_bloom,
|
process_transactions_all_bloom,
|
||||||
process_default_limit,
|
process_transactions_account_bloom,
|
||||||
]:
|
]:
|
||||||
r = None
|
r = None
|
||||||
try:
|
try:
|
||||||
|
@ -3,14 +3,12 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
import tempfile
|
|
||||||
|
|
||||||
# third-party imports
|
# third-party imports
|
||||||
import celery
|
import celery
|
||||||
import confini
|
import confini
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
import cic_cache.cli
|
|
||||||
from cic_cache.db import dsn_from_config
|
from cic_cache.db import dsn_from_config
|
||||||
from cic_cache.db.models.base import SessionBase
|
from cic_cache.db.models.base import SessionBase
|
||||||
from cic_cache.tasks.tx import *
|
from cic_cache.tasks.tx import *
|
||||||
@ -18,20 +16,35 @@ from cic_cache.tasks.tx import *
|
|||||||
logging.basicConfig(level=logging.WARNING)
|
logging.basicConfig(level=logging.WARNING)
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
# process args
|
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||||
arg_flags = cic_cache.cli.argflag_std_base
|
|
||||||
local_arg_flags = cic_cache.cli.argflag_local_task
|
|
||||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
argparser = argparse.ArgumentParser()
|
||||||
argparser.process_local_flags(local_arg_flags)
|
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||||
|
argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
|
||||||
|
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||||
|
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||||
|
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||||
|
|
||||||
args = argparser.parse_args()
|
args = argparser.parse_args()
|
||||||
|
|
||||||
# process config
|
if args.vv:
|
||||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
|
elif args.v:
|
||||||
|
logging.getLogger().setLevel(logging.INFO)
|
||||||
|
|
||||||
|
config = confini.Config(args.c, args.env_prefix)
|
||||||
|
config.process()
|
||||||
|
|
||||||
# connect to database
|
# connect to database
|
||||||
dsn = dsn_from_config(config, 'cic_cache')
|
dsn = dsn_from_config(config)
|
||||||
SessionBase.connect(dsn)
|
SessionBase.connect(dsn)
|
||||||
|
|
||||||
|
# verify database connection with minimal sanity query
|
||||||
|
#session = SessionBase.create_session()
|
||||||
|
#session.execute('select version_num from alembic_version')
|
||||||
|
#session.close()
|
||||||
|
|
||||||
# set up celery
|
# set up celery
|
||||||
current_app = celery.Celery(__name__)
|
current_app = celery.Celery(__name__)
|
||||||
|
|
||||||
@ -74,9 +87,9 @@ def main():
|
|||||||
elif args.v:
|
elif args.v:
|
||||||
argv.append('--loglevel=INFO')
|
argv.append('--loglevel=INFO')
|
||||||
argv.append('-Q')
|
argv.append('-Q')
|
||||||
argv.append(config.get('CELERY_QUEUE'))
|
argv.append(args.q)
|
||||||
argv.append('-n')
|
argv.append('-n')
|
||||||
argv.append(config.get('CELERY_QUEUE'))
|
argv.append(args.q)
|
||||||
|
|
||||||
current_app.worker_main(argv)
|
current_app.worker_main(argv)
|
||||||
|
|
||||||
|
@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
|
|||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
# process args
|
# process args
|
||||||
arg_flags = cic_cache.cli.argflag_std_base
|
arg_flags = cic_cache.cli.argflag_std_read
|
||||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
local_arg_flags = cic_cache.cli.argflag_local_sync
|
||||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||||
argparser.process_local_flags(local_arg_flags)
|
argparser.process_local_flags(local_arg_flags)
|
||||||
@ -50,7 +50,7 @@ args = argparser.parse_args()
|
|||||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||||
|
|
||||||
# connect to database
|
# connect to database
|
||||||
dsn = dsn_from_config(config, 'cic_cache')
|
dsn = dsn_from_config(config)
|
||||||
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
|
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
|
||||||
|
|
||||||
# set up rpc
|
# set up rpc
|
||||||
@ -95,10 +95,10 @@ def main():
|
|||||||
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
||||||
|
|
||||||
if len(syncer_backends) == 0:
|
if len(syncer_backends) == 0:
|
||||||
initial_block_start = int(config.get('SYNCER_OFFSET'))
|
initial_block_start = config.get('SYNCER_OFFSET')
|
||||||
initial_block_offset = int(block_offset)
|
initial_block_offset = block_offset
|
||||||
if config.get('SYNCER_NO_HISTORY'):
|
if config.get('SYNCER_NO_HISTORY'):
|
||||||
initial_block_start = initial_block_offset
|
initial_block_start = block_offset
|
||||||
initial_block_offset += 1
|
initial_block_offset += 1
|
||||||
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
||||||
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
||||||
|
@ -5,7 +5,7 @@ version = (
|
|||||||
0,
|
0,
|
||||||
2,
|
2,
|
||||||
1,
|
1,
|
||||||
'alpha.3',
|
'alpha.1',
|
||||||
)
|
)
|
||||||
|
|
||||||
version_object = semver.VersionInfo(
|
version_object = semver.VersionInfo(
|
||||||
|
3
apps/cic-cache/config/celery.ini
Normal file
3
apps/cic-cache/config/celery.ini
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[celery]
|
||||||
|
broker_url = redis:///
|
||||||
|
result_url = redis:///
|
3
apps/cic-cache/config/cic.ini
Normal file
3
apps/cic-cache/config/cic.ini
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[cic]
|
||||||
|
registry_address =
|
||||||
|
trust_address =
|
9
apps/cic-cache/config/database.ini
Normal file
9
apps/cic-cache/config/database.ini
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[database]
|
||||||
|
NAME=cic_cache
|
||||||
|
USER=postgres
|
||||||
|
PASSWORD=
|
||||||
|
HOST=localhost
|
||||||
|
PORT=5432
|
||||||
|
ENGINE=postgresql
|
||||||
|
DRIVER=psycopg2
|
||||||
|
DEBUG=0
|
3
apps/cic-cache/config/docker/cic.ini
Normal file
3
apps/cic-cache/config/docker/cic.ini
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
[cic]
|
||||||
|
registry_address =
|
||||||
|
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
|
9
apps/cic-cache/config/docker/database.ini
Normal file
9
apps/cic-cache/config/docker/database.ini
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
[database]
|
||||||
|
NAME=cic_cache
|
||||||
|
USER=grassroots
|
||||||
|
PASSWORD=
|
||||||
|
HOST=localhost
|
||||||
|
PORT=63432
|
||||||
|
ENGINE=postgresql
|
||||||
|
DRIVER=psycopg2
|
||||||
|
DEBUG=0
|
4
apps/cic-cache/config/docker/syncer.ini
Normal file
4
apps/cic-cache/config/docker/syncer.ini
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
[syncer]
|
||||||
|
loop_interval = 1
|
||||||
|
offset = 0
|
||||||
|
no_history = 0
|
2
apps/cic-cache/config/test/bancor.ini
Normal file
2
apps/cic-cache/config/test/bancor.ini
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
[bancor]
|
||||||
|
dir =
|
@ -1,3 +1,4 @@
|
|||||||
[cic]
|
[cic]
|
||||||
registry_address =
|
registry_address =
|
||||||
|
chain_spec =
|
||||||
trust_address =
|
trust_address =
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
[database]
|
[database]
|
||||||
PREFIX=cic-cache-test
|
NAME=cic-cache-test
|
||||||
USER=postgres
|
USER=postgres
|
||||||
PASSWORD=
|
PASSWORD=
|
||||||
HOST=localhost
|
HOST=localhost
|
||||||
|
5
apps/cic-cache/config/test/eth.ini
Normal file
5
apps/cic-cache/config/test/eth.ini
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
[eth]
|
||||||
|
#ws_provider = ws://localhost:8546
|
||||||
|
#ttp_provider = http://localhost:8545
|
||||||
|
provider = http://localhost:8545
|
||||||
|
#chain_id =
|
@ -1,4 +1,4 @@
|
|||||||
openapi: "3.0.2"
|
openapi: "3.0.3"
|
||||||
info:
|
info:
|
||||||
title: Grassroots Economics CIC Cache
|
title: Grassroots Economics CIC Cache
|
||||||
description: Cache of processed transaction data from Ethereum blockchain and worker queues
|
description: Cache of processed transaction data from Ethereum blockchain and worker queues
|
||||||
@ -9,34 +9,17 @@ info:
|
|||||||
email: will@grassecon.org
|
email: will@grassecon.org
|
||||||
license:
|
license:
|
||||||
name: GPLv3
|
name: GPLv3
|
||||||
version: 0.2.0
|
version: 0.1.0
|
||||||
|
|
||||||
paths:
|
paths:
|
||||||
/defaultlimit:
|
/tx/{offset}/{limit}:
|
||||||
summary: The default limit value of result sets.
|
description: Bloom filter for batch of latest transactions
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve default limit
|
|
||||||
operationId: limit.default
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Limit query successful
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/Limit"
|
|
||||||
|
|
||||||
/tx:
|
|
||||||
summary: Bloom filter for batch of latest transactions
|
|
||||||
description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
get:
|
||||||
tags:
|
tags:
|
||||||
- transactions
|
- transactions
|
||||||
description:
|
description:
|
||||||
Retrieve transactions
|
Retrieve transactions
|
||||||
operationId: tx.get.latest
|
operationId: tx.get
|
||||||
responses:
|
responses:
|
||||||
200:
|
200:
|
||||||
description: Transaction query successful.
|
description: Transaction query successful.
|
||||||
@ -46,109 +29,27 @@ paths:
|
|||||||
$ref: "#/components/schemas/BlocksBloom"
|
$ref: "#/components/schemas/BlocksBloom"
|
||||||
|
|
||||||
|
|
||||||
/tx/{limit}:
|
|
||||||
summary: Bloom filter for batch of latest transactions
|
|
||||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.latest.limit
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
parameters:
|
parameters:
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/tx/{limit}/{offset}:
|
|
||||||
summary: Bloom filter for batch of latest transactions
|
|
||||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.latest.range
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
- name: offset
|
||||||
in: path
|
in: path
|
||||||
required: true
|
|
||||||
schema:
|
schema:
|
||||||
type: integer
|
type: integer
|
||||||
format: int32
|
format: int32
|
||||||
|
|
||||||
|
|
||||||
/tx/{limit}/{offset}/{block_offset}:
|
|
||||||
summary: Bloom filter for batch of transactions since a particular block.
|
|
||||||
description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.latest.range.block.offset
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful. Results are ordered from oldest to newest.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
- name: limit
|
||||||
in: path
|
in: path
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
schema:
|
||||||
type: integer
|
type: integer
|
||||||
format: int32
|
format: int32
|
||||||
|
|
||||||
|
|
||||||
/tx/{limit}/{offset}/{block_offset}/{block_end}:
|
/tx/{address}/{offset}/{limit}:
|
||||||
summary: Bloom filter for batch of transactions within a particular block range.
|
description: Bloom filter for batch of latest transactions by account
|
||||||
description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
get:
|
||||||
tags:
|
tags:
|
||||||
- transactions
|
- transactions
|
||||||
description:
|
description:
|
||||||
Retrieve transactions
|
Retrieve transactions
|
||||||
operationId: tx.get.latest.range.block.range
|
operationId: tx.get
|
||||||
responses:
|
responses:
|
||||||
200:
|
200:
|
||||||
description: Transaction query successful.
|
description: Transaction query successful.
|
||||||
@ -157,49 +58,6 @@ paths:
|
|||||||
schema:
|
schema:
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
$ref: "#/components/schemas/BlocksBloom"
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_end
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/tx/{address}:
|
|
||||||
summary: Bloom filter for batch of latest transactions by account.
|
|
||||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.user
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
|
|
||||||
parameters:
|
parameters:
|
||||||
- name: address
|
- name: address
|
||||||
@ -207,342 +65,26 @@ paths:
|
|||||||
required: true
|
required: true
|
||||||
schema:
|
schema:
|
||||||
type: string
|
type: string
|
||||||
|
|
||||||
|
|
||||||
/tx/{address}/{limit}:
|
|
||||||
summary: Bloom filter for batch of latest transactions by account.
|
|
||||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.user.limit
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: address
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/tx/{address}/{limit}/{offset}:
|
|
||||||
summary: Bloom filter for batch of latest transactions by account
|
|
||||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: tx.get.user.range
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/BlocksBloom"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: address
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
- name: offset
|
||||||
in: path
|
in: path
|
||||||
required: true
|
|
||||||
schema:
|
schema:
|
||||||
type: integer
|
type: integer
|
||||||
format: int32
|
format: int32
|
||||||
|
|
||||||
|
|
||||||
/txa:
|
|
||||||
summary: Cached data for latest transactions.
|
|
||||||
description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.latest
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
|
|
||||||
/txa/{limit}:
|
|
||||||
summary: Cached data for latest transactions.
|
|
||||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.latest.limit
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
- name: limit
|
||||||
in: path
|
in: path
|
||||||
required: true
|
|
||||||
schema:
|
schema:
|
||||||
type: integer
|
type: integer
|
||||||
format: int32
|
format: int32
|
||||||
|
|
||||||
|
|
||||||
/txa/{limit}/{offset}:
|
|
||||||
summary: Cached data for latest transactions.
|
|
||||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.latest.range
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/txa/{limit}/{offset}/{block_offset}:
|
|
||||||
summary: Cached data for transactions since a particular block.
|
|
||||||
description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.latest.range.block.offset
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
/txa/{limit}/{offset}/{block_offset}/{block_end}:
|
|
||||||
summary: Cached data for transactions within a particular block range.
|
|
||||||
description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.latest.range.block.range
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: block_end
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/txa/{address}:
|
|
||||||
summary: Cached data for batch of latest transactions by account.
|
|
||||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.user
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: address
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
|
|
||||||
|
|
||||||
/txa/{address}/{limit}:
|
|
||||||
summary: Cached data for batch of latest transactions by account.
|
|
||||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.user.limit
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: address
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
/txa/{address}/{limit}/{offset}:
|
|
||||||
summary: Cached data for batch of latest transactions by account.
|
|
||||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
|
||||||
get:
|
|
||||||
tags:
|
|
||||||
- transactions
|
|
||||||
description:
|
|
||||||
Retrieve transactions
|
|
||||||
operationId: txa.get.user.range
|
|
||||||
responses:
|
|
||||||
200:
|
|
||||||
description: Transaction query successful.
|
|
||||||
content:
|
|
||||||
application/json:
|
|
||||||
schema:
|
|
||||||
$ref: "#/components/schemas/TransactionList"
|
|
||||||
|
|
||||||
parameters:
|
|
||||||
- name: address
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: string
|
|
||||||
- name: limit
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
- name: offset
|
|
||||||
in: path
|
|
||||||
required: true
|
|
||||||
schema:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
|
|
||||||
|
|
||||||
components:
|
components:
|
||||||
schemas:
|
schemas:
|
||||||
Limit:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
BlocksBloom:
|
BlocksBloom:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
low:
|
low:
|
||||||
type: integer
|
type: int
|
||||||
format: int32
|
format: int32
|
||||||
description: The lowest block number included in the filter
|
description: The lowest block number included in the filter
|
||||||
high:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
description: The highest block number included in the filter
|
|
||||||
block_filter:
|
block_filter:
|
||||||
type: string
|
type: string
|
||||||
format: byte
|
format: byte
|
||||||
@ -555,89 +97,6 @@ components:
|
|||||||
type: string
|
type: string
|
||||||
description: Hashing algorithm (currently only using sha256)
|
description: Hashing algorithm (currently only using sha256)
|
||||||
filter_rounds:
|
filter_rounds:
|
||||||
type: integer
|
type: int
|
||||||
format: int32
|
format: int32
|
||||||
description: Number of hash rounds used to create the filter
|
description: Number of hash rounds used to create the filter
|
||||||
TransactionList:
|
|
||||||
type: object
|
|
||||||
properties:
|
|
||||||
low:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
description: The lowest block number included in the result set
|
|
||||||
high:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
description: The highest block number included in the filter
|
|
||||||
data:
|
|
||||||
type: array
|
|
||||||
description: Cached transaction data
|
|
||||||
items:
|
|
||||||
$ref: "#/components/schemas/Transaction"
|
|
||||||
Transaction:
|
|
||||||
type: object
|
|
||||||
properties:
|
|
||||||
block_number:
|
|
||||||
type: integer
|
|
||||||
format: int64
|
|
||||||
description: Block number transaction was included in.
|
|
||||||
tx_hash:
|
|
||||||
type: string
|
|
||||||
description: Transaction hash, in hex.
|
|
||||||
date_block:
|
|
||||||
type: integer
|
|
||||||
format: int32
|
|
||||||
description: Block timestamp.
|
|
||||||
sender:
|
|
||||||
type: string
|
|
||||||
description: Spender address, in hex.
|
|
||||||
recipient:
|
|
||||||
type: string
|
|
||||||
description: Beneficiary address, in hex.
|
|
||||||
from_value:
|
|
||||||
type: integer
|
|
||||||
format: int64
|
|
||||||
description: Value deducted from spender's balance.
|
|
||||||
to_value:
|
|
||||||
type: integer
|
|
||||||
format: int64
|
|
||||||
description: Value added to beneficiary's balance.
|
|
||||||
source_token:
|
|
||||||
type: string
|
|
||||||
description: Network address of token in which `from_value` is denominated.
|
|
||||||
destination_token:
|
|
||||||
type: string
|
|
||||||
description: Network address of token in which `to_value` is denominated.
|
|
||||||
success:
|
|
||||||
type: boolean
|
|
||||||
description: Network consensus state on whether the transaction was successful or not.
|
|
||||||
tx_type:
|
|
||||||
type: string
|
|
||||||
enum:
|
|
||||||
- erc20.faucet
|
|
||||||
- faucet.give_to
|
|
||||||
|
|
||||||
examples:
|
|
||||||
data_last:
|
|
||||||
summary: Get the latest cached transactions, using the server's default limit.
|
|
||||||
value: "/txa"
|
|
||||||
|
|
||||||
data_limit:
|
|
||||||
summary: Get the last 42 cached transactions.
|
|
||||||
value: "/txa/42"
|
|
||||||
|
|
||||||
data_range:
|
|
||||||
summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
|
|
||||||
value: "/txa/42/13"
|
|
||||||
|
|
||||||
data_range_block_offset:
|
|
||||||
summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
|
|
||||||
value: "/txa/42/0/1337"
|
|
||||||
|
|
||||||
data_range_block_offset:
|
|
||||||
summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
|
|
||||||
value: "/txa/42/0/1337/1453"
|
|
||||||
|
|
||||||
data_range_block_range:
|
|
||||||
summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
|
|
||||||
value: "/txa/42/13/1337/1453"
|
|
||||||
|
@ -1,25 +1,31 @@
|
|||||||
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
|
# syntax = docker/dockerfile:1.2
|
||||||
|
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
|
||||||
|
|
||||||
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
|
# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
|
#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
|
||||||
|
#RUN pip install $pip_extra_index_url_flag .
|
||||||
|
#RUN pip install .[server]
|
||||||
|
|
||||||
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
|
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
|
||||||
ARG EXTRA_PIP_ARGS=""
|
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
|
||||||
ARG PIP_INDEX_URL=https://pypi.org/simple
|
|
||||||
|
|
||||||
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
|
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
|
||||||
pip install --index-url $PIP_INDEX_URL \
|
pip install --index-url https://pypi.org/simple \
|
||||||
--pre \
|
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL \
|
||||||
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
|
|
||||||
-r requirements.txt
|
-r requirements.txt
|
||||||
|
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN pip install . --extra-index-url $EXTRA_PIP_INDEX_URL
|
|
||||||
|
RUN python setup.py install
|
||||||
|
|
||||||
|
# ini files in config directory defines the configurable parameters for the application
|
||||||
|
# they can all be overridden by environment variables
|
||||||
|
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
|
||||||
|
COPY config/ /usr/local/etc/cic-cache/
|
||||||
|
|
||||||
# for db migrations
|
# for db migrations
|
||||||
COPY ./aux/wait-for-it/wait-for-it.sh ./
|
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
|
||||||
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
|
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
|
||||||
|
|
||||||
COPY /docker/start_tracker.sh ./start_tracker.sh
|
COPY /docker/start_tracker.sh ./start_tracker.sh
|
||||||
|
@ -2,5 +2,5 @@
|
|||||||
|
|
||||||
set -e
|
set -e
|
||||||
>&2 echo executing database migration
|
>&2 echo executing database migration
|
||||||
python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
||||||
set +e
|
set +e
|
||||||
|
@ -1,10 +0,0 @@
|
|||||||
#! /bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
pip install --extra-index-url https://pip.grassrootseconomics.net \
|
|
||||||
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
|
|
||||||
-r test_requirements.txt
|
|
||||||
|
|
||||||
export PYTHONPATH=. && pytest -x --cov=cic_cache --cov-fail-under=90 --cov-report term-missing tests
|
|
||||||
|
|
@ -1,15 +1,17 @@
|
|||||||
alembic==1.4.2
|
alembic==1.4.2
|
||||||
confini~=0.5.3
|
confini>=0.3.6rc4,<0.5.0
|
||||||
uwsgi==2.0.19.1
|
uwsgi==2.0.19.1
|
||||||
moolb~=0.2.0
|
moolb~=0.1.1b2
|
||||||
cic-eth-registry~=0.6.6
|
cic-eth-registry~=0.5.8a1
|
||||||
SQLAlchemy==1.3.20
|
SQLAlchemy==1.3.20
|
||||||
semver==2.13.0
|
semver==2.13.0
|
||||||
psycopg2==2.8.6
|
psycopg2==2.8.6
|
||||||
celery==4.4.7
|
celery==4.4.7
|
||||||
redis==3.5.3
|
redis==3.5.3
|
||||||
chainsyncer[sql]~=0.0.7
|
chainsyncer[sql]>=0.0.6a1,<0.1.0
|
||||||
erc20-faucet~=0.3.2
|
erc20-faucet>=0.2.4a2, <0.3.0
|
||||||
chainlib-eth~=0.0.15
|
#chainlib-eth==0.0.7a5,<0.1.0
|
||||||
eth-address-index~=0.2.4
|
chainlib-eth==0.0.7a5
|
||||||
okota~=0.2.5
|
#chainlib==0.0.7a4,<0.1.0
|
||||||
|
chainlib==0.0.7a4
|
||||||
|
eth-address-index>=0.1.4a1,<0.2.0
|
||||||
|
@ -1,55 +1,53 @@
|
|||||||
#!/usr/bin/python3
|
#!/usr/bin/python
|
||||||
|
|
||||||
# standard imports
|
|
||||||
import os
|
import os
|
||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
|
||||||
# external imports
|
|
||||||
import alembic
|
import alembic
|
||||||
from alembic.config import Config as AlembicConfig
|
from alembic.config import Config as AlembicConfig
|
||||||
import confini
|
import confini
|
||||||
|
|
||||||
# local imports
|
|
||||||
from cic_cache.db import dsn_from_config
|
from cic_cache.db import dsn_from_config
|
||||||
import cic_cache.cli
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.WARNING)
|
logging.basicConfig(level=logging.WARNING)
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
# BUG: the dbdir doesn't work after script install
|
# BUG: the dbdir doesn't work after script install
|
||||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
|
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||||
default_migrations_dir = os.path.join(dbdir, 'migrations')
|
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||||
configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
|
|
||||||
|
|
||||||
#config_dir = os.path.join('/usr/local/etc/cic-cache')
|
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||||
|
|
||||||
arg_flags = cic_cache.cli.argflag_std_base
|
argparser = argparse.ArgumentParser()
|
||||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||||
argparser.process_local_flags(local_arg_flags)
|
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
|
||||||
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
|
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
|
||||||
argparser.add_argument('-f', '--force', action='store_true', help='force action')
|
argparser.add_argument('-f', action='store_true', help='force action')
|
||||||
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
|
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||||
|
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||||
args = argparser.parse_args()
|
args = argparser.parse_args()
|
||||||
|
|
||||||
extra_args = {
|
if args.vv:
|
||||||
'reset': None,
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
'force': None,
|
elif args.v:
|
||||||
'migrations_dir': None,
|
logging.getLogger().setLevel(logging.INFO)
|
||||||
}
|
|
||||||
# process config
|
|
||||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
|
|
||||||
|
|
||||||
migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
|
config = confini.Config(args.c, args.env_prefix)
|
||||||
|
config.process()
|
||||||
|
config.censor('PASSWORD', 'DATABASE')
|
||||||
|
config.censor('PASSWORD', 'SSL')
|
||||||
|
logg.debug('config:\n{}'.format(config))
|
||||||
|
|
||||||
|
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
|
||||||
if not os.path.isdir(migrations_dir):
|
if not os.path.isdir(migrations_dir):
|
||||||
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
|
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
|
||||||
migrations_dir = os.path.join(args.migrations_dir, 'default')
|
migrations_dir = os.path.join(args.migrations_dir, 'default')
|
||||||
|
|
||||||
# connect to database
|
# connect to database
|
||||||
dsn = dsn_from_config(config, 'cic_cache')
|
dsn = dsn_from_config(config)
|
||||||
|
|
||||||
|
|
||||||
logg.info('using migrations dir {}'.format(migrations_dir))
|
logg.info('using migrations dir {}'.format(migrations_dir))
|
@ -1,7 +1,6 @@
|
|||||||
[metadata]
|
[metadata]
|
||||||
name = cic-cache
|
name = cic-cache
|
||||||
description = CIC Cache API and server
|
description = CIC Cache API and server
|
||||||
version = 0.3.0a2
|
|
||||||
author = Louis Holbrook
|
author = Louis Holbrook
|
||||||
author_email = dev@holbrook.no
|
author_email = dev@holbrook.no
|
||||||
url = https://gitlab.com/grassrootseconomics/cic-eth
|
url = https://gitlab.com/grassrootseconomics/cic-eth
|
||||||
@ -35,7 +34,7 @@ packages =
|
|||||||
cic_cache.runnable.daemons
|
cic_cache.runnable.daemons
|
||||||
cic_cache.runnable.daemons.filters
|
cic_cache.runnable.daemons.filters
|
||||||
scripts =
|
scripts =
|
||||||
./scripts/migrate_cic_cache.py
|
./scripts/migrate.py
|
||||||
|
|
||||||
[options.entry_points]
|
[options.entry_points]
|
||||||
console_scripts =
|
console_scripts =
|
||||||
|
@ -1,39 +1,38 @@
|
|||||||
from setuptools import setup
|
from setuptools import setup
|
||||||
|
|
||||||
# import configparser
|
import configparser
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
# import time
|
from cic_cache.version import (
|
||||||
|
version_object,
|
||||||
|
version_string
|
||||||
|
)
|
||||||
|
|
||||||
# from cic_cache.version import (
|
class PleaseCommitFirstError(Exception):
|
||||||
# version_object,
|
pass
|
||||||
# version_string
|
|
||||||
# )
|
def git_hash():
|
||||||
#
|
import subprocess
|
||||||
# class PleaseCommitFirstError(Exception):
|
git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
||||||
# pass
|
if len(git_diff.stdout) > 0:
|
||||||
#
|
raise PleaseCommitFirstError()
|
||||||
# def git_hash():
|
git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
||||||
# import subprocess
|
git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
||||||
# git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
return git_hash_brief
|
||||||
# if len(git_diff.stdout) > 0:
|
|
||||||
# raise PleaseCommitFirstError()
|
version_string = str(version_object)
|
||||||
# git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
|
||||||
# git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
try:
|
||||||
# return git_hash_brief
|
version_git = git_hash()
|
||||||
#
|
version_string += '+build.{}'.format(version_git)
|
||||||
# version_string = str(version_object)
|
except FileNotFoundError:
|
||||||
#
|
time_string_pair = str(time.time()).split('.')
|
||||||
# try:
|
version_string += '+build.{}{:<09d}'.format(
|
||||||
# version_git = git_hash()
|
time_string_pair[0],
|
||||||
# version_string += '+build.{}'.format(version_git)
|
int(time_string_pair[1]),
|
||||||
# except FileNotFoundError:
|
)
|
||||||
# time_string_pair = str(time.time()).split('.')
|
print('final version string will be {}'.format(version_string))
|
||||||
# version_string += '+build.{}{:<09d}'.format(
|
|
||||||
# time_string_pair[0],
|
|
||||||
# int(time_string_pair[1]),
|
|
||||||
# )
|
|
||||||
# print('final version string will be {}'.format(version_string))
|
|
||||||
|
|
||||||
requirements = []
|
requirements = []
|
||||||
f = open('requirements.txt', 'r')
|
f = open('requirements.txt', 'r')
|
||||||
@ -53,8 +52,9 @@ while True:
|
|||||||
test_requirements.append(l.rstrip())
|
test_requirements.append(l.rstrip())
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
# version=version_string,
|
version=version_string,
|
||||||
install_requires=requirements,
|
install_requires=requirements,
|
||||||
tests_require=test_requirements,
|
tests_require=test_requirements,
|
||||||
)
|
)
|
||||||
|
@ -6,5 +6,5 @@ sqlparse==0.4.1
|
|||||||
pytest-celery==0.0.0a1
|
pytest-celery==0.0.0a1
|
||||||
eth_tester==0.5.0b3
|
eth_tester==0.5.0b3
|
||||||
py-evm==0.3.0a20
|
py-evm==0.3.0a20
|
||||||
sarafu-faucet~=0.0.7a1
|
sarafu-faucet~=0.0.5a2
|
||||||
erc20-transfer-authorization~=0.3.6
|
erc20-transfer-authorization>=0.3.4a1,<0.4.0
|
||||||
|
@ -6,7 +6,6 @@ import datetime
|
|||||||
# external imports
|
# external imports
|
||||||
import pytest
|
import pytest
|
||||||
import moolb
|
import moolb
|
||||||
from chainlib.encode import TxHexNormalizer
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_cache import db
|
from cic_cache import db
|
||||||
@ -43,8 +42,6 @@ def txs(
|
|||||||
list_tokens,
|
list_tokens,
|
||||||
):
|
):
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
session = init_database
|
session = init_database
|
||||||
|
|
||||||
tx_number = 13
|
tx_number = 13
|
||||||
@ -57,10 +54,10 @@ def txs(
|
|||||||
tx_hash_first,
|
tx_hash_first,
|
||||||
list_defaults['block'],
|
list_defaults['block'],
|
||||||
tx_number,
|
tx_number,
|
||||||
tx_normalize.wallet_address(list_actors['alice']),
|
list_actors['alice'],
|
||||||
tx_normalize.wallet_address(list_actors['bob']),
|
list_actors['bob'],
|
||||||
tx_normalize.executable_address(list_tokens['foo']),
|
list_tokens['foo'],
|
||||||
tx_normalize.executable_address(list_tokens['foo']),
|
list_tokens['foo'],
|
||||||
1024,
|
1024,
|
||||||
2048,
|
2048,
|
||||||
True,
|
True,
|
||||||
@ -77,10 +74,10 @@ def txs(
|
|||||||
tx_hash_second,
|
tx_hash_second,
|
||||||
list_defaults['block']-1,
|
list_defaults['block']-1,
|
||||||
tx_number,
|
tx_number,
|
||||||
tx_normalize.wallet_address(list_actors['diane']),
|
list_actors['diane'],
|
||||||
tx_normalize.wallet_address(list_actors['alice']),
|
list_actors['alice'],
|
||||||
tx_normalize.executable_address(list_tokens['foo']),
|
list_tokens['foo'],
|
||||||
tx_normalize.wallet_address(list_tokens['foo']),
|
list_tokens['foo'],
|
||||||
1024,
|
1024,
|
||||||
2048,
|
2048,
|
||||||
False,
|
False,
|
||||||
@ -106,8 +103,6 @@ def more_txs(
|
|||||||
|
|
||||||
session = init_database
|
session = init_database
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
tx_number = 666
|
tx_number = 666
|
||||||
tx_hash = '0x' + os.urandom(32).hex()
|
tx_hash = '0x' + os.urandom(32).hex()
|
||||||
tx_signed = '0x' + os.urandom(128).hex()
|
tx_signed = '0x' + os.urandom(128).hex()
|
||||||
@ -120,10 +115,10 @@ def more_txs(
|
|||||||
tx_hash,
|
tx_hash,
|
||||||
list_defaults['block']+2,
|
list_defaults['block']+2,
|
||||||
tx_number,
|
tx_number,
|
||||||
tx_normalize.wallet_address(list_actors['alice']),
|
list_actors['alice'],
|
||||||
tx_normalize.wallet_address(list_actors['diane']),
|
list_actors['diane'],
|
||||||
tx_normalize.executable_address(list_tokens['bar']),
|
list_tokens['bar'],
|
||||||
tx_normalize.executable_address(list_tokens['bar']),
|
list_tokens['bar'],
|
||||||
2048,
|
2048,
|
||||||
4096,
|
4096,
|
||||||
False,
|
False,
|
||||||
|
@ -14,8 +14,7 @@ logg = logging.getLogger(__file__)
|
|||||||
@pytest.fixture(scope='session')
|
@pytest.fixture(scope='session')
|
||||||
def load_config():
|
def load_config():
|
||||||
config_dir = os.path.join(root_dir, 'config/test')
|
config_dir = os.path.join(root_dir, 'config/test')
|
||||||
schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
|
conf = confini.Config(config_dir, 'CICTEST')
|
||||||
conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
|
|
||||||
conf.process()
|
conf.process()
|
||||||
logg.debug('config {}'.format(conf))
|
logg.debug('config {}'.format(conf))
|
||||||
return conf
|
return conf
|
||||||
|
@ -24,15 +24,11 @@ def database_engine(
|
|||||||
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
||||||
SessionBase.transactional = False
|
SessionBase.transactional = False
|
||||||
SessionBase.poolable = False
|
SessionBase.poolable = False
|
||||||
name = 'cic_cache'
|
|
||||||
database_name = name
|
|
||||||
if load_config.get('DATABASE_PREFIX'):
|
|
||||||
database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
|
|
||||||
try:
|
try:
|
||||||
os.unlink(database_name)
|
os.unlink(load_config.get('DATABASE_NAME'))
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
dsn = dsn_from_config(load_config, name)
|
dsn = dsn_from_config(load_config)
|
||||||
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
|
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
|
||||||
return dsn
|
return dsn
|
||||||
|
|
||||||
|
@ -14,7 +14,7 @@ def test_api_all_data(
|
|||||||
):
|
):
|
||||||
|
|
||||||
env = {
|
env = {
|
||||||
'PATH_INFO': '/txa/100/0/410000/420000',
|
'PATH_INFO': '/txa/410000/420000',
|
||||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||||
}
|
}
|
||||||
j = process_transactions_all_data(init_database, env)
|
j = process_transactions_all_data(init_database, env)
|
||||||
@ -23,7 +23,7 @@ def test_api_all_data(
|
|||||||
assert len(o['data']) == 2
|
assert len(o['data']) == 2
|
||||||
|
|
||||||
env = {
|
env = {
|
||||||
'PATH_INFO': '/txa/100/0/420000/410000',
|
'PATH_INFO': '/txa/420000/410000',
|
||||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6,7 +6,6 @@ import json
|
|||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import pytest
|
import pytest
|
||||||
from chainlib.encode import TxHexNormalizer
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_cache import db
|
from cic_cache import db
|
||||||
@ -63,8 +62,6 @@ def test_cache_ranges(
|
|||||||
|
|
||||||
session = init_database
|
session = init_database
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
oldest = list_defaults['block'] - 1
|
oldest = list_defaults['block'] - 1
|
||||||
mid = list_defaults['block']
|
mid = list_defaults['block']
|
||||||
newest = list_defaults['block'] + 2
|
newest = list_defaults['block'] + 2
|
||||||
@ -103,39 +100,32 @@ def test_cache_ranges(
|
|||||||
assert b[1] == mid
|
assert b[1] == mid
|
||||||
|
|
||||||
# now check when supplying account
|
# now check when supplying account
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account(list_actors['alice'], 0, 100)
|
||||||
b = c.load_transactions_account(account, 0, 100)
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
b = c.load_transactions_account(list_actors['bob'], 0, 100)
|
||||||
b = c.load_transactions_account(account, 0, 100)
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == mid
|
assert b[1] == mid
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
b = c.load_transactions_account(list_actors['diane'], 0, 100)
|
||||||
b = c.load_transactions_account(account, 0, 100)
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
|
|
||||||
# add block filter to the mix
|
# add block filter to the mix
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == mid
|
assert b[1] == mid
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == oldest
|
assert b[1] == oldest
|
||||||
|
|
||||||
@ -150,8 +140,6 @@ def test_cache_ranges_data(
|
|||||||
|
|
||||||
session = init_database
|
session = init_database
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
oldest = list_defaults['block'] - 1
|
oldest = list_defaults['block'] - 1
|
||||||
mid = list_defaults['block']
|
mid = list_defaults['block']
|
||||||
newest = list_defaults['block'] + 2
|
newest = list_defaults['block'] + 2
|
||||||
@ -215,8 +203,7 @@ def test_cache_ranges_data(
|
|||||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||||
|
|
||||||
# now check when supplying account
|
# now check when supplying account
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
assert len(b[2]) == 3
|
assert len(b[2]) == 3
|
||||||
@ -224,15 +211,13 @@ def test_cache_ranges_data(
|
|||||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||||
assert b[2][2]['tx_hash'] == more_txs[2]
|
assert b[2][2]['tx_hash'] == more_txs[2]
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == mid
|
assert b[1] == mid
|
||||||
assert len(b[2]) == 1
|
assert len(b[2]) == 1
|
||||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
assert len(b[2]) == 2
|
assert len(b[2]) == 2
|
||||||
@ -240,31 +225,27 @@ def test_cache_ranges_data(
|
|||||||
assert b[2][1]['tx_hash'] == more_txs[2]
|
assert b[2][1]['tx_hash'] == more_txs[2]
|
||||||
|
|
||||||
# add block filter to the mix
|
# add block filter to the mix
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
assert len(b[2]) == 2
|
assert len(b[2]) == 2
|
||||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == newest
|
assert b[1] == newest
|
||||||
assert len(b[2]) == 2
|
assert len(b[2]) == 2
|
||||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
|
||||||
assert b[0] == mid
|
assert b[0] == mid
|
||||||
assert b[1] == mid
|
assert b[1] == mid
|
||||||
assert len(b[2]) == 1
|
assert len(b[2]) == 1
|
||||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||||
|
|
||||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
|
||||||
assert b[0] == oldest
|
assert b[0] == oldest
|
||||||
assert b[1] == oldest
|
assert b[1] == oldest
|
||||||
assert len(b[2]) == 1
|
assert len(b[2]) == 1
|
||||||
|
@ -82,7 +82,7 @@ def test_query_regex(
|
|||||||
[
|
[
|
||||||
('alice', None, None, [(420000, 13), (419999, 42)]),
|
('alice', None, None, [(420000, 13), (419999, 42)]),
|
||||||
('alice', None, 1, [(420000, 13)]),
|
('alice', None, 1, [(420000, 13)]),
|
||||||
('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
|
('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
|
||||||
('alice', 2, None, []), # 420000 == list_defaults['block']
|
('alice', 2, None, []), # 420000 == list_defaults['block']
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@ -107,11 +107,10 @@ def test_query_process_txs_account(
|
|||||||
path_info = '/tx/user/0x' + strip_0x(actor)
|
path_info = '/tx/user/0x' + strip_0x(actor)
|
||||||
if query_offset != None:
|
if query_offset != None:
|
||||||
path_info += '/' + str(query_offset)
|
path_info += '/' + str(query_offset)
|
||||||
if query_limit == None:
|
if query_limit != None:
|
||||||
query_limit = 100
|
if query_offset == None:
|
||||||
path_info += '/' + str(query_limit)
|
path_info += '/0'
|
||||||
if query_offset == None:
|
path_info += '/' + str(query_limit)
|
||||||
path_info += '/0'
|
|
||||||
env = {
|
env = {
|
||||||
'PATH_INFO': path_info,
|
'PATH_INFO': path_info,
|
||||||
}
|
}
|
||||||
@ -193,7 +192,7 @@ def test_query_process_txs_bloom(
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
'query_block_start, query_block_end, query_match_count',
|
'query_block_start, query_block_end, query_match_count',
|
||||||
[
|
[
|
||||||
(1, 42, 0),
|
(None, 42, 0),
|
||||||
(420000, 420001, 1),
|
(420000, 420001, 1),
|
||||||
(419999, 419999, 1), # matches are inclusive
|
(419999, 419999, 1), # matches are inclusive
|
||||||
(419999, 420000, 2),
|
(419999, 420000, 2),
|
||||||
@ -212,7 +211,7 @@ def test_query_process_txs_data(
|
|||||||
query_match_count,
|
query_match_count,
|
||||||
):
|
):
|
||||||
|
|
||||||
path_info = '/txa/100/0'
|
path_info = '/txa'
|
||||||
if query_block_start != None:
|
if query_block_start != None:
|
||||||
path_info += '/' + str(query_block_start)
|
path_info += '/' + str(query_block_start)
|
||||||
if query_block_end != None:
|
if query_block_end != None:
|
||||||
@ -228,5 +227,4 @@ def test_query_process_txs_data(
|
|||||||
assert r != None
|
assert r != None
|
||||||
|
|
||||||
o = json.loads(r[1])
|
o = json.loads(r[1])
|
||||||
logg.debug('oo {}'.format(o))
|
|
||||||
assert len(o['data']) == query_match_count
|
assert len(o['data']) == query_match_count
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
celery==4.4.7
|
celery==4.4.7
|
||||||
erc20-demurrage-token~=0.0.6
|
erc20-demurrage-token~=0.0.3a1
|
||||||
cic-eth-registry~=0.6.3
|
cic-eth-registry~=0.5.8a1
|
||||||
chainlib~=0.0.14
|
chainlib~=0.0.7a1
|
||||||
cic_eth~=0.12.6
|
cic_eth~=0.12.2a4
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
[metadata]
|
[metadata]
|
||||||
name = cic-eth-aux-erc20-demurrage-token
|
name = cic-eth-aux-erc20-demurrage-token
|
||||||
version = 0.0.3
|
version = 0.0.2a6
|
||||||
description = cic-eth tasks supporting erc20 demurrage token
|
description = cic-eth tasks supporting erc20 demurrage token
|
||||||
author = Louis Holbrook
|
author = Louis Holbrook
|
||||||
author_email = dev@holbrook.no
|
author_email = dev@holbrook.no
|
||||||
|
@ -1,16 +1,53 @@
|
|||||||
build-test-cic-eth:
|
.cic_eth_variables:
|
||||||
stage: test
|
variables:
|
||||||
tags:
|
APP_NAME: cic-eth
|
||||||
- integration
|
|
||||||
variables:
|
#build-mr-cic-eth:
|
||||||
APP_NAME: cic-eth
|
# extends:
|
||||||
MR_IMAGE_TAG: mr-$APP_NAME-$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA
|
# - .cic_eth_variables
|
||||||
script:
|
# - .py_build_target_dev
|
||||||
- cd apps/cic-eth
|
# rules:
|
||||||
- docker build -t $MR_IMAGE_TAG -f docker/Dockerfile .
|
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
- docker run $MR_IMAGE_TAG sh docker/run_tests.sh
|
# changes:
|
||||||
rules:
|
# - apps/cic-eth/**/*
|
||||||
|
# when: always
|
||||||
|
|
||||||
|
test-mr-cic-eth:
|
||||||
|
stage: test
|
||||||
|
tags:
|
||||||
|
- integration
|
||||||
|
extends:
|
||||||
|
- .cic_eth_variables
|
||||||
|
cache:
|
||||||
|
key:
|
||||||
|
files:
|
||||||
|
- test_requirements.txt
|
||||||
|
paths:
|
||||||
|
- /root/.cache/pip
|
||||||
|
image: registry.gitlab.com/grassrootseconomics/cic-internal-integration/$APP_NAME:$MR_IMAGE_TAG
|
||||||
|
script:
|
||||||
|
- cd apps/$APP_NAME/
|
||||||
|
- >
|
||||||
|
pip install --extra-index-url https://pip.grassrootseconomics.net:8433
|
||||||
|
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
|
||||||
|
-r admin_requirements.txt
|
||||||
|
-r services_requirements.txt
|
||||||
|
-r test_requirements.txt
|
||||||
|
- export PYTHONPATH=. && pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
|
||||||
|
allow_failure: true
|
||||||
|
needs: ["build-merge-request"]
|
||||||
|
rules:
|
||||||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||||||
changes:
|
changes:
|
||||||
- apps/$APP_NAME/**/*
|
- apps/$APP_NAME/**/*
|
||||||
when: always
|
when: always
|
||||||
|
|
||||||
|
#build-push-cic-eth:
|
||||||
|
# extends:
|
||||||
|
# - .py_build_push
|
||||||
|
# - .cic_eth_variables
|
||||||
|
# rules:
|
||||||
|
# - if: $CI_COMMIT_BRANCH == "master"
|
||||||
|
# changes:
|
||||||
|
# - apps/cic-eth/**/*
|
||||||
|
# when: always
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
SQLAlchemy==1.3.20
|
SQLAlchemy==1.3.20
|
||||||
hexathon~=0.1.0
|
cic-eth-registry>=0.5.6a2,<0.6.0
|
||||||
chainqueue~=0.0.6a4
|
hexathon~=0.0.1a7
|
||||||
eth-erc20~=0.1.5
|
chainqueue>=0.0.3a1,<0.1.0
|
||||||
|
eth-erc20>=0.0.10a3,<0.1.0
|
||||||
|
@ -4,6 +4,7 @@ import logging
|
|||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import celery
|
import celery
|
||||||
|
from chainlib.eth.constant import ZERO_ADDRESS
|
||||||
from chainlib.chain import ChainSpec
|
from chainlib.chain import ChainSpec
|
||||||
from hexathon import (
|
from hexathon import (
|
||||||
add_0x,
|
add_0x,
|
||||||
@ -19,17 +20,18 @@ from cic_eth.task import (
|
|||||||
CriticalSQLAlchemyTask,
|
CriticalSQLAlchemyTask,
|
||||||
)
|
)
|
||||||
from cic_eth.error import LockedError
|
from cic_eth.error import LockedError
|
||||||
from cic_eth.encode import (
|
|
||||||
tx_normalize,
|
|
||||||
ZERO_ADDRESS_NORMAL,
|
|
||||||
)
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_address(a):
|
||||||
|
if a == None:
|
||||||
|
return None
|
||||||
|
return add_0x(hex_uniform(strip_0x(a)))
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL, tx_hash=None):
|
def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL, tx_hash=None):
|
||||||
"""Task wrapper to set arbitrary locks
|
"""Task wrapper to set arbitrary locks
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -41,7 +43,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = '::'
|
chain_str = '::'
|
||||||
if chain_spec_dict != None:
|
if chain_spec_dict != None:
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
@ -51,7 +53,7 @@ def lock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lock
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=LockEnum.ALL):
|
def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS, flags=LockEnum.ALL):
|
||||||
"""Task wrapper to reset arbitrary locks
|
"""Task wrapper to reset arbitrary locks
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -63,7 +65,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = '::'
|
chain_str = '::'
|
||||||
if chain_spec_dict != None:
|
if chain_spec_dict != None:
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
@ -73,7 +75,7 @@ def unlock(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, flags=Lo
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
|
def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
|
||||||
"""Task wrapper to set send lock
|
"""Task wrapper to set send lock
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -83,7 +85,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
|
r = Lock.set(chain_str, LockEnum.SEND, address=address, tx_hash=tx_hash)
|
||||||
logg.debug('Send locked for {}, flag now {}'.format(address, r))
|
logg.debug('Send locked for {}, flag now {}'.format(address, r))
|
||||||
@ -91,7 +93,7 @@ def lock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_ha
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
|
||||||
"""Task wrapper to reset send lock
|
"""Task wrapper to reset send lock
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -101,7 +103,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
r = Lock.reset(chain_str, LockEnum.SEND, address=address)
|
r = Lock.reset(chain_str, LockEnum.SEND, address=address)
|
||||||
logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
|
logg.debug('Send unlocked for {}, flag now {}'.format(address, r))
|
||||||
@ -109,7 +111,7 @@ def unlock_send(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_hash=None):
|
def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS, tx_hash=None):
|
||||||
"""Task wrapper to set queue direct lock
|
"""Task wrapper to set queue direct lock
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -119,7 +121,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
|
r = Lock.set(chain_str, LockEnum.QUEUE, address=address, tx_hash=tx_hash)
|
||||||
logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
|
logg.debug('Queue direct locked for {}, flag now {}'.format(address, r))
|
||||||
@ -127,7 +129,7 @@ def lock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL, tx_h
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS):
|
||||||
"""Task wrapper to reset queue direct lock
|
"""Task wrapper to reset queue direct lock
|
||||||
|
|
||||||
:param chain_str: Chain spec string representation
|
:param chain_str: Chain spec string representation
|
||||||
@ -137,7 +139,7 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
|||||||
:returns: New lock state for address
|
:returns: New lock state for address
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(address)
|
address = normalize_address(address)
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
|
r = Lock.reset(chain_str, LockEnum.QUEUE, address=address)
|
||||||
logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
|
logg.debug('Queue direct unlocked for {}, flag now {}'.format(address, r))
|
||||||
@ -146,13 +148,12 @@ def unlock_queue(chained_input, chain_spec_dict, address=ZERO_ADDRESS_NORMAL):
|
|||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||||
def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
|
def check_lock(chained_input, chain_spec_dict, lock_flags, address=None):
|
||||||
if address != None:
|
address = normalize_address(address)
|
||||||
address = tx_normalize.wallet_address(address)
|
|
||||||
chain_str = '::'
|
chain_str = '::'
|
||||||
if chain_spec_dict != None:
|
if chain_spec_dict != None:
|
||||||
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
chain_str = str(ChainSpec.from_dict(chain_spec_dict))
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS_NORMAL, session=session)
|
r = Lock.check(chain_str, lock_flags, address=ZERO_ADDRESS, session=session)
|
||||||
if address != None:
|
if address != None:
|
||||||
r |= Lock.check(chain_str, lock_flags, address=address, session=session)
|
r |= Lock.check(chain_str, lock_flags, address=address, session=session)
|
||||||
if r > 0:
|
if r > 0:
|
||||||
|
@ -33,7 +33,6 @@ from cic_eth.admin.ctrl import (
|
|||||||
from cic_eth.queue.tx import queue_create
|
from cic_eth.queue.tx import queue_create
|
||||||
from cic_eth.eth.gas import create_check_gas_task
|
from cic_eth.eth.gas import create_check_gas_task
|
||||||
from cic_eth.task import BaseTask
|
from cic_eth.task import BaseTask
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
@ -74,7 +73,7 @@ def shift_nonce(self, chainspec_dict, tx_hash_orig_hex, delta=1):
|
|||||||
|
|
||||||
set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
|
set_cancel(chain_spec, strip_0x(tx['hash']), manual=True, session=session)
|
||||||
|
|
||||||
query_address = tx_normalize.wallet_address(address)
|
query_address = add_0x(hex_uniform(strip_0x(address))) # aaaaargh
|
||||||
q = session.query(Otx)
|
q = session.query(Otx)
|
||||||
q = q.join(TxCache)
|
q = q.join(TxCache)
|
||||||
q = q.filter(TxCache.sender==query_address)
|
q = q.filter(TxCache.sender==query_address)
|
||||||
|
@ -1,2 +1,21 @@
|
|||||||
|
# standard imports
|
||||||
|
import logging
|
||||||
|
|
||||||
|
# external imports
|
||||||
|
import celery
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.eth.erc20 import default_token
|
from cic_eth.task import BaseTask
|
||||||
|
|
||||||
|
celery_app = celery.current_app
|
||||||
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(bind=True, base=BaseTask)
|
||||||
|
def default_token(self):
|
||||||
|
return {
|
||||||
|
'symbol': self.default_token_symbol,
|
||||||
|
'address': self.default_token_address,
|
||||||
|
'name': self.default_token_name,
|
||||||
|
'decimals': self.default_token_decimals,
|
||||||
|
}
|
||||||
|
@ -32,6 +32,7 @@ from chainqueue.db.enum import (
|
|||||||
status_str,
|
status_str,
|
||||||
)
|
)
|
||||||
from chainqueue.error import TxStateChangeError
|
from chainqueue.error import TxStateChangeError
|
||||||
|
from chainqueue.sql.query import get_tx
|
||||||
from eth_erc20 import ERC20
|
from eth_erc20 import ERC20
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
@ -39,7 +40,6 @@ from cic_eth.db.models.base import SessionBase
|
|||||||
from cic_eth.db.models.role import AccountRole
|
from cic_eth.db.models.role import AccountRole
|
||||||
from cic_eth.db.models.nonce import Nonce
|
from cic_eth.db.models.nonce import Nonce
|
||||||
from cic_eth.error import InitializationError
|
from cic_eth.error import InitializationError
|
||||||
from cic_eth.queue.query import get_tx_local
|
|
||||||
|
|
||||||
app = celery.current_app
|
app = celery.current_app
|
||||||
|
|
||||||
@ -123,7 +123,7 @@ class AdminApi:
|
|||||||
return s_lock.apply_async()
|
return s_lock.apply_async()
|
||||||
|
|
||||||
|
|
||||||
def tag_account(self, chain_spec, tag, address):
|
def tag_account(self, tag, address_hex, chain_spec):
|
||||||
"""Persistently associate an address with a plaintext tag.
|
"""Persistently associate an address with a plaintext tag.
|
||||||
|
|
||||||
Some tags are known by the system and is used to resolve addresses to use for certain transactions.
|
Some tags are known by the system and is used to resolve addresses to use for certain transactions.
|
||||||
@ -138,7 +138,7 @@ class AdminApi:
|
|||||||
'cic_eth.eth.account.set_role',
|
'cic_eth.eth.account.set_role',
|
||||||
[
|
[
|
||||||
tag,
|
tag,
|
||||||
address,
|
address_hex,
|
||||||
chain_spec.asdict(),
|
chain_spec.asdict(),
|
||||||
],
|
],
|
||||||
queue=self.queue,
|
queue=self.queue,
|
||||||
@ -146,30 +146,6 @@ class AdminApi:
|
|||||||
return s_tag.apply_async()
|
return s_tag.apply_async()
|
||||||
|
|
||||||
|
|
||||||
def get_tag_account(self, chain_spec, tag=None, address=None):
|
|
||||||
if address != None:
|
|
||||||
s_tag = celery.signature(
|
|
||||||
'cic_eth.eth.account.role',
|
|
||||||
[
|
|
||||||
address,
|
|
||||||
chain_spec.asdict(),
|
|
||||||
],
|
|
||||||
queue=self.queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
else:
|
|
||||||
s_tag = celery.signature(
|
|
||||||
'cic_eth.eth.account.role_account',
|
|
||||||
[
|
|
||||||
tag,
|
|
||||||
chain_spec.asdict(),
|
|
||||||
],
|
|
||||||
queue=self.queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
return s_tag.apply_async()
|
|
||||||
|
|
||||||
|
|
||||||
def have_account(self, address_hex, chain_spec):
|
def have_account(self, address_hex, chain_spec):
|
||||||
s_have = celery.signature(
|
s_have = celery.signature(
|
||||||
'cic_eth.eth.account.have',
|
'cic_eth.eth.account.have',
|
||||||
@ -308,7 +284,7 @@ class AdminApi:
|
|||||||
tx_hash_hex = None
|
tx_hash_hex = None
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
for k in txs.keys():
|
for k in txs.keys():
|
||||||
tx_dict = get_tx_local(chain_spec, k, session=session)
|
tx_dict = get_tx(chain_spec, k, session=session)
|
||||||
if tx_dict['nonce'] == nonce:
|
if tx_dict['nonce'] == nonce:
|
||||||
tx_hash_hex = k
|
tx_hash_hex = k
|
||||||
session.close()
|
session.close()
|
||||||
@ -527,7 +503,7 @@ class AdminApi:
|
|||||||
queue=self.queue,
|
queue=self.queue,
|
||||||
)
|
)
|
||||||
t = s.apply_async()
|
t = s.apply_async()
|
||||||
role = t.get()[0][1]
|
role = t.get()
|
||||||
if role != None:
|
if role != None:
|
||||||
tx['sender_description'] = role
|
tx['sender_description'] = role
|
||||||
|
|
||||||
@ -580,7 +556,7 @@ class AdminApi:
|
|||||||
queue=self.queue,
|
queue=self.queue,
|
||||||
)
|
)
|
||||||
t = s.apply_async()
|
t = s.apply_async()
|
||||||
role = t.get()[0][1]
|
role = t.get()
|
||||||
if role != None:
|
if role != None:
|
||||||
tx['recipient_description'] = role
|
tx['recipient_description'] = role
|
||||||
|
|
||||||
|
@ -9,7 +9,6 @@ import logging
|
|||||||
# external imports
|
# external imports
|
||||||
import celery
|
import celery
|
||||||
from chainlib.chain import ChainSpec
|
from chainlib.chain import ChainSpec
|
||||||
from hexathon import strip_0x
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.api.base import ApiBase
|
from cic_eth.api.base import ApiBase
|
||||||
@ -17,50 +16,15 @@ from cic_eth.enum import LockEnum
|
|||||||
|
|
||||||
app = celery.current_app
|
app = celery.current_app
|
||||||
|
|
||||||
#logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
logg = logging.getLogger()
|
|
||||||
|
|
||||||
|
|
||||||
class Api(ApiBase):
|
class Api(ApiBase):
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def to_v_list(v, n):
|
|
||||||
"""Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
|
|
||||||
|
|
||||||
:param v: Arguments
|
|
||||||
:type v: str or list
|
|
||||||
:param n: Number of elements to generate arguments for
|
|
||||||
:type n: int
|
|
||||||
:rtype: list
|
|
||||||
:returns: list of assembled arguments
|
|
||||||
"""
|
|
||||||
if isinstance(v, str):
|
|
||||||
vv = v
|
|
||||||
v = []
|
|
||||||
for i in range(n):
|
|
||||||
v.append([vv])
|
|
||||||
elif not isinstance(v, list):
|
|
||||||
raise ValueError('argument must be single string, or list or strings or lists')
|
|
||||||
else:
|
|
||||||
if len(v) != n:
|
|
||||||
raise ValueError('v argument count must match integer n')
|
|
||||||
for i in range(n):
|
|
||||||
if isinstance(v[i], str):
|
|
||||||
v[i] = [v[i]]
|
|
||||||
elif not isinstance(v, list):
|
|
||||||
raise ValueError('proof argument must be single string, or list or strings or lists')
|
|
||||||
|
|
||||||
return v
|
|
||||||
|
|
||||||
|
|
||||||
def default_token(self):
|
def default_token(self):
|
||||||
"""Retrieves the default fallback token of the custodial network.
|
|
||||||
|
|
||||||
:returns: uuid of root task
|
|
||||||
:rtype: celery.Task
|
|
||||||
"""
|
|
||||||
s_token = celery.signature(
|
s_token = celery.signature(
|
||||||
'cic_eth.eth.erc20.default_token',
|
'cic_eth.admin.token.default_token',
|
||||||
[],
|
[],
|
||||||
queue=self.queue,
|
queue=self.queue,
|
||||||
)
|
)
|
||||||
@ -70,97 +34,6 @@ class Api(ApiBase):
|
|||||||
return s_token.apply_async()
|
return s_token.apply_async()
|
||||||
|
|
||||||
|
|
||||||
def token(self, token_symbol, proof=None):
|
|
||||||
"""Single-token alias for tokens method.
|
|
||||||
|
|
||||||
See tokens method for details.
|
|
||||||
|
|
||||||
:param token_symbol: Token symbol to look up
|
|
||||||
:type token_symbol: str
|
|
||||||
:param proof: Proofs to add to signature verification for the token
|
|
||||||
:type proof: str or list
|
|
||||||
:returns: uuid of root task
|
|
||||||
:rtype: celery.Task
|
|
||||||
"""
|
|
||||||
if not isinstance(token_symbol, str):
|
|
||||||
raise ValueError('token symbol must be string')
|
|
||||||
|
|
||||||
return self.tokens([token_symbol], proof=proof)
|
|
||||||
|
|
||||||
|
|
||||||
def tokens(self, token_symbols, proof=None):
|
|
||||||
"""Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
|
|
||||||
|
|
||||||
Token symbols are always strings, and should be specified using uppercase letters.
|
|
||||||
|
|
||||||
If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
|
|
||||||
|
|
||||||
The proof argument may be specified in a number of ways:
|
|
||||||
|
|
||||||
- as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
|
|
||||||
- as a single string, where the same proof is used for each token lookup
|
|
||||||
- as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
|
|
||||||
- as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
|
|
||||||
|
|
||||||
The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
|
|
||||||
|
|
||||||
This method is not intended to be used synchronously. Do so at your peril.
|
|
||||||
|
|
||||||
:param token_symbols: Token symbol strings to look up
|
|
||||||
:type token_symbol: list
|
|
||||||
:param proof: Proof(s) to verify tokens against
|
|
||||||
:type proof: None, str or list
|
|
||||||
:returns: uuid of root task
|
|
||||||
:rtype: celery.Task
|
|
||||||
"""
|
|
||||||
if not isinstance(token_symbols, list):
|
|
||||||
raise ValueError('token symbols argument must be list')
|
|
||||||
|
|
||||||
if proof == None:
|
|
||||||
logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
|
|
||||||
proof = ''
|
|
||||||
|
|
||||||
logg.debug('proof is {}'.format(proof))
|
|
||||||
l = len(token_symbols)
|
|
||||||
if len(proof) == 0:
|
|
||||||
l = 0
|
|
||||||
proof = Api.to_v_list(proof, l)
|
|
||||||
|
|
||||||
chain_spec_dict = self.chain_spec.asdict()
|
|
||||||
|
|
||||||
s_token_resolve = celery.signature(
|
|
||||||
'cic_eth.eth.erc20.resolve_tokens_by_symbol',
|
|
||||||
[
|
|
||||||
token_symbols,
|
|
||||||
chain_spec_dict,
|
|
||||||
],
|
|
||||||
queue=self.queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
s_token_info = celery.signature(
|
|
||||||
'cic_eth.eth.erc20.token_info',
|
|
||||||
[
|
|
||||||
chain_spec_dict,
|
|
||||||
proof,
|
|
||||||
],
|
|
||||||
queue=self.queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
s_token_verify = celery.signature(
|
|
||||||
'cic_eth.eth.erc20.verify_token_info',
|
|
||||||
[
|
|
||||||
chain_spec_dict,
|
|
||||||
self.callback_success,
|
|
||||||
self.callback_error,
|
|
||||||
],
|
|
||||||
queue=self.queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
s_token_info.link(s_token_verify)
|
|
||||||
s_token_resolve.link(s_token_info)
|
|
||||||
return s_token_resolve.apply_async()
|
|
||||||
|
|
||||||
|
|
||||||
# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
|
# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
|
||||||
# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
|
# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
|
||||||
#
|
#
|
||||||
@ -381,8 +254,6 @@ class Api(ApiBase):
|
|||||||
:returns: uuid of root task
|
:returns: uuid of root task
|
||||||
:rtype: celery.Task
|
:rtype: celery.Task
|
||||||
"""
|
"""
|
||||||
#from_address = strip_0x(from_address)
|
|
||||||
#to_address = strip_0x(to_address)
|
|
||||||
s_check = celery.signature(
|
s_check = celery.signature(
|
||||||
'cic_eth.admin.ctrl.check_lock',
|
'cic_eth.admin.ctrl.check_lock',
|
||||||
[
|
[
|
||||||
@ -515,7 +386,7 @@ class Api(ApiBase):
|
|||||||
:param password: Password to encode the password with in the backend (careful, you will have to remember it)
|
:param password: Password to encode the password with in the backend (careful, you will have to remember it)
|
||||||
:type password: str
|
:type password: str
|
||||||
:param register: Register the new account in accounts index backend
|
:param register: Register the new account in accounts index backend
|
||||||
:type register: bool
|
:type password: bool
|
||||||
:returns: uuid of root task
|
:returns: uuid of root task
|
||||||
:rtype: celery.Task
|
:rtype: celery.Task
|
||||||
"""
|
"""
|
||||||
@ -683,4 +554,3 @@ class Api(ApiBase):
|
|||||||
|
|
||||||
t = self.callback_success.apply_async([r])
|
t = self.callback_success.apply_async([r])
|
||||||
return t
|
return t
|
||||||
|
|
||||||
|
@ -1,10 +1,7 @@
|
|||||||
import logging
|
|
||||||
|
|
||||||
import celery
|
import celery
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
#logg = celery_app.log.get_default_logger()
|
logg = celery_app.log.get_default_logger()
|
||||||
logg = logging.getLogger()
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True)
|
@celery_app.task(bind=True)
|
||||||
|
@ -12,9 +12,8 @@ from cic_eth.db.models.base import SessionBase
|
|||||||
from cic_eth.db.enum import LockEnum
|
from cic_eth.db.enum import LockEnum
|
||||||
from cic_eth.error import LockedError
|
from cic_eth.error import LockedError
|
||||||
from cic_eth.admin.ctrl import check_lock
|
from cic_eth.admin.ctrl import check_lock
|
||||||
from cic_eth.eth.gas import have_gas_minimum
|
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger().getChild(__name__)
|
||||||
|
|
||||||
|
|
||||||
def health(*args, **kwargs):
|
def health(*args, **kwargs):
|
||||||
@ -32,15 +31,18 @@ def health(*args, **kwargs):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
|
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
|
||||||
min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
|
|
||||||
if config.get('ETH_MIN_FEE_PRICE'):
|
|
||||||
min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
|
|
||||||
|
|
||||||
r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
|
|
||||||
|
|
||||||
session.close()
|
session.close()
|
||||||
|
|
||||||
if not r:
|
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||||
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
|
o = balance(gas_provider)
|
||||||
|
r = rpc.do(o)
|
||||||
|
try:
|
||||||
|
r = int(r, 16)
|
||||||
|
except TypeError:
|
||||||
|
r = int(r)
|
||||||
|
gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
|
||||||
|
if r < gas_min:
|
||||||
|
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
|
||||||
|
return False
|
||||||
|
|
||||||
return r
|
return True
|
||||||
|
@ -1,18 +0,0 @@
|
|||||||
# external imports
|
|
||||||
from chainlib.chain import ChainSpec
|
|
||||||
|
|
||||||
# local imports
|
|
||||||
from cic_eth.admin.ctrl import check_lock
|
|
||||||
from cic_eth.enum import LockEnum
|
|
||||||
from cic_eth.error import LockedError
|
|
||||||
|
|
||||||
|
|
||||||
def health(*args, **kwargs):
|
|
||||||
config = kwargs['config']
|
|
||||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
|
||||||
|
|
||||||
try:
|
|
||||||
check_lock(None, chain_spec.asdict(), LockEnum.START)
|
|
||||||
except LockedError as e:
|
|
||||||
return False
|
|
||||||
return True
|
|
@ -16,22 +16,16 @@ class ArgumentParser(BaseArgumentParser):
|
|||||||
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
|
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
|
||||||
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
|
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
|
||||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||||
self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
|
self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
|
||||||
self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
|
self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
|
||||||
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
|
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
|
||||||
if local_arg_flags & CICFlag.CELERY:
|
if local_arg_flags & CICFlag.CELERY:
|
||||||
self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
|
|
||||||
self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
|
|
||||||
self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
|
|
||||||
self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
|
|
||||||
self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
|
|
||||||
self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
|
|
||||||
self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
|
|
||||||
self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
|
|
||||||
self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
|
|
||||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
|
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
|
||||||
if local_arg_flags & CICFlag.SYNCER:
|
if local_arg_flags & CICFlag.SYNCER:
|
||||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
|
||||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||||
if local_arg_flags & CICFlag.CHAIN:
|
if local_arg_flags & CICFlag.CHAIN:
|
||||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@ -24,7 +24,7 @@ class CICFlag(enum.IntEnum):
|
|||||||
# sync - nibble 4
|
# sync - nibble 4
|
||||||
SYNCER = 4096
|
SYNCER = 4096
|
||||||
|
|
||||||
argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
|
|
||||||
argflag_local_task = CICFlag.CELERY
|
argflag_local_task = CICFlag.CELERY
|
||||||
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
||||||
argflag_local_chain = CICFlag.CHAIN
|
argflag_local_chain = CICFlag.CHAIN
|
||||||
|
@ -1,18 +1,12 @@
|
|||||||
# standard imports
|
# standard imports
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
import urllib.parse
|
|
||||||
import copy
|
|
||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
from chainlib.eth.cli import (
|
from chainlib.eth.cli import (
|
||||||
Config as BaseConfig,
|
Config as BaseConfig,
|
||||||
Flag,
|
Flag,
|
||||||
)
|
)
|
||||||
from urlybird.merge import (
|
|
||||||
urlhostmerge,
|
|
||||||
urlmerge,
|
|
||||||
)
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from .base import CICFlag
|
from .base import CICFlag
|
||||||
@ -46,7 +40,6 @@ class Config(BaseConfig):
|
|||||||
if local_arg_flags & CICFlag.CHAIN:
|
if local_arg_flags & CICFlag.CHAIN:
|
||||||
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
||||||
|
|
||||||
|
|
||||||
if local_arg_flags & CICFlag.CELERY:
|
if local_arg_flags & CICFlag.CELERY:
|
||||||
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
||||||
|
|
||||||
@ -56,71 +49,15 @@ class Config(BaseConfig):
|
|||||||
|
|
||||||
config.dict_override(local_args_override, 'local cli args')
|
config.dict_override(local_args_override, 'local cli args')
|
||||||
|
|
||||||
local_celery_args_override = {}
|
|
||||||
if local_arg_flags & CICFlag.CELERY:
|
|
||||||
hostport = urlhostmerge(
|
|
||||||
None,
|
|
||||||
config.get('REDIS_HOST'),
|
|
||||||
config.get('REDIS_PORT'),
|
|
||||||
)
|
|
||||||
db = getattr(args, 'redis_db', None)
|
|
||||||
if db != None:
|
|
||||||
db = str(db)
|
|
||||||
|
|
||||||
redis_url = (
|
|
||||||
'redis',
|
|
||||||
hostport,
|
|
||||||
db,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
|
|
||||||
hostport = urlhostmerge(
|
|
||||||
celery_config_url[1],
|
|
||||||
getattr(args, 'celery_host', None),
|
|
||||||
getattr(args, 'celery_port', None),
|
|
||||||
)
|
|
||||||
db = getattr(args, 'redis_db', None)
|
|
||||||
if db != None:
|
|
||||||
db = str(db)
|
|
||||||
celery_arg_url = (
|
|
||||||
getattr(args, 'celery_scheme', None),
|
|
||||||
hostport,
|
|
||||||
db,
|
|
||||||
)
|
|
||||||
|
|
||||||
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
|
|
||||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
|
||||||
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
|
|
||||||
if not getattr(args, 'celery_no_result'):
|
|
||||||
local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
|
|
||||||
if local_celery_args_override['CELERY_RESULT_URL'] == None:
|
|
||||||
local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
|
|
||||||
celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
|
|
||||||
hostport = urlhostmerge(
|
|
||||||
celery_config_url[1],
|
|
||||||
getattr(args, 'celery_result_host', None),
|
|
||||||
getattr(args, 'celery_result_port', None),
|
|
||||||
)
|
|
||||||
celery_arg_url = (
|
|
||||||
getattr(args, 'celery_result_scheme', None),
|
|
||||||
hostport,
|
|
||||||
getattr(args, 'celery_result_db', None),
|
|
||||||
)
|
|
||||||
celery_url = urlmerge(celery_config_url, celery_arg_url)
|
|
||||||
logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
|
|
||||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
|
||||||
local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
|
|
||||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
|
||||||
|
|
||||||
config.dict_override(local_celery_args_override, 'local celery cli args')
|
|
||||||
|
|
||||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||||
redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
|
config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
|
||||||
redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
|
config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
|
||||||
config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
|
|
||||||
config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
|
if local_arg_flags & CICFlag.CELERY:
|
||||||
|
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||||
|
|
||||||
logg.debug('config loaded:\n{}'.format(config))
|
logg.debug('config loaded:\n{}'.format(config))
|
||||||
|
|
||||||
return config
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
@ -35,14 +35,14 @@ class RPC:
|
|||||||
def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
|
def from_config(config, use_signer=False, default_label='default', signer_label='signer'):
|
||||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||||
|
|
||||||
RPCConnection.register_location(config.get('RPC_PROVIDER'), chain_spec, default_label)
|
RPCConnection.register_location(config.get('RPC_HTTP_PROVIDER'), chain_spec, default_label)
|
||||||
if use_signer:
|
if use_signer:
|
||||||
|
|
||||||
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
|
RPCConnection.register_constructor(ConnType.UNIX, EthUnixSignerConnection, signer_label)
|
||||||
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
|
RPCConnection.register_constructor(ConnType.HTTP, EthHTTPSignerConnection, signer_label)
|
||||||
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
|
RPCConnection.register_constructor(ConnType.HTTP_SSL, EthHTTPSignerConnection, signer_label)
|
||||||
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label)
|
RPCConnection.register_location(config.get('SIGNER_PROVIDER'), chain_spec, signer_label)
|
||||||
rpc = RPC(chain_spec, config.get('RPC_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
rpc = RPC(chain_spec, config.get('RPC_HTTP_PROVIDER'), signer_provider=config.get('SIGNER_PROVIDER'))
|
||||||
logg.info('set up rpc: {}'.format(rpc))
|
logg.info('set up rpc: {}'.format(rpc))
|
||||||
return rpc
|
return rpc
|
||||||
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
[celery]
|
[celery]
|
||||||
broker_url =
|
broker_url = redis://localhost:6379
|
||||||
result_url =
|
result_url =
|
||||||
queue = cic-eth
|
queue = cic-eth
|
||||||
debug = 0
|
debug = 0
|
||||||
|
@ -2,5 +2,5 @@
|
|||||||
registry_address =
|
registry_address =
|
||||||
trust_address =
|
trust_address =
|
||||||
default_token_symbol =
|
default_token_symbol =
|
||||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
|
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||||
run_dir = /run
|
run_dir = /run
|
||||||
|
@ -1,6 +1,2 @@
|
|||||||
[eth]
|
[eth]
|
||||||
gas_holder_minimum_units = 180000
|
gas_gifter_minimum_balance = 10000000000000000000000
|
||||||
gas_holder_refill_units = 15
|
|
||||||
gas_holder_refill_threshold = 3
|
|
||||||
gas_gifter_refill_buffer = 3
|
|
||||||
min_fee_price = 1
|
|
||||||
|
@ -8,8 +8,8 @@ Create Date: 2021-04-02 18:41:20.864265
|
|||||||
import datetime
|
import datetime
|
||||||
from alembic import op
|
from alembic import op
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
from chainlib.eth.constant import ZERO_ADDRESS
|
||||||
from cic_eth.db.enum import LockEnum
|
from cic_eth.db.enum import LockEnum
|
||||||
from cic_eth.encode import ZERO_ADDRESS_NORMAL
|
|
||||||
|
|
||||||
|
|
||||||
# revision identifiers, used by Alembic.
|
# revision identifiers, used by Alembic.
|
||||||
@ -23,14 +23,14 @@ def upgrade():
|
|||||||
op.create_table(
|
op.create_table(
|
||||||
'lock',
|
'lock',
|
||||||
sa.Column('id', sa.Integer, primary_key=True),
|
sa.Column('id', sa.Integer, primary_key=True),
|
||||||
sa.Column("address", sa.String, nullable=True),
|
sa.Column("address", sa.String(42), nullable=True),
|
||||||
sa.Column('blockchain', sa.String),
|
sa.Column('blockchain', sa.String),
|
||||||
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
|
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
|
||||||
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
|
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
|
||||||
sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
|
sa.Column("otx_id", sa.Integer, sa.ForeignKey('otx.id'), nullable=True),
|
||||||
)
|
)
|
||||||
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
|
op.create_index('idx_chain_address', 'lock', ['blockchain', 'address'], unique=True)
|
||||||
op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS_NORMAL, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
|
op.execute("INSERT INTO lock (address, date_created, blockchain, flags) VALUES('{}', '{}', '::', {})".format(ZERO_ADDRESS, datetime.datetime.utcnow(), LockEnum.INIT | LockEnum.SEND | LockEnum.QUEUE))
|
||||||
|
|
||||||
|
|
||||||
def downgrade():
|
def downgrade():
|
||||||
|
@ -1,31 +0,0 @@
|
|||||||
"""Add gas cache
|
|
||||||
|
|
||||||
Revision ID: c91cafc3e0c1
|
|
||||||
Revises: aee12aeb47ec
|
|
||||||
Create Date: 2021-10-28 20:45:34.239865
|
|
||||||
|
|
||||||
"""
|
|
||||||
from alembic import op
|
|
||||||
import sqlalchemy as sa
|
|
||||||
|
|
||||||
|
|
||||||
# revision identifiers, used by Alembic.
|
|
||||||
revision = 'c91cafc3e0c1'
|
|
||||||
down_revision = 'aee12aeb47ec'
|
|
||||||
branch_labels = None
|
|
||||||
depends_on = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade():
|
|
||||||
op.create_table(
|
|
||||||
'gas_cache',
|
|
||||||
sa.Column('id', sa.Integer, primary_key=True),
|
|
||||||
sa.Column("address", sa.String, nullable=False),
|
|
||||||
sa.Column("tx_hash", sa.String, nullable=True),
|
|
||||||
sa.Column("method", sa.String, nullable=True),
|
|
||||||
sa.Column("value", sa.BIGINT(), nullable=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade():
|
|
||||||
op.drop_table('gas_cache')
|
|
@ -1,27 +0,0 @@
|
|||||||
# standard imports
|
|
||||||
import logging
|
|
||||||
|
|
||||||
# external imports
|
|
||||||
from sqlalchemy import Column, String, NUMERIC
|
|
||||||
|
|
||||||
# local imports
|
|
||||||
from .base import SessionBase
|
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class GasCache(SessionBase):
|
|
||||||
"""Provides gas budget cache for token operations
|
|
||||||
"""
|
|
||||||
__tablename__ = 'gas_cache'
|
|
||||||
|
|
||||||
address = Column(String())
|
|
||||||
tx_hash = Column(String())
|
|
||||||
method = Column(String())
|
|
||||||
value = Column(NUMERIC())
|
|
||||||
|
|
||||||
def __init__(self, address, method, value, tx_hash):
|
|
||||||
self.address = address
|
|
||||||
self.tx_hash = tx_hash
|
|
||||||
self.method = method
|
|
||||||
self.value = value
|
|
@ -4,12 +4,12 @@ import logging
|
|||||||
|
|
||||||
# third-party imports
|
# third-party imports
|
||||||
from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
|
from sqlalchemy import Column, String, Integer, DateTime, ForeignKey
|
||||||
|
from chainlib.eth.constant import ZERO_ADDRESS
|
||||||
from chainqueue.db.models.tx import TxCache
|
from chainqueue.db.models.tx import TxCache
|
||||||
from chainqueue.db.models.otx import Otx
|
from chainqueue.db.models.otx import Otx
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
from cic_eth.encode import ZERO_ADDRESS_NORMAL
|
|
||||||
|
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ class Lock(SessionBase):
|
|||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def set(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None, tx_hash=None):
|
def set(chain_str, flags, address=ZERO_ADDRESS, session=None, tx_hash=None):
|
||||||
"""Sets flags associated with the given address and chain.
|
"""Sets flags associated with the given address and chain.
|
||||||
|
|
||||||
If a flags entry does not exist it is created.
|
If a flags entry does not exist it is created.
|
||||||
@ -90,7 +90,7 @@ class Lock(SessionBase):
|
|||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def reset(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
|
def reset(chain_str, flags, address=ZERO_ADDRESS, session=None):
|
||||||
"""Resets flags associated with the given address and chain.
|
"""Resets flags associated with the given address and chain.
|
||||||
|
|
||||||
If the resulting flags entry value is 0, the entry will be deleted.
|
If the resulting flags entry value is 0, the entry will be deleted.
|
||||||
@ -134,7 +134,7 @@ class Lock(SessionBase):
|
|||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(chain_str, flags, address=ZERO_ADDRESS_NORMAL, session=None):
|
def check(chain_str, flags, address=ZERO_ADDRESS, session=None):
|
||||||
"""Checks whether all given flags are set for given address and chain.
|
"""Checks whether all given flags are set for given address and chain.
|
||||||
|
|
||||||
Does not validate the address against any other tables or components.
|
Does not validate the address against any other tables or components.
|
||||||
|
@ -12,7 +12,7 @@ from cic_eth.error import (
|
|||||||
IntegrityError,
|
IntegrityError,
|
||||||
)
|
)
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
class Nonce(SessionBase):
|
class Nonce(SessionBase):
|
||||||
@ -21,7 +21,7 @@ class Nonce(SessionBase):
|
|||||||
__tablename__ = 'nonce'
|
__tablename__ = 'nonce'
|
||||||
|
|
||||||
nonce = Column(Integer)
|
nonce = Column(Integer)
|
||||||
address_hex = Column(String(40))
|
address_hex = Column(String(42))
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -25,21 +25,7 @@ class AccountRole(SessionBase):
|
|||||||
address_hex = Column(String(42))
|
address_hex = Column(String(42))
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
# TODO:
|
||||||
def all(session=None):
|
|
||||||
session = SessionBase.bind_session(session)
|
|
||||||
|
|
||||||
pairs = []
|
|
||||||
|
|
||||||
q = session.query(AccountRole.tag, AccountRole.address_hex)
|
|
||||||
for r in q.all():
|
|
||||||
pairs.append((r[1], r[0]),)
|
|
||||||
|
|
||||||
SessionBase.release_session(session)
|
|
||||||
|
|
||||||
return pairs
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_address(tag, session):
|
def get_address(tag, session):
|
||||||
"""Get Ethereum address matching the given tag
|
"""Get Ethereum address matching the given tag
|
||||||
|
@ -1,16 +0,0 @@
|
|||||||
# external imports
|
|
||||||
from chainlib.eth.constant import ZERO_ADDRESS
|
|
||||||
from chainqueue.encode import TxHexNormalizer
|
|
||||||
from chainlib.eth.tx import unpack
|
|
||||||
|
|
||||||
tx_normalize = TxHexNormalizer()
|
|
||||||
|
|
||||||
ZERO_ADDRESS_NORMAL = tx_normalize.wallet_address(ZERO_ADDRESS)
|
|
||||||
|
|
||||||
|
|
||||||
def unpack_normal(signed_tx_bytes, chain_spec):
|
|
||||||
tx = unpack(signed_tx_bytes, chain_spec)
|
|
||||||
tx['hash'] = tx_normalize.tx_hash(tx['hash'])
|
|
||||||
tx['from'] = tx_normalize.wallet_address(tx['from'])
|
|
||||||
tx['to'] = tx_normalize.wallet_address(tx['to'])
|
|
||||||
return tx
|
|
@ -69,12 +69,9 @@ class StatusEnum(enum.IntEnum):
|
|||||||
class LockEnum(enum.IntEnum):
|
class LockEnum(enum.IntEnum):
|
||||||
"""
|
"""
|
||||||
STICKY: When set, reset is not possible
|
STICKY: When set, reset is not possible
|
||||||
INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
|
|
||||||
START: When set, startup is not possible, regardless of state
|
|
||||||
CREATE: Disable creation of accounts
|
CREATE: Disable creation of accounts
|
||||||
SEND: Disable sending to network
|
SEND: Disable sending to network
|
||||||
QUEUE: Disable queueing new or modified transactions
|
QUEUE: Disable queueing new or modified transactions
|
||||||
QUERY: Disable all queue state and transaction queries
|
|
||||||
"""
|
"""
|
||||||
STICKY=1
|
STICKY=1
|
||||||
INIT=2
|
INIT=2
|
||||||
@ -82,8 +79,7 @@ class LockEnum(enum.IntEnum):
|
|||||||
SEND=8
|
SEND=8
|
||||||
QUEUE=16
|
QUEUE=16
|
||||||
QUERY=32
|
QUERY=32
|
||||||
START=int(0x80000000)
|
ALL=int(0xfffffffffffffffe)
|
||||||
ALL=int(0x7ffffffe)
|
|
||||||
|
|
||||||
|
|
||||||
def status_str(v, bits_only=False):
|
def status_str(v, bits_only=False):
|
||||||
|
@ -48,6 +48,8 @@ class RoleMissingError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class IntegrityError(Exception):
|
class IntegrityError(Exception):
|
||||||
"""Exception raised to signal irregularities with deduplication and ordering of tasks
|
"""Exception raised to signal irregularities with deduplication and ordering of tasks
|
||||||
|
|
||||||
@ -64,10 +66,8 @@ class LockedError(Exception):
|
|||||||
|
|
||||||
class SeppukuError(Exception):
|
class SeppukuError(Exception):
|
||||||
"""Exception base class for all errors that should cause system shutdown
|
"""Exception base class for all errors that should cause system shutdown
|
||||||
|
|
||||||
"""
|
"""
|
||||||
def __init__(self, message, lockdown=False):
|
|
||||||
self.message = message
|
|
||||||
self.lockdown = lockdown
|
|
||||||
|
|
||||||
|
|
||||||
class SignerError(SeppukuError):
|
class SignerError(SeppukuError):
|
||||||
@ -85,8 +85,3 @@ class RoleAgencyError(SeppukuError):
|
|||||||
class YouAreBrokeError(Exception):
|
class YouAreBrokeError(Exception):
|
||||||
"""Exception raised when a value transfer is attempted without access to sufficient funds
|
"""Exception raised when a value transfer is attempted without access to sufficient funds
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class TrustError(Exception):
|
|
||||||
"""Exception raised when required trust proofs are missing for a request
|
|
||||||
"""
|
|
||||||
|
@ -13,8 +13,11 @@ from chainlib.eth.sign import (
|
|||||||
new_account,
|
new_account,
|
||||||
sign_message,
|
sign_message,
|
||||||
)
|
)
|
||||||
from chainlib.eth.address import to_checksum_address, is_address
|
from chainlib.eth.address import to_checksum_address
|
||||||
from chainlib.eth.tx import TxFormat
|
from chainlib.eth.tx import (
|
||||||
|
TxFormat,
|
||||||
|
unpack,
|
||||||
|
)
|
||||||
from chainlib.chain import ChainSpec
|
from chainlib.chain import ChainSpec
|
||||||
from chainlib.error import JSONRPCException
|
from chainlib.error import JSONRPCException
|
||||||
from eth_accounts_index.registry import AccountRegistry
|
from eth_accounts_index.registry import AccountRegistry
|
||||||
@ -31,7 +34,6 @@ from cic_eth.eth.gas import (
|
|||||||
from cic_eth.db.models.nonce import Nonce
|
from cic_eth.db.models.nonce import Nonce
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
from cic_eth.db.models.role import AccountRole
|
from cic_eth.db.models.role import AccountRole
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
from cic_eth.error import (
|
from cic_eth.error import (
|
||||||
RoleMissingError,
|
RoleMissingError,
|
||||||
SignerError,
|
SignerError,
|
||||||
@ -47,11 +49,6 @@ from cic_eth.eth.nonce import (
|
|||||||
from cic_eth.queue.tx import (
|
from cic_eth.queue.tx import (
|
||||||
register_tx,
|
register_tx,
|
||||||
)
|
)
|
||||||
from cic_eth.encode import (
|
|
||||||
unpack_normal,
|
|
||||||
ZERO_ADDRESS_NORMAL,
|
|
||||||
tx_normalize,
|
|
||||||
)
|
|
||||||
|
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
@ -86,7 +83,7 @@ def create(self, password, chain_spec_dict):
|
|||||||
# TODO: It seems infeasible that a can be None in any case, verify
|
# TODO: It seems infeasible that a can be None in any case, verify
|
||||||
if a == None:
|
if a == None:
|
||||||
raise SignerError('create account')
|
raise SignerError('create account')
|
||||||
a = tx_normalize.wallet_address(a)
|
|
||||||
logg.debug('created account {}'.format(a))
|
logg.debug('created account {}'.format(a))
|
||||||
|
|
||||||
# Initialize nonce provider record for account
|
# Initialize nonce provider record for account
|
||||||
@ -136,7 +133,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
|
|||||||
# Generate and sign transaction
|
# Generate and sign transaction
|
||||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||||
nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
|
nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
|
||||||
gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
|
gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
|
||||||
account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
||||||
(tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
(tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
||||||
rpc_signer.disconnect()
|
rpc_signer.disconnect()
|
||||||
@ -177,9 +174,6 @@ def gift(self, account_address, chain_spec_dict):
|
|||||||
"""
|
"""
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||||
|
|
||||||
if is_address(account_address):
|
|
||||||
account_address = tx_normalize.wallet_address(account_address)
|
|
||||||
|
|
||||||
logg.debug('gift account address {} to index'.format(account_address))
|
logg.debug('gift account address {} to index'.format(account_address))
|
||||||
queue = self.request.delivery_info.get('routing_key')
|
queue = self.request.delivery_info.get('routing_key')
|
||||||
|
|
||||||
@ -192,7 +186,7 @@ def gift(self, account_address, chain_spec_dict):
|
|||||||
# Generate and sign transaction
|
# Generate and sign transaction
|
||||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||||
nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
|
nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
|
||||||
gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
|
gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
|
||||||
faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
||||||
(tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
(tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
||||||
rpc_signer.disconnect()
|
rpc_signer.disconnect()
|
||||||
@ -253,9 +247,8 @@ def have(self, account, chain_spec_dict):
|
|||||||
|
|
||||||
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
|
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
|
||||||
def set_role(self, tag, address, chain_spec_dict):
|
def set_role(self, tag, address, chain_spec_dict):
|
||||||
if not is_address(address):
|
if not to_checksum_address(address):
|
||||||
raise ValueError('invalid address {}'.format(address))
|
raise ValueError('invalid checksum address {}'.format(address))
|
||||||
address = tx_normalize.wallet_address(address)
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
role = AccountRole.set(tag, address, session=session)
|
role = AccountRole.set(tag, address, session=session)
|
||||||
session.add(role)
|
session.add(role)
|
||||||
@ -266,46 +259,19 @@ def set_role(self, tag, address, chain_spec_dict):
|
|||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
@celery_app.task(bind=True, base=BaseTask)
|
||||||
def role(self, address, chain_spec_dict):
|
def role(self, address, chain_spec_dict):
|
||||||
"""Return account role for address and/or role
|
"""Return account role for address
|
||||||
|
|
||||||
:param account: Account to check
|
:param account: Account to check
|
||||||
:type account: str, 0x-hex
|
:type account: str, 0x-hex
|
||||||
:param chain_spec_dict: Chain spec dict representation
|
:param chain_str: Chain spec string representation
|
||||||
:type chain_spec_dict: dict
|
:type chain_str: str
|
||||||
:returns: Account, or None if not exists
|
:returns: Account, or None if not exists
|
||||||
:rtype: Varies
|
:rtype: Varies
|
||||||
"""
|
"""
|
||||||
session = self.create_session()
|
session = self.create_session()
|
||||||
role_tag = AccountRole.role_for(address, session=session)
|
role_tag = AccountRole.role_for(address, session=session)
|
||||||
session.close()
|
session.close()
|
||||||
return [(address, role_tag,)]
|
return role_tag
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def role_account(self, role_tag, chain_spec_dict):
|
|
||||||
"""Return address for role.
|
|
||||||
|
|
||||||
If the role parameter is None, will return addresses for all roles.
|
|
||||||
|
|
||||||
:param role_tag: Role to match
|
|
||||||
:type role_tag: str
|
|
||||||
:param chain_spec_dict: Chain spec dict representation
|
|
||||||
:type chain_spec_dict: dict
|
|
||||||
:returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
|
|
||||||
:rtype: list
|
|
||||||
"""
|
|
||||||
session = self.create_session()
|
|
||||||
|
|
||||||
pairs = None
|
|
||||||
if role_tag != None:
|
|
||||||
addr = AccountRole.get_address(role_tag, session=session)
|
|
||||||
pairs = [(addr, role_tag,)]
|
|
||||||
else:
|
|
||||||
pairs = AccountRole.all(session=session)
|
|
||||||
|
|
||||||
session.close()
|
|
||||||
|
|
||||||
return pairs
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
|
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
|
||||||
@ -329,19 +295,17 @@ def cache_gift_data(
|
|||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||||
|
|
||||||
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
|
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
|
||||||
tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
|
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
||||||
tx_data = Faucet.parse_give_to_request(tx['data'])
|
tx_data = Faucet.parse_give_to_request(tx['data'])
|
||||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
|
||||||
recipient_address = tx_normalize.wallet_address(tx['to'])
|
|
||||||
|
|
||||||
session = self.create_session()
|
session = self.create_session()
|
||||||
|
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx['hash'],
|
'hash': tx_hash_hex,
|
||||||
'from': sender_address,
|
'from': tx['from'],
|
||||||
'to': recipient_address,
|
'to': tx['to'],
|
||||||
'source_token': ZERO_ADDRESS_NORMAL,
|
'source_token': ZERO_ADDRESS,
|
||||||
'destination_token': ZERO_ADDRESS_NORMAL,
|
'destination_token': ZERO_ADDRESS,
|
||||||
'from_value': 0,
|
'from_value': 0,
|
||||||
'to_value': 0,
|
'to_value': 0,
|
||||||
}
|
}
|
||||||
@ -370,19 +334,17 @@ def cache_account_data(
|
|||||||
:rtype: tuple
|
:rtype: tuple
|
||||||
"""
|
"""
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||||
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
|
tx_signed_raw_bytes = bytes.fromhex(tx_signed_raw_hex[2:])
|
||||||
tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
|
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
||||||
tx_data = AccountsIndex.parse_add_request(tx['data'])
|
tx_data = AccountsIndex.parse_add_request(tx['data'])
|
||||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
|
||||||
recipient_address = tx_normalize.wallet_address(tx['to'])
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx['hash'],
|
'hash': tx_hash_hex,
|
||||||
'from': sender_address,
|
'from': tx['from'],
|
||||||
'to': recipient_address,
|
'to': tx['to'],
|
||||||
'source_token': ZERO_ADDRESS_NORMAL,
|
'source_token': ZERO_ADDRESS,
|
||||||
'destination_token': ZERO_ADDRESS_NORMAL,
|
'destination_token': ZERO_ADDRESS,
|
||||||
'from_value': 0,
|
'from_value': 0,
|
||||||
'to_value': 0,
|
'to_value': 0,
|
||||||
}
|
}
|
||||||
|
@ -10,19 +10,12 @@ from chainlib.eth.tx import (
|
|||||||
TxFormat,
|
TxFormat,
|
||||||
unpack,
|
unpack,
|
||||||
)
|
)
|
||||||
from chainlib.eth.contract import (
|
|
||||||
ABIContractEncoder,
|
|
||||||
)
|
|
||||||
from cic_eth_registry import CICRegistry
|
from cic_eth_registry import CICRegistry
|
||||||
from cic_eth_registry.erc20 import ERC20Token
|
from cic_eth_registry.erc20 import ERC20Token
|
||||||
from hexathon import (
|
from hexathon import strip_0x
|
||||||
strip_0x,
|
|
||||||
add_0x,
|
|
||||||
)
|
|
||||||
from chainqueue.error import NotLocalTxError
|
from chainqueue.error import NotLocalTxError
|
||||||
from eth_erc20 import ERC20
|
from eth_erc20 import ERC20
|
||||||
from chainqueue.sql.tx import cache_tx_dict
|
from chainqueue.sql.tx import cache_tx_dict
|
||||||
from okota.token_index.index import to_identifier
|
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
@ -34,26 +27,24 @@ from cic_eth.error import (
|
|||||||
YouAreBrokeError,
|
YouAreBrokeError,
|
||||||
)
|
)
|
||||||
from cic_eth.queue.tx import register_tx
|
from cic_eth.queue.tx import register_tx
|
||||||
from cic_eth.eth.gas import create_check_gas_task
|
from cic_eth.eth.gas import (
|
||||||
from cic_eth.eth.util import CacheGasOracle
|
create_check_gas_task,
|
||||||
|
MaxGasOracle,
|
||||||
|
)
|
||||||
from cic_eth.ext.address import translate_address
|
from cic_eth.ext.address import translate_address
|
||||||
from cic_eth.task import (
|
from cic_eth.task import (
|
||||||
CriticalSQLAlchemyTask,
|
CriticalSQLAlchemyTask,
|
||||||
CriticalWeb3Task,
|
CriticalWeb3Task,
|
||||||
CriticalSQLAlchemyAndSignerTask,
|
CriticalSQLAlchemyAndSignerTask,
|
||||||
BaseTask,
|
|
||||||
)
|
)
|
||||||
from cic_eth.eth.nonce import CustodialTaskNonceOracle
|
from cic_eth.eth.nonce import CustodialTaskNonceOracle
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
from cic_eth.eth.trust import verify_proofs
|
|
||||||
from cic_eth.error import SignerError
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=CriticalWeb3Task)
|
@celery_app.task(base=CriticalWeb3Task)
|
||||||
def balance(self, tokens, holder_address, chain_spec_dict):
|
def balance(tokens, holder_address, chain_spec_dict):
|
||||||
"""Return token balances for a list of tokens for given address
|
"""Return token balances for a list of tokens for given address
|
||||||
|
|
||||||
:param tokens: Token addresses
|
:param tokens: Token addresses
|
||||||
@ -71,10 +62,8 @@ def balance(self, tokens, holder_address, chain_spec_dict):
|
|||||||
|
|
||||||
for t in tokens:
|
for t in tokens:
|
||||||
address = t['address']
|
address = t['address']
|
||||||
logg.debug('address {} {}'.format(address, holder_address))
|
token = ERC20Token(chain_spec, rpc, address)
|
||||||
gas_oracle = self.create_gas_oracle(rpc, min_price=self.min_fee_price)
|
c = ERC20(chain_spec)
|
||||||
token = ERC20Token(chain_spec, rpc, add_0x(address))
|
|
||||||
c = ERC20(chain_spec, gas_oracle=gas_oracle)
|
|
||||||
o = c.balance_of(address, holder_address, sender_address=caller_address)
|
o = c.balance_of(address, holder_address, sender_address=caller_address)
|
||||||
r = rpc.do(o)
|
r = rpc.do(o)
|
||||||
t['balance_network'] = c.parse_balance(r)
|
t['balance_network'] = c.parse_balance(r)
|
||||||
@ -157,12 +146,8 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
|
|||||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||||
|
|
||||||
session = self.create_session()
|
session = self.create_session()
|
||||||
|
|
||||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||||
enc = ABIContractEncoder()
|
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||||
enc.method('transferFrom')
|
|
||||||
method = enc.get()
|
|
||||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
|
|
||||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||||
try:
|
try:
|
||||||
(tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
(tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||||
@ -232,12 +217,8 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
|
|||||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||||
|
|
||||||
session = self.create_session()
|
session = self.create_session()
|
||||||
|
|
||||||
enc = ABIContractEncoder()
|
|
||||||
enc.method('transfer')
|
|
||||||
method = enc.get()
|
|
||||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
|
|
||||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||||
|
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||||
try:
|
try:
|
||||||
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||||
@ -305,12 +286,8 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
|
|||||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||||
|
|
||||||
session = self.create_session()
|
session = self.create_session()
|
||||||
|
|
||||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||||
enc = ABIContractEncoder()
|
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||||
enc.method('approve')
|
|
||||||
method = enc.get()
|
|
||||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
|
|
||||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||||
try:
|
try:
|
||||||
(tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
|
(tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||||
@ -394,20 +371,16 @@ def cache_transfer_data(
|
|||||||
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
||||||
|
|
||||||
tx_data = ERC20.parse_transfer_request(tx['data'])
|
tx_data = ERC20.parse_transfer_request(tx['data'])
|
||||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
recipient_address = tx_data[0]
|
||||||
recipient_address = tx_normalize.wallet_address(tx_data[0])
|
|
||||||
token_value = tx_data[1]
|
token_value = tx_data[1]
|
||||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
|
||||||
destination_token_address = source_token_address
|
|
||||||
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx_hash_hex,
|
'hash': tx_hash_hex,
|
||||||
'from': sender_address,
|
'from': tx['from'],
|
||||||
'to': recipient_address,
|
'to': recipient_address,
|
||||||
'source_token': source_token_address,
|
'source_token': tx['to'],
|
||||||
'destination_token': destination_token_address,
|
'destination_token': tx['to'],
|
||||||
'from_value': token_value,
|
'from_value': token_value,
|
||||||
'to_value': token_value,
|
'to_value': token_value,
|
||||||
}
|
}
|
||||||
@ -439,16 +412,14 @@ def cache_transfer_from_data(
|
|||||||
spender_address = tx_data[0]
|
spender_address = tx_data[0]
|
||||||
recipient_address = tx_data[1]
|
recipient_address = tx_data[1]
|
||||||
token_value = tx_data[2]
|
token_value = tx_data[2]
|
||||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
|
||||||
destination_token_address = source_token_address
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx_hash_hex,
|
'hash': tx_hash_hex,
|
||||||
'from': tx['from'],
|
'from': tx['from'],
|
||||||
'to': recipient_address,
|
'to': recipient_address,
|
||||||
'source_token': source_token_address,
|
'source_token': tx['to'],
|
||||||
'destination_token': destination_token_address,
|
'destination_token': tx['to'],
|
||||||
'from_value': token_value,
|
'from_value': token_value,
|
||||||
'to_value': token_value,
|
'to_value': token_value,
|
||||||
}
|
}
|
||||||
@ -477,19 +448,16 @@ def cache_approve_data(
|
|||||||
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
||||||
|
|
||||||
tx_data = ERC20.parse_approve_request(tx['data'])
|
tx_data = ERC20.parse_approve_request(tx['data'])
|
||||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
recipient_address = tx_data[0]
|
||||||
recipient_address = tx_normalize.wallet_address(tx_data[0])
|
|
||||||
token_value = tx_data[1]
|
token_value = tx_data[1]
|
||||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
|
||||||
destination_token_address = source_token_address
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx_hash_hex,
|
'hash': tx_hash_hex,
|
||||||
'from': sender_address,
|
'from': tx['from'],
|
||||||
'to': recipient_address,
|
'to': recipient_address,
|
||||||
'source_token': source_token_address,
|
'source_token': tx['to'],
|
||||||
'destination_token': destination_token_address,
|
'destination_token': tx['to'],
|
||||||
'from_value': token_value,
|
'from_value': token_value,
|
||||||
'to_value': token_value,
|
'to_value': token_value,
|
||||||
}
|
}
|
||||||
@ -497,69 +465,3 @@ def cache_approve_data(
|
|||||||
session.close()
|
session.close()
|
||||||
return (tx_hash_hex, cache_id)
|
return (tx_hash_hex, cache_id)
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def token_info(self, tokens, chain_spec_dict, proofs=[]):
|
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
|
||||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
|
||||||
|
|
||||||
i = 0
|
|
||||||
|
|
||||||
for token in tokens:
|
|
||||||
result_data = []
|
|
||||||
token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
|
|
||||||
token_chain_object.load(rpc)
|
|
||||||
|
|
||||||
token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
|
|
||||||
token_proofs = [token_symbol_proof_hex]
|
|
||||||
if len(proofs) > 0:
|
|
||||||
token_proofs += proofs[i]
|
|
||||||
|
|
||||||
tokens[i] = {
|
|
||||||
'decimals': token_chain_object.decimals,
|
|
||||||
'name': token_chain_object.name,
|
|
||||||
'symbol': token_chain_object.symbol,
|
|
||||||
'address': tx_normalize.executable_address(token_chain_object.address),
|
|
||||||
'proofs': token_proofs,
|
|
||||||
'converters': tokens[i]['converters'],
|
|
||||||
}
|
|
||||||
i += 1
|
|
||||||
|
|
||||||
return tokens
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
|
|
||||||
queue = self.request.delivery_info.get('routing_key')
|
|
||||||
|
|
||||||
for token in tokens:
|
|
||||||
s = celery.signature(
|
|
||||||
'cic_eth.eth.trust.verify_proofs',
|
|
||||||
[
|
|
||||||
token,
|
|
||||||
token['address'],
|
|
||||||
token['proofs'],
|
|
||||||
chain_spec_dict,
|
|
||||||
success_callback,
|
|
||||||
error_callback,
|
|
||||||
],
|
|
||||||
queue=queue,
|
|
||||||
)
|
|
||||||
|
|
||||||
if success_callback != None:
|
|
||||||
s.link(success_callback)
|
|
||||||
if error_callback != None:
|
|
||||||
s.on_error(error_callback)
|
|
||||||
s.apply_async()
|
|
||||||
|
|
||||||
return tokens
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def default_token(self):
|
|
||||||
return {
|
|
||||||
'symbol': self.default_token_symbol,
|
|
||||||
'address': self.default_token_address,
|
|
||||||
'name': self.default_token_name,
|
|
||||||
'decimals': self.default_token_decimals,
|
|
||||||
}
|
|
||||||
|
@ -3,17 +3,10 @@ import logging
|
|||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import celery
|
import celery
|
||||||
from hexathon import (
|
from hexathon import strip_0x
|
||||||
strip_0x,
|
from chainlib.eth.constant import ZERO_ADDRESS
|
||||||
add_0x,
|
|
||||||
)
|
|
||||||
#from chainlib.eth.constant import ZERO_ADDRESS
|
|
||||||
from chainlib.chain import ChainSpec
|
from chainlib.chain import ChainSpec
|
||||||
from chainlib.eth.address import (
|
from chainlib.eth.address import is_checksum_address
|
||||||
is_checksum_address,
|
|
||||||
to_checksum_address,
|
|
||||||
is_address
|
|
||||||
)
|
|
||||||
from chainlib.connection import RPCConnection
|
from chainlib.connection import RPCConnection
|
||||||
from chainqueue.db.enum import StatusBits
|
from chainqueue.db.enum import StatusBits
|
||||||
from chainqueue.sql.tx import cache_tx_dict
|
from chainqueue.sql.tx import cache_tx_dict
|
||||||
@ -28,6 +21,7 @@ from chainlib.eth.error import (
|
|||||||
from chainlib.eth.tx import (
|
from chainlib.eth.tx import (
|
||||||
TxFactory,
|
TxFactory,
|
||||||
TxFormat,
|
TxFormat,
|
||||||
|
unpack,
|
||||||
)
|
)
|
||||||
from chainlib.eth.contract import (
|
from chainlib.eth.contract import (
|
||||||
abi_decode_single,
|
abi_decode_single,
|
||||||
@ -41,7 +35,6 @@ from chainqueue.db.models.tx import TxCache
|
|||||||
from chainqueue.db.models.otx import Otx
|
from chainqueue.db.models.otx import Otx
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.db.models.gas_cache import GasCache
|
|
||||||
from cic_eth.db.models.role import AccountRole
|
from cic_eth.db.models.role import AccountRole
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
from cic_eth.error import (
|
from cic_eth.error import (
|
||||||
@ -52,7 +45,6 @@ from cic_eth.eth.nonce import CustodialTaskNonceOracle
|
|||||||
from cic_eth.queue.tx import (
|
from cic_eth.queue.tx import (
|
||||||
queue_create,
|
queue_create,
|
||||||
register_tx,
|
register_tx,
|
||||||
unpack,
|
|
||||||
)
|
)
|
||||||
from cic_eth.queue.query import get_tx
|
from cic_eth.queue.query import get_tx
|
||||||
from cic_eth.task import (
|
from cic_eth.task import (
|
||||||
@ -61,61 +53,17 @@ from cic_eth.task import (
|
|||||||
CriticalSQLAlchemyAndSignerTask,
|
CriticalSQLAlchemyAndSignerTask,
|
||||||
CriticalWeb3AndSignerTask,
|
CriticalWeb3AndSignerTask,
|
||||||
)
|
)
|
||||||
from cic_eth.encode import (
|
|
||||||
tx_normalize,
|
|
||||||
ZERO_ADDRESS_NORMAL,
|
|
||||||
unpack_normal,
|
|
||||||
)
|
|
||||||
from cic_eth.error import SeppukuError
|
|
||||||
from cic_eth.eth.util import MAXIMUM_FEE_UNITS
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
|
MAXIMUM_FEE_UNITS = 8000000
|
||||||
|
|
||||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
class MaxGasOracle:
|
||||||
def apply_gas_value_cache(address, method, value, tx_hash):
|
|
||||||
return apply_gas_value_cache_local(address, method, value, tx_hash)
|
|
||||||
|
|
||||||
|
def gas(code=None):
|
||||||
def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
|
return MAXIMUM_FEE_UNITS
|
||||||
address = tx_normalize.executable_address(address)
|
|
||||||
tx_hash = tx_normalize.tx_hash(tx_hash)
|
|
||||||
value = int(value)
|
|
||||||
|
|
||||||
session = SessionBase.bind_session(session)
|
|
||||||
q = session.query(GasCache)
|
|
||||||
q = q.filter(GasCache.address==address)
|
|
||||||
q = q.filter(GasCache.method==method)
|
|
||||||
o = q.first()
|
|
||||||
|
|
||||||
if o == None:
|
|
||||||
o = GasCache(address, method, value, tx_hash)
|
|
||||||
elif value > o.value:
|
|
||||||
o.value = value
|
|
||||||
o.tx_hash = strip_0x(tx_hash)
|
|
||||||
|
|
||||||
session.add(o)
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
SessionBase.release_session(session)
|
|
||||||
|
|
||||||
|
|
||||||
def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
|
|
||||||
if rpc == None:
|
|
||||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
|
||||||
o = balance(add_0x(address))
|
|
||||||
r = rpc.do(o)
|
|
||||||
try:
|
|
||||||
r = int(r)
|
|
||||||
except ValueError:
|
|
||||||
r = strip_0x(r)
|
|
||||||
r = int(r, 16)
|
|
||||||
logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
|
|
||||||
if r < min_gas:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
|
def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
|
||||||
@ -182,16 +130,16 @@ def cache_gas_data(
|
|||||||
"""
|
"""
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||||
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
|
tx_signed_raw_bytes = bytes.fromhex(strip_0x(tx_signed_raw_hex))
|
||||||
tx = unpack_normal(tx_signed_raw_bytes, chain_spec)
|
tx = unpack(tx_signed_raw_bytes, chain_spec)
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
|
|
||||||
tx_dict = {
|
tx_dict = {
|
||||||
'hash': tx['hash'],
|
'hash': tx_hash_hex,
|
||||||
'from': tx['from'],
|
'from': tx['from'],
|
||||||
'to': tx['to'],
|
'to': tx['to'],
|
||||||
'source_token': ZERO_ADDRESS_NORMAL,
|
'source_token': ZERO_ADDRESS,
|
||||||
'destination_token': ZERO_ADDRESS_NORMAL,
|
'destination_token': ZERO_ADDRESS,
|
||||||
'from_value': tx['value'],
|
'from_value': tx['value'],
|
||||||
'to_value': tx['value'],
|
'to_value': tx['value'],
|
||||||
}
|
}
|
||||||
@ -202,7 +150,7 @@ def cache_gas_data(
|
|||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
|
@celery_app.task(bind=True, throws=(OutOfGasError), base=CriticalSQLAlchemyAndWeb3Task)
|
||||||
def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
|
def check_gas(self, tx_hashes, chain_spec_dict, txs=[], address=None, gas_required=MAXIMUM_FEE_UNITS):
|
||||||
"""Check the gas level of the sender address of a transaction.
|
"""Check the gas level of the sender address of a transaction.
|
||||||
|
|
||||||
If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
|
If the account balance is not sufficient for the required gas, gas refill is requested and OutOfGasError raiser.
|
||||||
@ -222,23 +170,8 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
|
|||||||
:return: Signed raw transaction data list
|
:return: Signed raw transaction data list
|
||||||
:rtype: param txs, unchanged
|
:rtype: param txs, unchanged
|
||||||
"""
|
"""
|
||||||
rpc_format_address = None
|
|
||||||
if address != None:
|
|
||||||
if not is_address(address):
|
|
||||||
raise ValueError('invalid address {}'.format(address))
|
|
||||||
address = tx_normalize.wallet_address(address)
|
|
||||||
address = add_0x(address)
|
|
||||||
|
|
||||||
tx_hashes = []
|
|
||||||
txs = []
|
|
||||||
for tx_hash in tx_hashes_hex:
|
|
||||||
tx_hash = tx_normalize.tx_hash(tx_hash)
|
|
||||||
tx_hashes.append(tx_hash)
|
|
||||||
for tx in txs_hex:
|
|
||||||
tx = tx_normalize.tx_wire(tx)
|
|
||||||
txs.append(tx)
|
|
||||||
|
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||||
|
logg.debug('txs {} tx_hashes {}'.format(txs, tx_hashes))
|
||||||
|
|
||||||
addresspass = None
|
addresspass = None
|
||||||
if len(txs) == 0:
|
if len(txs) == 0:
|
||||||
@ -254,7 +187,8 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
|
|||||||
raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
|
raise ValueError('txs passed to check gas must all have same sender; had {} got {}'.format(address, tx['from']))
|
||||||
addresspass.append(address)
|
addresspass.append(address)
|
||||||
|
|
||||||
rpc_format_address = add_0x(to_checksum_address(address))
|
if not is_checksum_address(address):
|
||||||
|
raise ValueError('invalid address {}'.format(address))
|
||||||
|
|
||||||
queue = self.request.delivery_info.get('routing_key')
|
queue = self.request.delivery_info.get('routing_key')
|
||||||
|
|
||||||
@ -262,7 +196,7 @@ def check_gas(self, tx_hashes_hex, chain_spec_dict, txs_hex=[], address=None, ga
|
|||||||
|
|
||||||
gas_balance = 0
|
gas_balance = 0
|
||||||
try:
|
try:
|
||||||
o = balance(rpc_format_address)
|
o = balance(address)
|
||||||
r = conn.do(o)
|
r = conn.do(o)
|
||||||
conn.disconnect()
|
conn.disconnect()
|
||||||
gas_balance = abi_decode_single(ABIContractType.UINT256, r)
|
gas_balance = abi_decode_single(ABIContractType.UINT256, r)
|
||||||
@ -370,7 +304,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
|
|||||||
# Determine value of gas tokens to send
|
# Determine value of gas tokens to send
|
||||||
# if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
|
# if an uncompleted gas refill for the same recipient already exists, we still need to spend the nonce
|
||||||
# however, we will perform a 0-value transaction instead
|
# however, we will perform a 0-value transaction instead
|
||||||
recipient_address = tx_normalize.wallet_address(recipient_address)
|
|
||||||
zero_amount = False
|
zero_amount = False
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
|
status_filter = StatusBits.FINAL | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
|
||||||
@ -397,13 +330,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
|
|||||||
# set up evm RPC connection
|
# set up evm RPC connection
|
||||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||||
|
|
||||||
# check the gas balance of the gifter
|
|
||||||
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
|
|
||||||
raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
|
|
||||||
|
|
||||||
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
|
|
||||||
logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
|
|
||||||
|
|
||||||
# set up transaction builder
|
# set up transaction builder
|
||||||
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
|
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
|
||||||
gas_oracle = self.create_gas_oracle(rpc)
|
gas_oracle = self.create_gas_oracle(rpc)
|
||||||
@ -452,7 +378,6 @@ def resend_with_higher_gas(self, txold_hash_hex, chain_spec_dict, gas=None, defa
|
|||||||
:returns: Transaction hash
|
:returns: Transaction hash
|
||||||
:rtype: str, 0x-hex
|
:rtype: str, 0x-hex
|
||||||
"""
|
"""
|
||||||
txold_hash_hex = tx_normalize.tx_hash(txold_hash_hex)
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
|
|
||||||
otx = Otx.load(txold_hash_hex, session)
|
otx = Otx.load(txold_hash_hex, session)
|
||||||
|
@ -2,9 +2,8 @@
|
|||||||
from chainlib.eth.constant import ZERO_ADDRESS
|
from chainlib.eth.constant import ZERO_ADDRESS
|
||||||
from chainlib.status import Status as TxStatus
|
from chainlib.status import Status as TxStatus
|
||||||
from cic_eth_registry.erc20 import ERC20Token
|
from cic_eth_registry.erc20 import ERC20Token
|
||||||
from hexathon import add_0x
|
|
||||||
|
|
||||||
# local impor:ts
|
# local imports
|
||||||
from cic_eth.ext.address import translate_address
|
from cic_eth.ext.address import translate_address
|
||||||
|
|
||||||
|
|
||||||
@ -45,8 +44,8 @@ class ExtendedTx:
|
|||||||
destination = source
|
destination = source
|
||||||
if destination_value == None:
|
if destination_value == None:
|
||||||
destination_value = source_value
|
destination_value = source_value
|
||||||
st = ERC20Token(self.chain_spec, self.rpc, add_0x(source))
|
st = ERC20Token(self.chain_spec, self.rpc, source)
|
||||||
dt = ERC20Token(self.chain_spec, self.rpc, add_0x(destination))
|
dt = ERC20Token(self.chain_spec, self.rpc, destination)
|
||||||
self.source_token = source
|
self.source_token = source
|
||||||
self.source_token_symbol = st.symbol
|
self.source_token_symbol = st.symbol
|
||||||
self.source_token_name = st.name
|
self.source_token_name = st.name
|
||||||
|
@ -3,12 +3,11 @@ import logging
|
|||||||
|
|
||||||
# external imports
|
# external imports
|
||||||
import celery
|
import celery
|
||||||
from chainlib.eth.address import is_checksum_address, is_address, strip_0x
|
from chainlib.eth.address import is_checksum_address
|
||||||
|
|
||||||
# local imports
|
# local imports
|
||||||
from cic_eth.db.models.role import AccountRole
|
from cic_eth.db.models.role import AccountRole
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
from cic_eth.task import CriticalSQLAlchemyTask
|
from cic_eth.task import CriticalSQLAlchemyTask
|
||||||
from cic_eth.db.models.nonce import (
|
from cic_eth.db.models.nonce import (
|
||||||
Nonce,
|
Nonce,
|
||||||
@ -43,8 +42,7 @@ class CustodialTaskNonceOracle():
|
|||||||
:returns: Nonce
|
:returns: Nonce
|
||||||
:rtype: number
|
:rtype: number
|
||||||
"""
|
"""
|
||||||
address = tx_normalize.wallet_address(self.address)
|
r = NonceReservation.release(self.address, self.uuid, session=self.session)
|
||||||
r = NonceReservation.release(address, self.uuid, session=self.session)
|
|
||||||
return r[1]
|
return r[1]
|
||||||
|
|
||||||
|
|
||||||
@ -60,18 +58,17 @@ def reserve_nonce(self, chained_input, chain_spec_dict, signer_address=None):
|
|||||||
address = chained_input
|
address = chained_input
|
||||||
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
|
logg.debug('non-explicit address for reserve nonce, using arg head {}'.format(chained_input))
|
||||||
else:
|
else:
|
||||||
if is_address(signer_address):
|
if is_checksum_address(signer_address):
|
||||||
address = signer_address
|
address = signer_address
|
||||||
logg.debug('explicit address for reserve nonce {}'.format(signer_address))
|
logg.debug('explicit address for reserve nonce {}'.format(signer_address))
|
||||||
else:
|
else:
|
||||||
address = AccountRole.get_address(signer_address, session=session)
|
address = AccountRole.get_address(signer_address, session=session)
|
||||||
logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
|
logg.debug('role for reserve nonce {} -> {}'.format(signer_address, address))
|
||||||
|
|
||||||
if not is_address(address):
|
if not is_checksum_address(address):
|
||||||
raise ValueError('invalid result when resolving address for nonce {}'.format(address))
|
raise ValueError('invalid result when resolving address for nonce {}'.format(address))
|
||||||
|
|
||||||
root_id = self.request.root_id
|
root_id = self.request.root_id
|
||||||
address = tx_normalize.wallet_address(address)
|
|
||||||
r = NonceReservation.next(address, root_id, session=session)
|
r = NonceReservation.next(address, root_id, session=session)
|
||||||
logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
|
logg.debug('nonce {} reserved for address {} task {}'.format(r[1], address, r[0]))
|
||||||
|
|
||||||
|
@ -1,77 +0,0 @@
|
|||||||
# standard imports
|
|
||||||
import logging
|
|
||||||
|
|
||||||
# external imports
|
|
||||||
import celery
|
|
||||||
from eth_address_declarator import Declarator
|
|
||||||
from chainlib.connection import RPCConnection
|
|
||||||
from chainlib.chain import ChainSpec
|
|
||||||
from cic_eth.db.models.role import AccountRole
|
|
||||||
from cic_eth_registry import CICRegistry
|
|
||||||
from hexathon import strip_0x
|
|
||||||
|
|
||||||
# local imports
|
|
||||||
from cic_eth.task import BaseTask
|
|
||||||
from cic_eth.error import TrustError
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
|
||||||
logg = logging.getLogger()
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
|
|
||||||
proof = strip_0x(proof)
|
|
||||||
|
|
||||||
proofs = []
|
|
||||||
|
|
||||||
logg.debug('proof count {}'.format(len(proofs)))
|
|
||||||
if len(proofs) == 0:
|
|
||||||
logg.debug('error {}'.format(len(proofs)))
|
|
||||||
raise TrustError('foo')
|
|
||||||
|
|
||||||
return (chained_input, (proof, proofs))
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=BaseTask)
|
|
||||||
def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
|
|
||||||
queue = self.request.delivery_info.get('routing_key')
|
|
||||||
|
|
||||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
|
||||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
|
||||||
|
|
||||||
session = self.create_session()
|
|
||||||
sender_address = AccountRole.get_address('DEFAULT', session)
|
|
||||||
|
|
||||||
registry = CICRegistry(chain_spec, rpc)
|
|
||||||
declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
|
|
||||||
|
|
||||||
declarator = Declarator(chain_spec)
|
|
||||||
|
|
||||||
have_proofs = {}
|
|
||||||
|
|
||||||
for proof in proofs:
|
|
||||||
|
|
||||||
proof = strip_0x(proof)
|
|
||||||
|
|
||||||
have_proofs[proof] = []
|
|
||||||
|
|
||||||
for trusted_address in self.trusted_addresses:
|
|
||||||
o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
|
|
||||||
r = rpc.do(o)
|
|
||||||
declarations = declarator.parse_declaration(r)
|
|
||||||
logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
|
|
||||||
|
|
||||||
for declaration in declarations:
|
|
||||||
declaration = strip_0x(declaration)
|
|
||||||
if declaration == proof:
|
|
||||||
logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
|
|
||||||
have_proofs[proof].append(trusted_address)
|
|
||||||
|
|
||||||
out_proofs = {}
|
|
||||||
for proof in have_proofs.keys():
|
|
||||||
if len(have_proofs[proof]) == 0:
|
|
||||||
logg.error('missing signer for proof {} subject {}'.format(proof, subject))
|
|
||||||
raise TrustError((subject, proof,))
|
|
||||||
out_proofs[proof] = have_proofs[proof]
|
|
||||||
|
|
||||||
return (chained_input, out_proofs)
|
|
@ -1,54 +0,0 @@
|
|||||||
# standard imports
|
|
||||||
import logging
|
|
||||||
|
|
||||||
# external imports
|
|
||||||
from chainlib.eth.gas import RPCGasOracle
|
|
||||||
from hexathon import strip_0x
|
|
||||||
|
|
||||||
# local imports
|
|
||||||
from cic_eth.db.models.gas_cache import GasCache
|
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
from cic_eth.db.models.base import SessionBase
|
|
||||||
|
|
||||||
MAXIMUM_FEE_UNITS = 8000000
|
|
||||||
|
|
||||||
logg = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class MaxGasOracle(RPCGasOracle):
|
|
||||||
|
|
||||||
def get_fee_units(self, code=None):
|
|
||||||
return MAXIMUM_FEE_UNITS
|
|
||||||
|
|
||||||
|
|
||||||
class CacheGasOracle(MaxGasOracle):
|
|
||||||
"""Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
|
|
||||||
|
|
||||||
:todo: instead of max units, connect a pluggable gas heuristics engine.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
|
|
||||||
super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
|
|
||||||
self.value = None
|
|
||||||
self.address = address
|
|
||||||
self.method = method
|
|
||||||
|
|
||||||
address = tx_normalize.executable_address(address)
|
|
||||||
session = SessionBase.bind_session(session)
|
|
||||||
q = session.query(GasCache)
|
|
||||||
q = q.filter(GasCache.address==address)
|
|
||||||
if method != None:
|
|
||||||
method = strip_0x(method)
|
|
||||||
q = q.filter(GasCache.method==method)
|
|
||||||
o = q.first()
|
|
||||||
if o != None:
|
|
||||||
self.value = int(o.value)
|
|
||||||
|
|
||||||
SessionBase.release_session(session)
|
|
||||||
|
|
||||||
|
|
||||||
def get_fee_units(self, code=None):
|
|
||||||
if self.value != None:
|
|
||||||
logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
|
|
||||||
return self.value
|
|
||||||
return super(CacheGasOracle, self).get_fee_units(code=code)
|
|
@ -32,7 +32,6 @@ from potaahto.symbols import snake_and_camel
|
|||||||
from cic_eth.queue.time import tx_times
|
from cic_eth.queue.time import tx_times
|
||||||
from cic_eth.task import BaseTask
|
from cic_eth.task import BaseTask
|
||||||
from cic_eth.db.models.base import SessionBase
|
from cic_eth.db.models.base import SessionBase
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
@ -135,7 +134,7 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
|
|||||||
tx_address = transfer_data[0]
|
tx_address = transfer_data[0]
|
||||||
tx_token_value = transfer_data[1]
|
tx_token_value = transfer_data[1]
|
||||||
|
|
||||||
if tx_normalize.wallet_address(address) == tx_normalize.wallet_address(tx_address):
|
if address == tx_address:
|
||||||
status = StatusEnum.SENT
|
status = StatusEnum.SENT
|
||||||
try:
|
try:
|
||||||
o = receipt(tx['hash'])
|
o = receipt(tx['hash'])
|
||||||
@ -153,8 +152,8 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
|
|||||||
times = tx_times(tx['hash'], chain_spec)
|
times = tx_times(tx['hash'], chain_spec)
|
||||||
tx_r = {
|
tx_r = {
|
||||||
'hash': tx['hash'],
|
'hash': tx['hash'],
|
||||||
'sender': tx_normalize.wallet_address(tx['from']),
|
'sender': tx['from'],
|
||||||
'recipient': tx_normalize.wallet_address(tx_address),
|
'recipient': tx_address,
|
||||||
'source_value': tx_token_value,
|
'source_value': tx_token_value,
|
||||||
'destination_value': tx_token_value,
|
'destination_value': tx_token_value,
|
||||||
'source_token': tx['to'],
|
'source_token': tx['to'],
|
||||||
@ -165,12 +164,12 @@ def list_tx_by_bloom(self, bloomspec, address, chain_spec_dict):
|
|||||||
tx_r['date_created'] = times['queue']
|
tx_r['date_created'] = times['queue']
|
||||||
else:
|
else:
|
||||||
tx_r['date_created'] = times['network']
|
tx_r['date_created'] = times['network']
|
||||||
txs[strip_0x(tx['hash'])] = tx_r
|
txs[tx['hash']] = tx_r
|
||||||
break
|
break
|
||||||
|
|
||||||
return txs
|
return txs
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# TODO: Surely it must be possible to optimize this
|
# TODO: Surely it must be possible to optimize this
|
||||||
# TODO: DRY this with callback filter in cic_eth/runnable/manager
|
# TODO: DRY this with callback filter in cic_eth/runnable/manager
|
||||||
# TODO: Remove redundant fields from end representation (timestamp, tx_hash)
|
# TODO: Remove redundant fields from end representation (timestamp, tx_hash)
|
||||||
@ -231,8 +230,6 @@ def tx_collate(self, tx_batches, chain_spec_dict, offset, limit, newest_first=Tr
|
|||||||
except UnknownContractError:
|
except UnknownContractError:
|
||||||
logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
|
logg.error('verify failed on tx {}, skipping'.format(tx['hash']))
|
||||||
continue
|
continue
|
||||||
tx['recipient'] = tx_normalize.wallet_address(tx['recipient'])
|
|
||||||
tx['sender'] = tx_normalize.wallet_address(tx['sender'])
|
|
||||||
txs.append(tx)
|
txs.append(tx)
|
||||||
|
|
||||||
return txs
|
return txs
|
||||||
|
@ -4,21 +4,18 @@ import tempfile
|
|||||||
import logging
|
import logging
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
# local imports
|
# local impors
|
||||||
from cic_eth.task import BaseTask
|
from cic_eth.task import BaseTask
|
||||||
|
|
||||||
#logg = logging.getLogger(__name__)
|
#logg = logging.getLogger(__name__)
|
||||||
logg = logging.getLogger()
|
logg = logging.getLogger()
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='function')
|
@pytest.fixture(scope='function')
|
||||||
def init_celery_tasks(
|
def init_celery_tasks(
|
||||||
contract_roles,
|
contract_roles,
|
||||||
):
|
):
|
||||||
BaseTask.call_address = contract_roles['DEFAULT']
|
BaseTask.call_address = contract_roles['DEFAULT']
|
||||||
BaseTask.trusted_addresses = [
|
|
||||||
contract_roles['TRUSTED_DECLARATOR'],
|
|
||||||
contract_roles['CONTRACT_DEPLOYER'],
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
# celery fixtures
|
# celery fixtures
|
||||||
@ -41,7 +38,6 @@ def celery_includes():
|
|||||||
'cic_eth.callbacks.noop',
|
'cic_eth.callbacks.noop',
|
||||||
'cic_eth.callbacks.http',
|
'cic_eth.callbacks.http',
|
||||||
'cic_eth.pytest.mock.filter',
|
'cic_eth.pytest.mock.filter',
|
||||||
'cic_eth.pytest.mock.callback',
|
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@ -8,14 +8,15 @@ import confini
|
|||||||
|
|
||||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||||
root_dir = os.path.dirname(os.path.dirname(script_dir))
|
root_dir = os.path.dirname(os.path.dirname(script_dir))
|
||||||
config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
|
|
||||||
logg = logging.getLogger(__name__)
|
logg = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session')
|
@pytest.fixture(scope='session')
|
||||||
def load_config():
|
def load_config():
|
||||||
override_config_dir = os.path.join(root_dir, 'config', 'test')
|
config_dir = os.environ.get('CONFINI_DIR')
|
||||||
conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
|
if config_dir == None:
|
||||||
|
config_dir = os.path.join(root_dir, 'config/test')
|
||||||
|
conf = confini.Config(config_dir, 'CICTEST')
|
||||||
conf.process()
|
conf.process()
|
||||||
logg.debug('config {}'.format(conf))
|
logg.debug('config {}'.format(conf))
|
||||||
return conf
|
return conf
|
||||||
|
@ -1,2 +1 @@
|
|||||||
from .filter import *
|
from .filter import *
|
||||||
from .callback import *
|
|
||||||
|
@ -1,38 +0,0 @@
|
|||||||
# standard imports
|
|
||||||
import os
|
|
||||||
import logging
|
|
||||||
import mmap
|
|
||||||
|
|
||||||
# standard imports
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
# external imports
|
|
||||||
import celery
|
|
||||||
|
|
||||||
#logg = logging.getLogger(__name__)
|
|
||||||
logg = logging.getLogger()
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
|
||||||
|
|
||||||
|
|
||||||
class CallbackTask(celery.Task):
|
|
||||||
|
|
||||||
mmap_path = tempfile.mkdtemp()
|
|
||||||
|
|
||||||
|
|
||||||
@celery_app.task(bind=True, base=CallbackTask)
|
|
||||||
def test_callback(self, a, b, c):
|
|
||||||
s = 'ok'
|
|
||||||
if c > 0:
|
|
||||||
s = 'err'
|
|
||||||
|
|
||||||
fp = os.path.join(self.mmap_path, b)
|
|
||||||
f = open(fp, 'wb+')
|
|
||||||
f.write(b'\x00')
|
|
||||||
f.seek(0)
|
|
||||||
m = mmap.mmap(f.fileno(), length=1)
|
|
||||||
m.write(c.to_bytes(1, 'big'))
|
|
||||||
m.close()
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))
|
|
@ -15,7 +15,6 @@ from chainqueue.db.enum import (
|
|||||||
# local imports
|
# local imports
|
||||||
from cic_eth.db import SessionBase
|
from cic_eth.db import SessionBase
|
||||||
from cic_eth.task import CriticalSQLAlchemyTask
|
from cic_eth.task import CriticalSQLAlchemyTask
|
||||||
from cic_eth.encode import tx_normalize
|
|
||||||
|
|
||||||
celery_app = celery.current_app
|
celery_app = celery.current_app
|
||||||
|
|
||||||
@ -23,9 +22,6 @@ logg = logging.getLogger()
|
|||||||
|
|
||||||
|
|
||||||
def __balance_outgoing_compatible(token_address, holder_address):
|
def __balance_outgoing_compatible(token_address, holder_address):
|
||||||
token_address = tx_normalize.executable_address(token_address)
|
|
||||||
holder_address = tx_normalize.wallet_address(holder_address)
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
q = session.query(TxCache.from_value)
|
q = session.query(TxCache.from_value)
|
||||||
q = q.join(Otx)
|
q = q.join(Otx)
|
||||||
@ -62,9 +58,6 @@ def balance_outgoing(tokens, holder_address, chain_spec_dict):
|
|||||||
|
|
||||||
|
|
||||||
def __balance_incoming_compatible(token_address, receiver_address):
|
def __balance_incoming_compatible(token_address, receiver_address):
|
||||||
token_address = tx_normalize.executable_address(token_address)
|
|
||||||
receiver_address = tx_normalize.wallet_address(receiver_address)
|
|
||||||
|
|
||||||
session = SessionBase.create_session()
|
session = SessionBase.create_session()
|
||||||
q = session.query(TxCache.to_value)
|
q = session.query(TxCache.to_value)
|
||||||
q = q.join(Otx)
|
q = q.join(Otx)
|
||||||
@ -72,7 +65,7 @@ def __balance_incoming_compatible(token_address, receiver_address):
|
|||||||
status_compare = dead()
|
status_compare = dead()
|
||||||
q = q.filter(Otx.status.op('&')(status_compare)==0)
|
q = q.filter(Otx.status.op('&')(status_compare)==0)
|
||||||
# TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed.
|
# TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed.
|
||||||
#q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
|
q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
|
||||||
q = q.filter(TxCache.destination_token_address==token_address)
|
q = q.filter(TxCache.destination_token_address==token_address)
|
||||||
delta = 0
|
delta = 0
|
||||||
for r in q.all():
|
for r in q.all():
|
||||||
@ -117,7 +110,7 @@ def assemble_balances(balances_collection):
|
|||||||
logg.debug('received collection {}'.format(balances_collection))
|
logg.debug('received collection {}'.format(balances_collection))
|
||||||
for c in balances_collection:
|
for c in balances_collection:
|
||||||
for b in c:
|
for b in c:
|
||||||
address = tx_normalize.executable_address(b['address'])
|
address = b['address']
|
||||||
if tokens.get(address) == None:
|
if tokens.get(address) == None:
|
||||||
tokens[address] = {
|
tokens[address] = {
|
||||||
'address': address,
|
'address': address,
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user