Compare commits
1 Commits
master
...
bvander/re
| Author | SHA1 | Date | |
|---|---|---|---|
| 2537fa8920 |
1
.gitignore
vendored
1
.gitignore
vendored
@ -15,3 +15,4 @@ build/
|
||||
.idea
|
||||
**/.vim
|
||||
**/*secret.yaml
|
||||
.env
|
||||
|
||||
@ -10,7 +10,6 @@ include:
|
||||
#- local: 'apps/data-seeding/.gitlab-ci.yml'
|
||||
|
||||
stages:
|
||||
- version
|
||||
- build
|
||||
- test
|
||||
- deploy
|
||||
@ -21,41 +20,9 @@ variables:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
COMPOSE_DOCKER_CLI_BUILD: "1"
|
||||
CI_DEBUG_TRACE: "true"
|
||||
SEMVERBOT_VERSION: "0.2.0"
|
||||
|
||||
#before_script:
|
||||
# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
|
||||
version:
|
||||
#image: python:3.7-stretch
|
||||
image: registry.gitlab.com/grassrootseconomics/cic-base-images/ci-version:b01318ae
|
||||
stage: version
|
||||
tags:
|
||||
- integration
|
||||
script:
|
||||
- mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts && chmod 644 ~/.ssh/known_hosts
|
||||
- eval $(ssh-agent -s)
|
||||
- ssh-add <(echo "$SSH_PRIVATE_KEY")
|
||||
- git remote set-url origin git@gitlab.com:grassrootseconomics/cic-internal-integration.git
|
||||
- export TAG=$(sbot predict version -m auto)
|
||||
- |
|
||||
if [[ -z $TAG ]]
|
||||
then
|
||||
echo "tag could not be set $@"
|
||||
exit 1
|
||||
fi
|
||||
- echo $TAG > version
|
||||
- git tag -a v$TAG -m "ci tagged"
|
||||
- git push origin v$TAG
|
||||
artifacts:
|
||||
paths:
|
||||
- version
|
||||
rules:
|
||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
||||
when: always
|
||||
- if: $CI_COMMIT_REF_NAME == "master"
|
||||
when: always
|
||||
before_script:
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
|
||||
# runs on protected branches and pushes to repo
|
||||
build-push:
|
||||
@ -63,17 +30,12 @@ build-push:
|
||||
tags:
|
||||
- integration
|
||||
#script:
|
||||
# - TAG=$CI_Cbefore_script:
|
||||
before_script:
|
||||
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
|
||||
# - TAG=$CI_COMMIT_REF_SLUG-$CI_COMMIT_SHORT_SHA sh ./scripts/build-push.sh
|
||||
script:
|
||||
- TAG=latest ./scripts/build-push.sh
|
||||
- TAG=$(cat ./version) ./scripts/build-push.sh
|
||||
- TAG=latest sh ./scripts/build-push.sh
|
||||
rules:
|
||||
- if: $CI_COMMIT_REF_PROTECTED == "true"
|
||||
when: always
|
||||
- if: $CI_COMMIT_REF_NAME == "master"
|
||||
when: always
|
||||
|
||||
deploy-dev:
|
||||
stage: deploy
|
||||
|
||||
@ -1,44 +0,0 @@
|
||||
<!---
|
||||
Please read this!
|
||||
|
||||
Before opening a new issue, make sure to search for keywords in the issues
|
||||
filtered by the "bug" label:
|
||||
|
||||
- https://gitlab.com/groups/grassrootseconomics/-/issues?scope=all&state=all&label_name[]=bug
|
||||
|
||||
and verify the issue you're about to submit isn't a duplicate.
|
||||
--->
|
||||
|
||||
### Summary
|
||||
|
||||
<!-- Summarize the bug encountered concisely. -->
|
||||
|
||||
### Steps to reproduce
|
||||
|
||||
<!-- Describe how one can reproduce the issue - this is very important. Please use an ordered list. -->
|
||||
|
||||
### Example Project
|
||||
|
||||
<!-- If possible, please create an example project here on GitLab.com that exhibits the problematic
|
||||
behavior, and link to it here in the bug report. If you are using an older version of GitLab, this
|
||||
will also determine whether the bug is fixed in a more recent version. -->
|
||||
|
||||
### What is the current *bug* behavior?
|
||||
|
||||
<!-- Describe what actually happens. -->
|
||||
|
||||
### What is the expected *correct* behavior?
|
||||
|
||||
<!-- Describe what you should see instead. -->
|
||||
|
||||
### Relevant logs and/or screenshots
|
||||
|
||||
<!-- Paste any relevant logs - please use code blocks (```) to format console output, logs, and code
|
||||
as it's tough to read otherwise. -->
|
||||
|
||||
|
||||
### Possible fixes
|
||||
|
||||
<!-- If you can, link to the line of code that might be responsible for the problem. -->
|
||||
|
||||
/label ~"bug"
|
||||
@ -1,16 +0,0 @@
|
||||
[git]
|
||||
|
||||
[git.config]
|
||||
email = "semverbot@grassroots.org"
|
||||
name = "semvervot"
|
||||
|
||||
[git.tags]
|
||||
prefix = "v"
|
||||
|
||||
[semver]
|
||||
mode = "git-commit"
|
||||
|
||||
[semver.detection]
|
||||
patch = ["fix", "[fix]", "patch", "[patch]"]
|
||||
minor = ["minor", "[minor]", "feat", "[feat]", "release", "[release]", "bump", "[bump]"]
|
||||
major = ["BREAKING CHANGE"]
|
||||
@ -1,83 +0,0 @@
|
||||
# Contributor Covenant Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as
|
||||
contributors and maintainers pledge to make participation in our project and
|
||||
our community a harassment-free experience for everyone, regardless of age, body
|
||||
size, disability, ethnicity, sex characteristics, gender identity and expression,
|
||||
level of experience, education, socio-economic status, nationality, personal
|
||||
appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment
|
||||
include:
|
||||
|
||||
* Using welcoming and inclusive language
|
||||
* Being respectful of differing viewpoints and experiences
|
||||
* Gracefully accepting constructive criticism
|
||||
* Focusing on what is best for the community
|
||||
* Showing empathy towards other community members
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
||||
advances
|
||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
||||
* Public or private harassment
|
||||
* Publishing others' private information, such as a physical or electronic
|
||||
address, without explicit permission
|
||||
* Other conduct which could reasonably be considered inappropriate in a
|
||||
professional setting
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable
|
||||
behavior and are expected to take appropriate and fair corrective action in
|
||||
response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or
|
||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
||||
permanently any contributor for other behaviors that they deem inappropriate,
|
||||
threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies within all project spaces, and it also applies when
|
||||
an individual is representing the project or its community in public spaces.
|
||||
Examples of representing a project or community include using an official
|
||||
project e-mail address, posting via an official social media account, or acting
|
||||
as an appointed representative at an online or offline event. Representation of
|
||||
a project may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
|
||||
complaints will be reviewed and investigated and will result in a response that
|
||||
is deemed necessary and appropriate to the circumstances. The project team is
|
||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
||||
Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
||||
faith may face temporary or permanent repercussions as determined by other
|
||||
members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see
|
||||
https://www.contributor-covenant.org/faq
|
||||
|
||||
|
||||
|
||||
|
||||
...Try to keep in mind the immortal
|
||||
words of Bill and Ted, "Be excellent to each other."
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
Hello and welcome to the CIC Stack repository. Targeted for use with the ethereum virtual machine and a ussd capable telecom provider.
|
||||
|
||||
__To request a change to the code please fork this repository and sumbit a merge request.__
|
||||
|
||||
__If there is a Grassroots Economics Kanban Issue please include that in our MR it will help us track contributions. Karibu sana!__
|
||||
|
||||
__Visit the Development Kanban board here: https://gitlab.com/grassrootseconomics/cic-internal-integration/-/boards/2419764__
|
||||
|
||||
__Ask a question in our dev chat:__
|
||||
|
||||
[Mattermost](https://chat.grassrootseconomics.net/cic/channels/dev)
|
||||
|
||||
[Discord](https://discord.gg/XWunwAsX)
|
||||
|
||||
[Matrix, IRC soon?]
|
||||
|
||||
@ -1,117 +0,0 @@
|
||||
# CORE TEAM CONTRIBUTION GUIDE
|
||||
|
||||
# 1. Transparency
|
||||
|
||||
1.1 Use work logs for reflection of work done, aswell as telling your peers about changes that may affect their own tasks
|
||||
|
||||
1.2 A work log SHOULD be submitted after a "unit of work" is complete.
|
||||
|
||||
1.2.1 A "unit of work" should not span more than one full day's worth of work.
|
||||
|
||||
1.2.2 A "unit of work" should be small enough that the log entries give useful insight.
|
||||
|
||||
1.3 Individual logs are reviewed in weekly meetings
|
||||
|
||||
<!--1.4 Bullet point list of topics and one or more sub-points describing each item in short sentences, eg;
|
||||
|
||||
```
|
||||
- Core
|
||||
* fixed foo
|
||||
* fixed bar
|
||||
- Frontend
|
||||
* connected bar to baz
|
||||
|
||||
```-->
|
||||
|
||||
1.4 Work log format is defined in []()
|
||||
|
||||
1.5 Link to issue/MR in bullet point where appropriate
|
||||
|
||||
1.6
|
||||
|
||||
|
||||
# 2. Code hygiene
|
||||
|
||||
2.1 Keep function names and variable names short
|
||||
|
||||
2.2 Keep code files, functions and test fixtures short
|
||||
|
||||
2.3 The less magic the better. Recombinable and replaceable is king
|
||||
|
||||
2.4 Group imports by `standard`, `external`, `local`, `test` - in that order
|
||||
|
||||
2.5 Only auto-import when necessary, and always with a minimum of side-effects
|
||||
|
||||
2.6 Use custom errors. Let them bubble up
|
||||
|
||||
2.7 No logs in tight loops
|
||||
|
||||
2.8 Keep executable main routine minimal. Pass variables (do not use globals) in main business logic function
|
||||
|
||||
2.9 Test coverage MUST be kept higher than 90% after changes
|
||||
|
||||
2.10 Docstrings. Always. Always!
|
||||
|
||||
|
||||
# 3. Versioning
|
||||
|
||||
3.1 Use [Semantic Versioning](https://semver.org/)
|
||||
|
||||
3.2 When merging code, explicit dependencies SHOULD NOT use pre-release version
|
||||
|
||||
|
||||
# 4. Issues
|
||||
|
||||
4.1 Issue title should use [Convention Commit structure](https://www.conventionalcommits.org/en/v1.0.0-beta.2/)
|
||||
|
||||
4.2 Issues need proper problem statement
|
||||
|
||||
4.2.1. What is the current state
|
||||
|
||||
4.2.2. If current state is not behaving as expected, what was the expected state
|
||||
|
||||
4.2.3. What is the desired new state.
|
||||
|
||||
4.3 Issues need proper resolution statement
|
||||
|
||||
4.3.1. Bullet point list of short sentences describing practical steps to reach desired state
|
||||
|
||||
4.3.2. Builet point list of external resources informing the issue and resolution
|
||||
|
||||
4.4 Tasks needs to be appropriately labelled using GROUP labels.
|
||||
|
||||
|
||||
# 5. Code submission
|
||||
|
||||
5.1 A branch and new MR is always created BEFORE THE WORK STARTS
|
||||
|
||||
5.2 An MR should solve ONE SINGLE PART of a problem
|
||||
|
||||
5.3 Every MR should have at least ONE ISSUE associated with it. Ideally issue can be closed when MR is merged
|
||||
|
||||
5.4 MRs should not be open for more than one week (during normal operation periods)
|
||||
|
||||
5.5 MR should ideally not be longer than 400 lines of changes of logic
|
||||
|
||||
5.6 MRs that MOVE or DELETE code should not CHANGE that same code in a single MR. Scope MOVEs and DELETEs in separate commits (or even better, separate MRs) for transparency
|
||||
|
||||
|
||||
# 6. Code reviews
|
||||
|
||||
6.1 At least one peer review before merge
|
||||
|
||||
6.2 If MR is too long, evaluate whether this affects the quality of the review negatively. If it does, expect to be asked to split it up
|
||||
|
||||
6.3 Evaluate changes against associated issues' problem statement and proposed resolution steps. If there is a mismatch, either MR needs to change or issue needs to be amended accordingly
|
||||
|
||||
6.4 Make sure all technical debt introduced by MR is documented in issues. Add them according to criteria in section ISSUES if not
|
||||
|
||||
6.5 If CI is not working, reviewer MUST make sure code builds and runs
|
||||
|
||||
6.6 Behave!
|
||||
|
||||
6.6.1 Don't be a jerk
|
||||
|
||||
6.6.2 Don't block needlessly
|
||||
|
||||
6.6.3 Say please
|
||||
37
README.md
37
README.md
@ -1,9 +1,11 @@
|
||||
# Community Inclusion Currency Stack (CIC Stack)
|
||||
|
||||
A custodial evm wallet for executing transactions via USSD
|
||||
# cic-internal-integration
|
||||
|
||||
## Getting started
|
||||
|
||||
### Preperation
|
||||
|
||||
Copy the .env_sample file to .env and populate with appropriate env vars
|
||||
|
||||
This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
|
||||
|
||||
```
|
||||
@ -11,9 +13,32 @@ export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
export DOCKER_BUILDKIT=1
|
||||
```
|
||||
|
||||
To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
|
||||
start services, database, redis and local ethereum node
|
||||
```
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
## Documentation
|
||||
Run app/contract-migration to deploy contracts
|
||||
```
|
||||
RUN_MASK=3 docker-compose up contract-migration
|
||||
```
|
||||
|
||||
[https://docs.grassecon.org/software/](https://docs.grassecon.org/software/)
|
||||
stop cluster
|
||||
```
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
stop cluster and delete data
|
||||
```
|
||||
docker-compose down -v --remove-orphans
|
||||
```
|
||||
|
||||
rebuild an images
|
||||
```
|
||||
docker-compose up --build <service_name>
|
||||
```
|
||||
|
||||
to delete the buildkit cache
|
||||
```
|
||||
docker builder prune --filter type=exec.cachemount
|
||||
```
|
||||
|
||||
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
3
apps/cic-base-os/aux/wait-for-it/.gitignore
vendored
@ -1,3 +0,0 @@
|
||||
**/*.pyc
|
||||
.pydevproject
|
||||
/vendor/
|
||||
@ -1,7 +0,0 @@
|
||||
language: python
|
||||
python:
|
||||
- "2.7"
|
||||
|
||||
script:
|
||||
- python test/wait-for-it.py
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
The MIT License (MIT)
|
||||
Copyright (c) 2016 Giles Hall
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||
of the Software, and to permit persons to whom the Software is furnished to do
|
||||
so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@ -1,75 +0,0 @@
|
||||
# wait-for-it
|
||||
|
||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
||||
host and TCP port. It is useful for synchronizing the spin-up of
|
||||
interdependent services, such as linked docker containers. Since it is a pure
|
||||
bash script, it does not have any external dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
```text
|
||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
||||
and if it is available, echo the message `google is up`.
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
||||
the timeout value to 0 will disable the timeout:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
The subcommand will be executed regardless if the service is up or not. If you
|
||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
||||
fail:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
||||
```
|
||||
|
||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
||||
and determine how to proceed:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
$ echo $?
|
||||
0
|
||||
$ ./wait-for-it.sh www.google.com:81
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
||||
$ echo $?
|
||||
124
|
||||
```
|
||||
|
||||
## Community
|
||||
|
||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
||||
@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Use this script to test if a given TCP host/port are available
|
||||
|
||||
WAITFORIT_cmdname=${0##*/}
|
||||
|
||||
echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
|
||||
|
||||
usage()
|
||||
{
|
||||
cat << USAGE >&2
|
||||
Usage:
|
||||
$WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for()
|
||||
{
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
else
|
||||
echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
|
||||
fi
|
||||
WAITFORIT_start_ts=$(date +%s)
|
||||
while :
|
||||
do
|
||||
if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
|
||||
nc -z $WAITFORIT_HOST $WAITFORIT_PORT
|
||||
WAITFORIT_result=$?
|
||||
else
|
||||
(echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
|
||||
WAITFORIT_result=$?
|
||||
fi
|
||||
if [[ $WAITFORIT_result -eq 0 ]]; then
|
||||
WAITFORIT_end_ts=$(date +%s)
|
||||
echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return $WAITFORIT_result
|
||||
}
|
||||
|
||||
wait_for_wrapper()
|
||||
{
|
||||
# In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
|
||||
if [[ $WAITFORIT_QUIET -eq 1 ]]; then
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
else
|
||||
timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
|
||||
fi
|
||||
WAITFORIT_PID=$!
|
||||
trap "kill -INT -$WAITFORIT_PID" INT
|
||||
wait $WAITFORIT_PID
|
||||
WAITFORIT_RESULT=$?
|
||||
if [[ $WAITFORIT_RESULT -ne 0 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
|
||||
fi
|
||||
return $WAITFORIT_RESULT
|
||||
}
|
||||
|
||||
# process arguments
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
case "$1" in
|
||||
*:* )
|
||||
WAITFORIT_hostport=(${1//:/ })
|
||||
WAITFORIT_HOST=${WAITFORIT_hostport[0]}
|
||||
WAITFORIT_PORT=${WAITFORIT_hostport[1]}
|
||||
shift 1
|
||||
;;
|
||||
--child)
|
||||
WAITFORIT_CHILD=1
|
||||
shift 1
|
||||
;;
|
||||
-q | --quiet)
|
||||
WAITFORIT_QUIET=1
|
||||
shift 1
|
||||
;;
|
||||
-s | --strict)
|
||||
WAITFORIT_STRICT=1
|
||||
shift 1
|
||||
;;
|
||||
-h)
|
||||
WAITFORIT_HOST="$2"
|
||||
if [[ $WAITFORIT_HOST == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--host=*)
|
||||
WAITFORIT_HOST="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-p)
|
||||
WAITFORIT_PORT="$2"
|
||||
if [[ $WAITFORIT_PORT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--port=*)
|
||||
WAITFORIT_PORT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
-t)
|
||||
WAITFORIT_TIMEOUT="$2"
|
||||
if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
|
||||
shift 2
|
||||
;;
|
||||
--timeout=*)
|
||||
WAITFORIT_TIMEOUT="${1#*=}"
|
||||
shift 1
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
WAITFORIT_CLI=("$@")
|
||||
break
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
echoerr "Unknown argument: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
|
||||
echoerr "Error: you need to provide a host and port to test."
|
||||
usage
|
||||
fi
|
||||
|
||||
WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
|
||||
WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
|
||||
WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
|
||||
WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
|
||||
|
||||
# Check to see if timeout is from busybox?
|
||||
WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
|
||||
WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
|
||||
|
||||
WAITFORIT_BUSYTIMEFLAG=""
|
||||
if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
|
||||
WAITFORIT_ISBUSY=1
|
||||
# Check if busybox timeout uses -t flag
|
||||
# (recent Alpine versions don't support -t anymore)
|
||||
if timeout &>/dev/stdout | grep -q -e '-t '; then
|
||||
WAITFORIT_BUSYTIMEFLAG="-t"
|
||||
fi
|
||||
else
|
||||
WAITFORIT_ISBUSY=0
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CHILD -gt 0 ]]; then
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
exit $WAITFORIT_RESULT
|
||||
else
|
||||
if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
|
||||
wait_for_wrapper
|
||||
WAITFORIT_RESULT=$?
|
||||
else
|
||||
wait_for
|
||||
WAITFORIT_RESULT=$?
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ $WAITFORIT_CLI != "" ]]; then
|
||||
if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
|
||||
echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
||||
exec "${WAITFORIT_CLI[@]}"
|
||||
else
|
||||
exit $WAITFORIT_RESULT
|
||||
fi
|
||||
@ -1 +1 @@
|
||||
include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*
|
||||
include *requirements.txt cic_cache/data/config/*
|
||||
|
||||
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
3
apps/cic-cache/aux/wait-for-it/.gitignore
vendored
@ -1,3 +0,0 @@
|
||||
**/*.pyc
|
||||
.pydevproject
|
||||
/vendor/
|
||||
@ -1,7 +0,0 @@
|
||||
language: python
|
||||
python:
|
||||
- "2.7"
|
||||
|
||||
script:
|
||||
- python test/wait-for-it.py
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
# wait-for-it
|
||||
|
||||
`wait-for-it.sh` is a pure bash script that will wait on the availability of a
|
||||
host and TCP port. It is useful for synchronizing the spin-up of
|
||||
interdependent services, such as linked docker containers. Since it is a pure
|
||||
bash script, it does not have any external dependencies.
|
||||
|
||||
## Usage
|
||||
|
||||
```text
|
||||
wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
|
||||
-h HOST | --host=HOST Host or IP under test
|
||||
-p PORT | --port=PORT TCP port under test
|
||||
Alternatively, you specify the host and port as host:port
|
||||
-s | --strict Only execute subcommand if the test succeeds
|
||||
-q | --quiet Don't output any status messages
|
||||
-t TIMEOUT | --timeout=TIMEOUT
|
||||
Timeout in seconds, zero for no timeout
|
||||
-- COMMAND ARGS Execute command with args after the test finishes
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
For example, let's test to see if we can access port 80 on `www.google.com`,
|
||||
and if it is available, echo the message `google is up`.
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
You can set your own timeout with the `-t` or `--timeout=` option. Setting
|
||||
the timeout value to 0 will disable the timeout:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
|
||||
wait-for-it.sh: waiting for www.google.com:80 without a timeout
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
google is up
|
||||
```
|
||||
|
||||
The subcommand will be executed regardless if the service is up or not. If you
|
||||
wish to execute the subcommand only if the service is up, add the `--strict`
|
||||
argument. In this example, we will test port 81 on `www.google.com` which will
|
||||
fail:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
|
||||
wait-for-it.sh: waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
|
||||
wait-for-it.sh: strict mode, refusing to execute subprocess
|
||||
```
|
||||
|
||||
If you don't want to execute a subcommand, leave off the `--` argument. This
|
||||
way, you can test the exit condition of `wait-for-it.sh` in your own scripts,
|
||||
and determine how to proceed:
|
||||
|
||||
```text
|
||||
$ ./wait-for-it.sh www.google.com:80
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:80
|
||||
wait-for-it.sh: www.google.com:80 is available after 0 seconds
|
||||
$ echo $?
|
||||
0
|
||||
$ ./wait-for-it.sh www.google.com:81
|
||||
wait-for-it.sh: waiting 15 seconds for www.google.com:81
|
||||
wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
|
||||
$ echo $?
|
||||
124
|
||||
```
|
||||
|
||||
## Community
|
||||
|
||||
*Debian*: There is a [Debian package](https://tracker.debian.org/pkg/wait-for-it).
|
||||
@ -14,7 +14,7 @@ class ArgumentParser(BaseArgumentParser):
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-cache', help='Task queue')
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
||||
self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
|
||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
trust_address =
|
||||
health_modules =
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||
|
||||
@ -3,8 +3,7 @@ engine =
|
||||
driver =
|
||||
host =
|
||||
port =
|
||||
#name = cic-cache
|
||||
prefix =
|
||||
name = cic-cache
|
||||
user =
|
||||
password =
|
||||
debug = 0
|
||||
|
||||
@ -9,26 +9,21 @@ from .list import (
|
||||
tag_transaction,
|
||||
add_tag,
|
||||
)
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def dsn_from_config(config, name):
|
||||
def dsn_from_config(config):
|
||||
scheme = config.get('DATABASE_ENGINE')
|
||||
if config.get('DATABASE_DRIVER') != None:
|
||||
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
|
||||
|
||||
database_name = name
|
||||
if config.get('DATABASE_PREFIX'):
|
||||
database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
|
||||
dsn = ''
|
||||
if config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
SessionBase.poolable = False
|
||||
dsn = '{}:///{}'.format(
|
||||
scheme,
|
||||
database_name,
|
||||
config.get('DATABASE_NAME'),
|
||||
)
|
||||
|
||||
else:
|
||||
@ -38,7 +33,7 @@ def dsn_from_config(config, name):
|
||||
config.get('DATABASE_PASSWORD'),
|
||||
config.get('DATABASE_HOST'),
|
||||
config.get('DATABASE_PORT'),
|
||||
database_name,
|
||||
config.get('DATABASE_NAME'),
|
||||
)
|
||||
logg.debug('parsed dsn from config: {}'.format(dsn))
|
||||
return dsn
|
||||
|
||||
@ -5,11 +5,7 @@ import re
|
||||
import base64
|
||||
|
||||
# external imports
|
||||
from hexathon import (
|
||||
add_0x,
|
||||
strip_0x,
|
||||
)
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
from hexathon import add_0x
|
||||
|
||||
# local imports
|
||||
from cic_cache.cache import (
|
||||
@ -20,72 +16,27 @@ from cic_cache.cache import (
|
||||
logg = logging.getLogger(__name__)
|
||||
#logg = logging.getLogger()
|
||||
|
||||
re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
||||
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
|
||||
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
||||
re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
|
||||
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
|
||||
re_default_limit = r'/defaultlimit/?'
|
||||
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
|
||||
|
||||
DEFAULT_LIMIT = 100
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
def parse_query_account(r):
|
||||
address = strip_0x(r[1])
|
||||
#address = tx_normalize.wallet_address(address)
|
||||
limit = DEFAULT_LIMIT
|
||||
g = r.groups()
|
||||
if len(g) > 3:
|
||||
limit = int(r[4])
|
||||
if limit == 0:
|
||||
limit = DEFAULT_LIMIT
|
||||
offset = 0
|
||||
if len(g) > 4:
|
||||
offset = int(r[6])
|
||||
|
||||
logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
|
||||
|
||||
return (address, offset, limit,)
|
||||
|
||||
|
||||
# r is an re.Match
|
||||
def parse_query_any(r):
|
||||
limit = DEFAULT_LIMIT
|
||||
offset = 0
|
||||
block_offset = None
|
||||
block_end = None
|
||||
if r.lastindex != None:
|
||||
if r.lastindex > 0:
|
||||
limit = int(r[1])
|
||||
if r.lastindex > 1:
|
||||
offset = int(r[2])
|
||||
if r.lastindex > 2:
|
||||
block_offset = int(r[3])
|
||||
if r.lastindex > 3:
|
||||
block_end = int(r[4])
|
||||
if block_end < block_offset:
|
||||
raise ValueError('cart before the horse, dude')
|
||||
|
||||
logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
|
||||
|
||||
return (offset, limit, block_offset, block_end,)
|
||||
|
||||
|
||||
def process_default_limit(session, env):
|
||||
r = re.match(re_default_limit, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
|
||||
return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_account_bloom(session, env):
|
||||
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match account bloom')
|
||||
|
||||
(address, offset, limit,) = parse_query_account(r)
|
||||
address = r[1]
|
||||
if r[2] == None:
|
||||
address = add_0x(address)
|
||||
offset = 0
|
||||
if r.lastindex > 2:
|
||||
offset = r[4]
|
||||
limit = DEFAULT_LIMIT
|
||||
if r.lastindex > 4:
|
||||
limit = r[6]
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
|
||||
@ -108,9 +59,13 @@ def process_transactions_all_bloom(session, env):
|
||||
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match all bloom')
|
||||
|
||||
(limit, offset, block_offset, block_end,) = parse_query_any(r)
|
||||
offset = DEFAULT_LIMIT
|
||||
if r.lastindex > 0:
|
||||
offset = r[1]
|
||||
limit = 0
|
||||
if r.lastindex > 1:
|
||||
limit = r[2]
|
||||
|
||||
c = BloomCache(session)
|
||||
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
|
||||
@ -133,16 +88,17 @@ def process_transactions_all_data(session, env):
|
||||
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||
# return None
|
||||
logg.debug('match all data')
|
||||
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||
return None
|
||||
|
||||
logg.debug('got data request {}'.format(env))
|
||||
|
||||
(offset, limit, block_offset, block_end) = parse_query_any(r)
|
||||
block_offset = r[1]
|
||||
block_end = r[2]
|
||||
if int(r[2]) < int(r[1]):
|
||||
raise ValueError('cart before the horse, dude')
|
||||
|
||||
c = DataCache(session)
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
|
||||
|
||||
for r in tx_cache:
|
||||
r['date_block'] = r['date_block'].timestamp()
|
||||
@ -157,30 +113,3 @@ def process_transactions_all_data(session, env):
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
|
||||
def process_transactions_account_data(session, env):
|
||||
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
|
||||
if not r:
|
||||
return None
|
||||
logg.debug('match account data')
|
||||
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
|
||||
# return None
|
||||
|
||||
(address, offset, limit,) = parse_query_account(r)
|
||||
|
||||
c = DataCache(session)
|
||||
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
|
||||
|
||||
for r in tx_cache:
|
||||
r['date_block'] = r['date_block'].timestamp()
|
||||
|
||||
o = {
|
||||
'low': lowest_block,
|
||||
'high': highest_block,
|
||||
'data': tx_cache,
|
||||
}
|
||||
|
||||
j = json.dumps(o)
|
||||
|
||||
return ('application/json', j.encode('utf-8'),)
|
||||
|
||||
@ -8,33 +8,42 @@ import base64
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.runnable.daemons.query import (
|
||||
process_default_limit,
|
||||
process_transactions_account_bloom,
|
||||
process_transactions_account_data,
|
||||
process_transactions_all_bloom,
|
||||
process_transactions_all_data,
|
||||
)
|
||||
import cic_cache.cli
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||
|
||||
arg_flags = cic_cache.cli.argflag_std_read
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
args = argparser.parse_args()
|
||||
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config:\n{}'.format(config))
|
||||
|
||||
dsn = dsn_from_config(config)
|
||||
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
|
||||
|
||||
|
||||
@ -46,11 +55,9 @@ def application(env, start_response):
|
||||
|
||||
session = SessionBase.create_session()
|
||||
for handler in [
|
||||
process_transactions_account_data,
|
||||
process_transactions_account_bloom,
|
||||
process_transactions_all_data,
|
||||
process_transactions_all_bloom,
|
||||
process_default_limit,
|
||||
process_transactions_account_bloom,
|
||||
]:
|
||||
r = None
|
||||
try:
|
||||
|
||||
@ -3,14 +3,12 @@ import logging
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import tempfile
|
||||
|
||||
# third-party imports
|
||||
import celery
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
import cic_cache.cli
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.tasks.tx import *
|
||||
@ -18,20 +16,35 @@ from cic_cache.tasks.tx import *
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
# process args
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
local_arg_flags = cic_cache.cli.argflag_local_task
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('-q', type=str, default='cic-cache', help='queue name for worker tasks')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
|
||||
args = argparser.parse_args()
|
||||
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
dsn = dsn_from_config(config)
|
||||
SessionBase.connect(dsn)
|
||||
|
||||
# verify database connection with minimal sanity query
|
||||
#session = SessionBase.create_session()
|
||||
#session.execute('select version_num from alembic_version')
|
||||
#session.close()
|
||||
|
||||
# set up celery
|
||||
current_app = celery.Celery(__name__)
|
||||
|
||||
@ -74,9 +87,9 @@ def main():
|
||||
elif args.v:
|
||||
argv.append('--loglevel=INFO')
|
||||
argv.append('-Q')
|
||||
argv.append(config.get('CELERY_QUEUE'))
|
||||
argv.append(args.q)
|
||||
argv.append('-n')
|
||||
argv.append(config.get('CELERY_QUEUE'))
|
||||
argv.append(args.q)
|
||||
|
||||
current_app.worker_main(argv)
|
||||
|
||||
|
||||
@ -40,7 +40,7 @@ logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
# process args
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
arg_flags = cic_cache.cli.argflag_std_read
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
@ -50,7 +50,7 @@ args = argparser.parse_args()
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
dsn = dsn_from_config(config)
|
||||
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
|
||||
|
||||
# set up rpc
|
||||
@ -95,10 +95,10 @@ def main():
|
||||
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
||||
|
||||
if len(syncer_backends) == 0:
|
||||
initial_block_start = int(config.get('SYNCER_OFFSET'))
|
||||
initial_block_offset = int(block_offset)
|
||||
initial_block_start = config.get('SYNCER_OFFSET')
|
||||
initial_block_offset = block_offset
|
||||
if config.get('SYNCER_NO_HISTORY'):
|
||||
initial_block_start = initial_block_offset
|
||||
initial_block_start = block_offset
|
||||
initial_block_offset += 1
|
||||
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
||||
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
||||
|
||||
@ -5,7 +5,7 @@ version = (
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
'alpha.3',
|
||||
'alpha.2',
|
||||
)
|
||||
|
||||
version_object = semver.VersionInfo(
|
||||
|
||||
3
apps/cic-cache/config/celery.ini
Normal file
3
apps/cic-cache/config/celery.ini
Normal file
@ -0,0 +1,3 @@
|
||||
[celery]
|
||||
broker_url = redis:///
|
||||
result_url = redis:///
|
||||
3
apps/cic-cache/config/cic.ini
Normal file
3
apps/cic-cache/config/cic.ini
Normal file
@ -0,0 +1,3 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
trust_address =
|
||||
9
apps/cic-cache/config/database.ini
Normal file
9
apps/cic-cache/config/database.ini
Normal file
@ -0,0 +1,9 @@
|
||||
[database]
|
||||
NAME=cic_cache
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=0
|
||||
3
apps/cic-cache/config/docker/celery.ini
Normal file
3
apps/cic-cache/config/docker/celery.ini
Normal file
@ -0,0 +1,3 @@
|
||||
[celery]
|
||||
broker_url = redis://localhost:63379
|
||||
result_url = redis://localhost:63379
|
||||
3
apps/cic-cache/config/docker/cic.ini
Normal file
3
apps/cic-cache/config/docker/cic.ini
Normal file
@ -0,0 +1,3 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C
|
||||
9
apps/cic-cache/config/docker/database.ini
Normal file
9
apps/cic-cache/config/docker/database.ini
Normal file
@ -0,0 +1,9 @@
|
||||
[database]
|
||||
NAME=cic_cache
|
||||
USER=grassroots
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
PORT=63432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=0
|
||||
4
apps/cic-cache/config/docker/syncer.ini
Normal file
4
apps/cic-cache/config/docker/syncer.ini
Normal file
@ -0,0 +1,4 @@
|
||||
[syncer]
|
||||
loop_interval = 1
|
||||
offset = 0
|
||||
no_history = 0
|
||||
2
apps/cic-cache/config/test/bancor.ini
Normal file
2
apps/cic-cache/config/test/bancor.ini
Normal file
@ -0,0 +1,2 @@
|
||||
[bancor]
|
||||
dir =
|
||||
@ -1,3 +1,4 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
chain_spec =
|
||||
trust_address =
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
[database]
|
||||
PREFIX=cic-cache-test
|
||||
NAME=cic-cache-test
|
||||
USER=postgres
|
||||
PASSWORD=
|
||||
HOST=localhost
|
||||
|
||||
5
apps/cic-cache/config/test/eth.ini
Normal file
5
apps/cic-cache/config/test/eth.ini
Normal file
@ -0,0 +1,5 @@
|
||||
[eth]
|
||||
#ws_provider = ws://localhost:8546
|
||||
#ttp_provider = http://localhost:8545
|
||||
provider = http://localhost:8545
|
||||
#chain_id =
|
||||
@ -1,4 +1,4 @@
|
||||
openapi: "3.0.2"
|
||||
openapi: "3.0.3"
|
||||
info:
|
||||
title: Grassroots Economics CIC Cache
|
||||
description: Cache of processed transaction data from Ethereum blockchain and worker queues
|
||||
@ -9,34 +9,17 @@ info:
|
||||
email: will@grassecon.org
|
||||
license:
|
||||
name: GPLv3
|
||||
version: 0.2.0
|
||||
version: 0.1.0
|
||||
|
||||
paths:
|
||||
/defaultlimit:
|
||||
summary: The default limit value of result sets.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve default limit
|
||||
operationId: limit.default
|
||||
responses:
|
||||
200:
|
||||
description: Limit query successful
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Limit"
|
||||
|
||||
/tx:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
/tx/{offset}/{limit}:
|
||||
description: Bloom filter for batch of latest transactions
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest
|
||||
operationId: tx.get
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
@ -46,153 +29,27 @@ paths:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
|
||||
/tx/{limit}:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{limit}/{offset}:
|
||||
summary: Bloom filter for batch of latest transactions
|
||||
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from newest to oldest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{limit}/{offset}/{block_offset}:
|
||||
summary: Bloom filter for batch of transactions since a particular block.
|
||||
description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.range.block.offset
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful. Results are ordered from oldest to newest.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{limit}/{offset}/{block_offset}/{block_end}:
|
||||
summary: Bloom filter for batch of transactions within a particular block range.
|
||||
description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
/tx/{address}/{offset}/{limit}:
|
||||
description: Bloom filter for batch of latest transactions by account
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.latest.range.block.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_end
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{address}:
|
||||
summary: Bloom filter for batch of latest transactions by account.
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user
|
||||
operationId: tx.get
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
@ -201,30 +58,6 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
|
||||
|
||||
/tx/{address}/{limit}:
|
||||
summary: Bloom filter for batch of latest transactions by account.
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
@ -232,317 +65,26 @@ paths:
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/tx/{address}/{limit}/{offset}:
|
||||
summary: Bloom filter for batch of latest transactions by account
|
||||
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: tx.get.user.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/BlocksBloom"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
|
||||
/txa/{limit}:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{limit}/{offset}:
|
||||
summary: Cached data for latest transactions.
|
||||
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{limit}/{offset}/{block_offset}:
|
||||
summary: Cached data for transactions since a particular block.
|
||||
description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range.block.offset
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
/txa/{limit}/{offset}/{block_offset}/{block_end}:
|
||||
summary: Cached data for transactions within a particular block range.
|
||||
description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.latest.range.block.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: block_end
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{address}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
|
||||
|
||||
/txa/{address}/{limit}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user.limit
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
/txa/{address}/{limit}/{offset}:
|
||||
summary: Cached data for batch of latest transactions by account.
|
||||
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
|
||||
get:
|
||||
tags:
|
||||
- transactions
|
||||
description:
|
||||
Retrieve transactions
|
||||
operationId: txa.get.user.range
|
||||
responses:
|
||||
200:
|
||||
description: Transaction query successful.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/TransactionList"
|
||||
|
||||
parameters:
|
||||
- name: address
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
- name: offset
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: integer
|
||||
format: int32
|
||||
|
||||
|
||||
components:
|
||||
schemas:
|
||||
Limit:
|
||||
type: integer
|
||||
format: int32
|
||||
BlocksBloom:
|
||||
type: object
|
||||
properties:
|
||||
low:
|
||||
type: integer
|
||||
type: int
|
||||
format: int32
|
||||
description: The lowest block number included in the filter
|
||||
high:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The highest block number included in the filter
|
||||
block_filter:
|
||||
type: string
|
||||
format: byte
|
||||
@ -555,89 +97,6 @@ components:
|
||||
type: string
|
||||
description: Hashing algorithm (currently only using sha256)
|
||||
filter_rounds:
|
||||
type: integer
|
||||
type: int
|
||||
format: int32
|
||||
description: Number of hash rounds used to create the filter
|
||||
TransactionList:
|
||||
type: object
|
||||
properties:
|
||||
low:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The lowest block number included in the result set
|
||||
high:
|
||||
type: integer
|
||||
format: int32
|
||||
description: The highest block number included in the filter
|
||||
data:
|
||||
type: array
|
||||
description: Cached transaction data
|
||||
items:
|
||||
$ref: "#/components/schemas/Transaction"
|
||||
Transaction:
|
||||
type: object
|
||||
properties:
|
||||
block_number:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Block number transaction was included in.
|
||||
tx_hash:
|
||||
type: string
|
||||
description: Transaction hash, in hex.
|
||||
date_block:
|
||||
type: integer
|
||||
format: int32
|
||||
description: Block timestamp.
|
||||
sender:
|
||||
type: string
|
||||
description: Spender address, in hex.
|
||||
recipient:
|
||||
type: string
|
||||
description: Beneficiary address, in hex.
|
||||
from_value:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Value deducted from spender's balance.
|
||||
to_value:
|
||||
type: integer
|
||||
format: int64
|
||||
description: Value added to beneficiary's balance.
|
||||
source_token:
|
||||
type: string
|
||||
description: Network address of token in which `from_value` is denominated.
|
||||
destination_token:
|
||||
type: string
|
||||
description: Network address of token in which `to_value` is denominated.
|
||||
success:
|
||||
type: boolean
|
||||
description: Network consensus state on whether the transaction was successful or not.
|
||||
tx_type:
|
||||
type: string
|
||||
enum:
|
||||
- erc20.faucet
|
||||
- faucet.give_to
|
||||
|
||||
examples:
|
||||
data_last:
|
||||
summary: Get the latest cached transactions, using the server's default limit.
|
||||
value: "/txa"
|
||||
|
||||
data_limit:
|
||||
summary: Get the last 42 cached transactions.
|
||||
value: "/txa/42"
|
||||
|
||||
data_range:
|
||||
summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
|
||||
value: "/txa/42/13"
|
||||
|
||||
data_range_block_offset:
|
||||
summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
|
||||
value: "/txa/42/0/1337"
|
||||
|
||||
data_range_block_offset:
|
||||
summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
|
||||
value: "/txa/42/0/1337/1453"
|
||||
|
||||
data_range_block_range:
|
||||
summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
|
||||
value: "/txa/42/13/1337/1453"
|
||||
|
||||
@ -1,25 +1,32 @@
|
||||
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics"
|
||||
|
||||
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
|
||||
# syntax = docker/dockerfile:1.2
|
||||
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-55da5f4e as dev
|
||||
|
||||
# RUN pip install $pip_extra_index_url_flag cic-base[full_graph]==0.1.2b9
|
||||
|
||||
COPY requirements.txt .
|
||||
#RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
|
||||
#RUN pip install $pip_extra_index_url_flag .
|
||||
#RUN pip install .[server]
|
||||
|
||||
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
|
||||
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
|
||||
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
|
||||
ARG EXTRA_PIP_ARGS=""
|
||||
ARG PIP_INDEX_URL=https://pypi.org/simple
|
||||
|
||||
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
|
||||
pip install --index-url $PIP_INDEX_URL \
|
||||
--pre \
|
||||
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
|
||||
pip install --index-url https://pypi.org/simple \
|
||||
--extra-index-url $GITLAB_PYTHON_REGISTRY --extra-index-url $EXTRA_INDEX_URL $EXTRA_PIP_ARGS \
|
||||
-r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY . .
|
||||
RUN pip install . --extra-index-url $EXTRA_PIP_INDEX_URL
|
||||
RUN python setup.py install
|
||||
|
||||
# ini files in config directory defines the configurable parameters for the application
|
||||
# they can all be overridden by environment variables
|
||||
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
|
||||
COPY config/ /usr/local/etc/cic-cache/
|
||||
|
||||
# for db migrations
|
||||
COPY ./aux/wait-for-it/wait-for-it.sh ./
|
||||
RUN git clone https://github.com/vishnubob/wait-for-it.git /usr/local/bin/wait-for-it/
|
||||
COPY cic_cache/db/migrations/ /usr/local/share/cic-cache/alembic/
|
||||
|
||||
COPY /docker/start_tracker.sh ./start_tracker.sh
|
||||
|
||||
@ -2,5 +2,5 @@
|
||||
|
||||
set -e
|
||||
>&2 echo executing database migration
|
||||
python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
||||
python scripts/migrate.py -c /usr/local/etc/cic-cache --migrations-dir /usr/local/share/cic-cache/alembic -vv
|
||||
set +e
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
set -e
|
||||
|
||||
pip install --extra-index-url https://pip.grassrootseconomics.net \
|
||||
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
|
||||
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
|
||||
-r test_requirements.txt
|
||||
|
||||
|
||||
@ -1,15 +1,14 @@
|
||||
alembic==1.4.2
|
||||
confini~=0.5.3
|
||||
confini>=0.3.6rc4,<0.5.0
|
||||
uwsgi==2.0.19.1
|
||||
moolb~=0.2.0
|
||||
cic-eth-registry~=0.6.6
|
||||
moolb~=0.1.1b2
|
||||
cic-eth-registry~=0.6.1a1
|
||||
SQLAlchemy==1.3.20
|
||||
semver==2.13.0
|
||||
psycopg2==2.8.6
|
||||
celery==4.4.7
|
||||
redis==3.5.3
|
||||
chainsyncer[sql]~=0.0.7
|
||||
erc20-faucet~=0.3.2
|
||||
chainlib-eth~=0.0.15
|
||||
eth-address-index~=0.2.4
|
||||
okota~=0.2.5
|
||||
chainsyncer[sql]>=0.0.6a3,<0.1.0
|
||||
erc20-faucet>=0.3.2a2, <0.4.0
|
||||
chainlib-eth>=0.0.9a14,<0.1.0
|
||||
eth-address-index>=0.2.3a4,<0.3.0
|
||||
|
||||
@ -1,55 +1,53 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
# standard imports
|
||||
#!/usr/bin/python
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
|
||||
# external imports
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
import confini
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import dsn_from_config
|
||||
import cic_cache.cli
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
# BUG: the dbdir doesn't work after script install
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
|
||||
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
default_migrations_dir = os.path.join(dbdir, 'migrations')
|
||||
configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations')
|
||||
|
||||
#config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
config_dir = os.path.join('/usr/local/etc/cic-cache')
|
||||
|
||||
arg_flags = cic_cache.cli.argflag_std_base
|
||||
local_arg_flags = cic_cache.cli.argflag_local_sync
|
||||
argparser = cic_cache.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-c', type=str, default=config_dir, help='config file')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
|
||||
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
|
||||
argparser.add_argument('-f', '--force', action='store_true', help='force action')
|
||||
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
|
||||
argparser.add_argument('-f', action='store_true', help='force action')
|
||||
argparser.add_argument('-v', action='store_true', help='be verbose')
|
||||
argparser.add_argument('-vv', action='store_true', help='be more verbose')
|
||||
args = argparser.parse_args()
|
||||
|
||||
extra_args = {
|
||||
'reset': None,
|
||||
'force': None,
|
||||
'migrations_dir': None,
|
||||
}
|
||||
# process config
|
||||
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
|
||||
if args.vv:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
elif args.v:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
|
||||
migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
|
||||
config = confini.Config(args.c, args.env_prefix)
|
||||
config.process()
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config:\n{}'.format(config))
|
||||
|
||||
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
|
||||
if not os.path.isdir(migrations_dir):
|
||||
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
|
||||
migrations_dir = os.path.join(args.migrations_dir, 'default')
|
||||
|
||||
# connect to database
|
||||
dsn = dsn_from_config(config, 'cic_cache')
|
||||
dsn = dsn_from_config(config)
|
||||
|
||||
|
||||
logg.info('using migrations dir {}'.format(migrations_dir))
|
||||
@ -1,7 +1,6 @@
|
||||
[metadata]
|
||||
name = cic-cache
|
||||
description = CIC Cache API and server
|
||||
version = 0.3.0a2
|
||||
author = Louis Holbrook
|
||||
author_email = dev@holbrook.no
|
||||
url = https://gitlab.com/grassrootseconomics/cic-eth
|
||||
@ -35,7 +34,7 @@ packages =
|
||||
cic_cache.runnable.daemons
|
||||
cic_cache.runnable.daemons.filters
|
||||
scripts =
|
||||
./scripts/migrate_cic_cache.py
|
||||
./scripts/migrate.py
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
|
||||
@ -1,39 +1,38 @@
|
||||
from setuptools import setup
|
||||
|
||||
# import configparser
|
||||
import configparser
|
||||
import os
|
||||
import time
|
||||
|
||||
# import time
|
||||
from cic_cache.version import (
|
||||
version_object,
|
||||
version_string
|
||||
)
|
||||
|
||||
# from cic_cache.version import (
|
||||
# version_object,
|
||||
# version_string
|
||||
# )
|
||||
#
|
||||
# class PleaseCommitFirstError(Exception):
|
||||
# pass
|
||||
#
|
||||
# def git_hash():
|
||||
# import subprocess
|
||||
# git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
||||
# if len(git_diff.stdout) > 0:
|
||||
# raise PleaseCommitFirstError()
|
||||
# git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
||||
# git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
||||
# return git_hash_brief
|
||||
#
|
||||
# version_string = str(version_object)
|
||||
#
|
||||
# try:
|
||||
# version_git = git_hash()
|
||||
# version_string += '+build.{}'.format(version_git)
|
||||
# except FileNotFoundError:
|
||||
# time_string_pair = str(time.time()).split('.')
|
||||
# version_string += '+build.{}{:<09d}'.format(
|
||||
# time_string_pair[0],
|
||||
# int(time_string_pair[1]),
|
||||
# )
|
||||
# print('final version string will be {}'.format(version_string))
|
||||
class PleaseCommitFirstError(Exception):
|
||||
pass
|
||||
|
||||
def git_hash():
|
||||
import subprocess
|
||||
git_diff = subprocess.run(['git', 'diff'], capture_output=True)
|
||||
if len(git_diff.stdout) > 0:
|
||||
raise PleaseCommitFirstError()
|
||||
git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
|
||||
git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
|
||||
return git_hash_brief
|
||||
|
||||
version_string = str(version_object)
|
||||
|
||||
try:
|
||||
version_git = git_hash()
|
||||
version_string += '+build.{}'.format(version_git)
|
||||
except FileNotFoundError:
|
||||
time_string_pair = str(time.time()).split('.')
|
||||
version_string += '+build.{}{:<09d}'.format(
|
||||
time_string_pair[0],
|
||||
int(time_string_pair[1]),
|
||||
)
|
||||
print('final version string will be {}'.format(version_string))
|
||||
|
||||
requirements = []
|
||||
f = open('requirements.txt', 'r')
|
||||
@ -53,8 +52,9 @@ while True:
|
||||
test_requirements.append(l.rstrip())
|
||||
f.close()
|
||||
|
||||
|
||||
setup(
|
||||
# version=version_string,
|
||||
version=version_string,
|
||||
install_requires=requirements,
|
||||
tests_require=test_requirements,
|
||||
)
|
||||
|
||||
@ -7,4 +7,4 @@ pytest-celery==0.0.0a1
|
||||
eth_tester==0.5.0b3
|
||||
py-evm==0.3.0a20
|
||||
sarafu-faucet~=0.0.7a1
|
||||
erc20-transfer-authorization~=0.3.6
|
||||
erc20-transfer-authorization>=0.3.5a1,<0.4.0
|
||||
|
||||
@ -6,7 +6,6 @@ import datetime
|
||||
# external imports
|
||||
import pytest
|
||||
import moolb
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
|
||||
# local imports
|
||||
from cic_cache import db
|
||||
@ -43,8 +42,6 @@ def txs(
|
||||
list_tokens,
|
||||
):
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_number = 13
|
||||
@ -57,10 +54,10 @@ def txs(
|
||||
tx_hash_first,
|
||||
list_defaults['block'],
|
||||
tx_number,
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.wallet_address(list_actors['bob']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
list_actors['alice'],
|
||||
list_actors['bob'],
|
||||
list_tokens['foo'],
|
||||
list_tokens['foo'],
|
||||
1024,
|
||||
2048,
|
||||
True,
|
||||
@ -77,10 +74,10 @@ def txs(
|
||||
tx_hash_second,
|
||||
list_defaults['block']-1,
|
||||
tx_number,
|
||||
tx_normalize.wallet_address(list_actors['diane']),
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.executable_address(list_tokens['foo']),
|
||||
tx_normalize.wallet_address(list_tokens['foo']),
|
||||
list_actors['diane'],
|
||||
list_actors['alice'],
|
||||
list_tokens['foo'],
|
||||
list_tokens['foo'],
|
||||
1024,
|
||||
2048,
|
||||
False,
|
||||
@ -106,8 +103,6 @@ def more_txs(
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
tx_number = 666
|
||||
tx_hash = '0x' + os.urandom(32).hex()
|
||||
tx_signed = '0x' + os.urandom(128).hex()
|
||||
@ -120,10 +115,10 @@ def more_txs(
|
||||
tx_hash,
|
||||
list_defaults['block']+2,
|
||||
tx_number,
|
||||
tx_normalize.wallet_address(list_actors['alice']),
|
||||
tx_normalize.wallet_address(list_actors['diane']),
|
||||
tx_normalize.executable_address(list_tokens['bar']),
|
||||
tx_normalize.executable_address(list_tokens['bar']),
|
||||
list_actors['alice'],
|
||||
list_actors['diane'],
|
||||
list_tokens['bar'],
|
||||
list_tokens['bar'],
|
||||
2048,
|
||||
4096,
|
||||
False,
|
||||
|
||||
@ -14,8 +14,7 @@ logg = logging.getLogger(__file__)
|
||||
@pytest.fixture(scope='session')
|
||||
def load_config():
|
||||
config_dir = os.path.join(root_dir, 'config/test')
|
||||
schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
|
||||
conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
|
||||
conf = confini.Config(config_dir, 'CICTEST')
|
||||
conf.process()
|
||||
logg.debug('config {}'.format(conf))
|
||||
return conf
|
||||
|
||||
@ -24,15 +24,11 @@ def database_engine(
|
||||
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
SessionBase.transactional = False
|
||||
SessionBase.poolable = False
|
||||
name = 'cic_cache'
|
||||
database_name = name
|
||||
if load_config.get('DATABASE_PREFIX'):
|
||||
database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
|
||||
try:
|
||||
os.unlink(database_name)
|
||||
os.unlink(load_config.get('DATABASE_NAME'))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
dsn = dsn_from_config(load_config, name)
|
||||
dsn = dsn_from_config(load_config)
|
||||
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
|
||||
return dsn
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@ def test_api_all_data(
|
||||
):
|
||||
|
||||
env = {
|
||||
'PATH_INFO': '/txa/100/0/410000/420000',
|
||||
'PATH_INFO': '/txa/410000/420000',
|
||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||
}
|
||||
j = process_transactions_all_data(init_database, env)
|
||||
@ -23,7 +23,7 @@ def test_api_all_data(
|
||||
assert len(o['data']) == 2
|
||||
|
||||
env = {
|
||||
'PATH_INFO': '/txa/100/0/420000/410000',
|
||||
'PATH_INFO': '/txa/420000/410000',
|
||||
'HTTP_X_CIC_CACHE_MODE': 'all',
|
||||
}
|
||||
|
||||
|
||||
@ -6,7 +6,6 @@ import json
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
from chainlib.encode import TxHexNormalizer
|
||||
|
||||
# local imports
|
||||
from cic_cache import db
|
||||
@ -63,8 +62,6 @@ def test_cache_ranges(
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
oldest = list_defaults['block'] - 1
|
||||
mid = list_defaults['block']
|
||||
newest = list_defaults['block'] + 2
|
||||
@ -103,39 +100,32 @@ def test_cache_ranges(
|
||||
assert b[1] == mid
|
||||
|
||||
# now check when supplying account
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
b = c.load_transactions_account(list_actors['alice'], 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
b = c.load_transactions_account(list_actors['bob'], 0, 100)
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account(account, 0, 100)
|
||||
b = c.load_transactions_account(list_actors['diane'], 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
|
||||
# add block filter to the mix
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
||||
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
|
||||
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == oldest
|
||||
|
||||
@ -150,8 +140,6 @@ def test_cache_ranges_data(
|
||||
|
||||
session = init_database
|
||||
|
||||
tx_normalize = TxHexNormalizer()
|
||||
|
||||
oldest = list_defaults['block'] - 1
|
||||
mid = list_defaults['block']
|
||||
newest = list_defaults['block'] + 2
|
||||
@ -215,8 +203,7 @@ def test_cache_ranges_data(
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
# now check when supplying account
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 3
|
||||
@ -224,15 +211,13 @@ def test_cache_ranges_data(
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
assert b[2][2]['tx_hash'] == more_txs[2]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100)
|
||||
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
|
||||
assert b[0] == oldest
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
@ -240,31 +225,27 @@ def test_cache_ranges_data(
|
||||
assert b[2][1]['tx_hash'] == more_txs[2]
|
||||
|
||||
# add block filter to the mix
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
||||
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['alice'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
|
||||
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == newest
|
||||
assert len(b[2]) == 2
|
||||
assert b[2][0]['tx_hash'] == more_txs[0]
|
||||
assert b[2][1]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['bob'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == mid
|
||||
assert b[1] == mid
|
||||
assert len(b[2]) == 1
|
||||
assert b[2][0]['tx_hash'] == more_txs[1]
|
||||
|
||||
account = tx_normalize.wallet_address(list_actors['diane'])
|
||||
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
|
||||
assert b[0] == oldest
|
||||
assert b[1] == oldest
|
||||
assert len(b[2]) == 1
|
||||
|
||||
@ -82,7 +82,7 @@ def test_query_regex(
|
||||
[
|
||||
('alice', None, None, [(420000, 13), (419999, 42)]),
|
||||
('alice', None, 1, [(420000, 13)]),
|
||||
('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
|
||||
('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
|
||||
('alice', 2, None, []), # 420000 == list_defaults['block']
|
||||
],
|
||||
)
|
||||
@ -107,11 +107,10 @@ def test_query_process_txs_account(
|
||||
path_info = '/tx/user/0x' + strip_0x(actor)
|
||||
if query_offset != None:
|
||||
path_info += '/' + str(query_offset)
|
||||
if query_limit == None:
|
||||
query_limit = 100
|
||||
path_info += '/' + str(query_limit)
|
||||
if query_offset == None:
|
||||
path_info += '/0'
|
||||
if query_limit != None:
|
||||
if query_offset == None:
|
||||
path_info += '/0'
|
||||
path_info += '/' + str(query_limit)
|
||||
env = {
|
||||
'PATH_INFO': path_info,
|
||||
}
|
||||
@ -193,7 +192,7 @@ def test_query_process_txs_bloom(
|
||||
@pytest.mark.parametrize(
|
||||
'query_block_start, query_block_end, query_match_count',
|
||||
[
|
||||
(1, 42, 0),
|
||||
(None, 42, 0),
|
||||
(420000, 420001, 1),
|
||||
(419999, 419999, 1), # matches are inclusive
|
||||
(419999, 420000, 2),
|
||||
@ -212,7 +211,7 @@ def test_query_process_txs_data(
|
||||
query_match_count,
|
||||
):
|
||||
|
||||
path_info = '/txa/100/0'
|
||||
path_info = '/txa'
|
||||
if query_block_start != None:
|
||||
path_info += '/' + str(query_block_start)
|
||||
if query_block_end != None:
|
||||
@ -228,5 +227,4 @@ def test_query_process_txs_data(
|
||||
assert r != None
|
||||
|
||||
o = json.loads(r[1])
|
||||
logg.debug('oo {}'.format(o))
|
||||
assert len(o['data']) == query_match_count
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
celery==4.4.7
|
||||
erc20-demurrage-token~=0.0.6
|
||||
cic-eth-registry~=0.6.3
|
||||
chainlib~=0.0.14
|
||||
cic_eth~=0.12.6
|
||||
erc20-demurrage-token~=0.0.3a1
|
||||
cic-eth-registry>=0.6.1a2,<0.7.0
|
||||
cic-eth[services]~=0.12.4a8
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
[metadata]
|
||||
name = cic-eth-aux-erc20-demurrage-token
|
||||
version = 0.0.3
|
||||
version = 0.0.2a6
|
||||
description = cic-eth tasks supporting erc20 demurrage token
|
||||
author = Louis Holbrook
|
||||
author_email = dev@holbrook.no
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
SQLAlchemy==1.3.20
|
||||
hexathon~=0.1.0
|
||||
chainqueue~=0.0.6a4
|
||||
eth-erc20~=0.1.5
|
||||
cic-eth-registry>=0.6.1a3,<0.7.0
|
||||
hexathon~=0.0.1a8
|
||||
chainqueue>=0.0.4a6,<0.1.0
|
||||
eth-erc20>=0.1.2a2,<0.2.0
|
||||
|
||||
@ -1,2 +1,21 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
|
||||
# local imports
|
||||
from cic_eth.eth.erc20 import default_token
|
||||
from cic_eth.task import BaseTask
|
||||
|
||||
celery_app = celery.current_app
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def default_token(self):
|
||||
return {
|
||||
'symbol': self.default_token_symbol,
|
||||
'address': self.default_token_address,
|
||||
'name': self.default_token_name,
|
||||
'decimals': self.default_token_decimals,
|
||||
}
|
||||
|
||||
@ -123,7 +123,7 @@ class AdminApi:
|
||||
return s_lock.apply_async()
|
||||
|
||||
|
||||
def tag_account(self, chain_spec, tag, address):
|
||||
def tag_account(self, tag, address_hex, chain_spec):
|
||||
"""Persistently associate an address with a plaintext tag.
|
||||
|
||||
Some tags are known by the system and is used to resolve addresses to use for certain transactions.
|
||||
@ -138,7 +138,7 @@ class AdminApi:
|
||||
'cic_eth.eth.account.set_role',
|
||||
[
|
||||
tag,
|
||||
address,
|
||||
address_hex,
|
||||
chain_spec.asdict(),
|
||||
],
|
||||
queue=self.queue,
|
||||
@ -146,30 +146,6 @@ class AdminApi:
|
||||
return s_tag.apply_async()
|
||||
|
||||
|
||||
def get_tag_account(self, chain_spec, tag=None, address=None):
|
||||
if address != None:
|
||||
s_tag = celery.signature(
|
||||
'cic_eth.eth.account.role',
|
||||
[
|
||||
address,
|
||||
chain_spec.asdict(),
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
|
||||
else:
|
||||
s_tag = celery.signature(
|
||||
'cic_eth.eth.account.role_account',
|
||||
[
|
||||
tag,
|
||||
chain_spec.asdict(),
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
|
||||
return s_tag.apply_async()
|
||||
|
||||
|
||||
def have_account(self, address_hex, chain_spec):
|
||||
s_have = celery.signature(
|
||||
'cic_eth.eth.account.have',
|
||||
@ -527,7 +503,7 @@ class AdminApi:
|
||||
queue=self.queue,
|
||||
)
|
||||
t = s.apply_async()
|
||||
role = t.get()[0][1]
|
||||
role = t.get()
|
||||
if role != None:
|
||||
tx['sender_description'] = role
|
||||
|
||||
@ -580,7 +556,7 @@ class AdminApi:
|
||||
queue=self.queue,
|
||||
)
|
||||
t = s.apply_async()
|
||||
role = t.get()[0][1]
|
||||
role = t.get()
|
||||
if role != None:
|
||||
tx['recipient_description'] = role
|
||||
|
||||
|
||||
@ -17,50 +17,15 @@ from cic_eth.enum import LockEnum
|
||||
|
||||
app = celery.current_app
|
||||
|
||||
#logg = logging.getLogger(__name__)
|
||||
logg = logging.getLogger()
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Api(ApiBase):
|
||||
|
||||
@staticmethod
|
||||
def to_v_list(v, n):
|
||||
"""Translate an arbitrary number of string and/or list arguments to a list of list of string arguments
|
||||
|
||||
:param v: Arguments
|
||||
:type v: str or list
|
||||
:param n: Number of elements to generate arguments for
|
||||
:type n: int
|
||||
:rtype: list
|
||||
:returns: list of assembled arguments
|
||||
"""
|
||||
if isinstance(v, str):
|
||||
vv = v
|
||||
v = []
|
||||
for i in range(n):
|
||||
v.append([vv])
|
||||
elif not isinstance(v, list):
|
||||
raise ValueError('argument must be single string, or list or strings or lists')
|
||||
else:
|
||||
if len(v) != n:
|
||||
raise ValueError('v argument count must match integer n')
|
||||
for i in range(n):
|
||||
if isinstance(v[i], str):
|
||||
v[i] = [v[i]]
|
||||
elif not isinstance(v, list):
|
||||
raise ValueError('proof argument must be single string, or list or strings or lists')
|
||||
|
||||
return v
|
||||
|
||||
|
||||
|
||||
def default_token(self):
|
||||
"""Retrieves the default fallback token of the custodial network.
|
||||
|
||||
:returns: uuid of root task
|
||||
:rtype: celery.Task
|
||||
"""
|
||||
s_token = celery.signature(
|
||||
'cic_eth.eth.erc20.default_token',
|
||||
'cic_eth.admin.token.default_token',
|
||||
[],
|
||||
queue=self.queue,
|
||||
)
|
||||
@ -70,97 +35,6 @@ class Api(ApiBase):
|
||||
return s_token.apply_async()
|
||||
|
||||
|
||||
def token(self, token_symbol, proof=None):
|
||||
"""Single-token alias for tokens method.
|
||||
|
||||
See tokens method for details.
|
||||
|
||||
:param token_symbol: Token symbol to look up
|
||||
:type token_symbol: str
|
||||
:param proof: Proofs to add to signature verification for the token
|
||||
:type proof: str or list
|
||||
:returns: uuid of root task
|
||||
:rtype: celery.Task
|
||||
"""
|
||||
if not isinstance(token_symbol, str):
|
||||
raise ValueError('token symbol must be string')
|
||||
|
||||
return self.tokens([token_symbol], proof=proof)
|
||||
|
||||
|
||||
def tokens(self, token_symbols, proof=None):
|
||||
"""Perform a token data lookup from the token index. The token index will enforce unique associations between token symbol and contract address.
|
||||
|
||||
Token symbols are always strings, and should be specified using uppercase letters.
|
||||
|
||||
If the proof argument is included, the network will be queried for trusted signatures on the given proof(s). There must exist at least one trusted signature for every given proof for every token. Trusted signatures for the custodial system are provided at service startup.
|
||||
|
||||
The proof argument may be specified in a number of ways:
|
||||
|
||||
- as None, in which case proof checks are skipped (although there may still be builtin proof checks being performed)
|
||||
- as a single string, where the same proof is used for each token lookup
|
||||
- as an array of strings, where the respective proof is used for the respective token. number of proofs must match the number of tokens.
|
||||
- as an array of lists, where the respective proofs in each list is used for the respective token. number of lists of proofs must match the number of tokens.
|
||||
|
||||
The success callback provided at the Api object instantiation will receive individual calls for each token that passes the proof checks. Each token that does not pass is passed to the Api error callback.
|
||||
|
||||
This method is not intended to be used synchronously. Do so at your peril.
|
||||
|
||||
:param token_symbols: Token symbol strings to look up
|
||||
:type token_symbol: list
|
||||
:param proof: Proof(s) to verify tokens against
|
||||
:type proof: None, str or list
|
||||
:returns: uuid of root task
|
||||
:rtype: celery.Task
|
||||
"""
|
||||
if not isinstance(token_symbols, list):
|
||||
raise ValueError('token symbols argument must be list')
|
||||
|
||||
if proof == None:
|
||||
logg.debug('looking up tokens without external proof check: {}'.format(','.join(token_symbols)))
|
||||
proof = ''
|
||||
|
||||
logg.debug('proof is {}'.format(proof))
|
||||
l = len(token_symbols)
|
||||
if len(proof) == 0:
|
||||
l = 0
|
||||
proof = Api.to_v_list(proof, l)
|
||||
|
||||
chain_spec_dict = self.chain_spec.asdict()
|
||||
|
||||
s_token_resolve = celery.signature(
|
||||
'cic_eth.eth.erc20.resolve_tokens_by_symbol',
|
||||
[
|
||||
token_symbols,
|
||||
chain_spec_dict,
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
|
||||
s_token_info = celery.signature(
|
||||
'cic_eth.eth.erc20.token_info',
|
||||
[
|
||||
chain_spec_dict,
|
||||
proof,
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
|
||||
s_token_verify = celery.signature(
|
||||
'cic_eth.eth.erc20.verify_token_info',
|
||||
[
|
||||
chain_spec_dict,
|
||||
self.callback_success,
|
||||
self.callback_error,
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
|
||||
s_token_info.link(s_token_verify)
|
||||
s_token_resolve.link(s_token_info)
|
||||
return s_token_resolve.apply_async()
|
||||
|
||||
|
||||
# def convert_transfer(self, from_address, to_address, target_return, minimum_return, from_token_symbol, to_token_symbol):
|
||||
# """Executes a chain of celery tasks that performs conversion between two ERC20 tokens, and transfers to a specified receipient after convert has completed.
|
||||
#
|
||||
@ -515,7 +389,7 @@ class Api(ApiBase):
|
||||
:param password: Password to encode the password with in the backend (careful, you will have to remember it)
|
||||
:type password: str
|
||||
:param register: Register the new account in accounts index backend
|
||||
:type register: bool
|
||||
:type password: bool
|
||||
:returns: uuid of root task
|
||||
:rtype: celery.Task
|
||||
"""
|
||||
@ -683,4 +557,3 @@ class Api(ApiBase):
|
||||
|
||||
t = self.callback_success.apply_async([r])
|
||||
return t
|
||||
|
||||
|
||||
@ -1,10 +1,7 @@
|
||||
import logging
|
||||
|
||||
import celery
|
||||
|
||||
celery_app = celery.current_app
|
||||
#logg = celery_app.log.get_default_logger()
|
||||
logg = logging.getLogger()
|
||||
logg = celery_app.log.get_default_logger()
|
||||
|
||||
|
||||
@celery_app.task(bind=True)
|
||||
|
||||
@ -12,9 +12,8 @@ from cic_eth.db.models.base import SessionBase
|
||||
from cic_eth.db.enum import LockEnum
|
||||
from cic_eth.error import LockedError
|
||||
from cic_eth.admin.ctrl import check_lock
|
||||
from cic_eth.eth.gas import have_gas_minimum
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
logg = logging.getLogger().getChild(__name__)
|
||||
|
||||
|
||||
def health(*args, **kwargs):
|
||||
@ -32,15 +31,18 @@ def health(*args, **kwargs):
|
||||
return True
|
||||
|
||||
gas_provider = AccountRole.get_address('GAS_GIFTER', session=session)
|
||||
min_gas = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
|
||||
if config.get('ETH_MIN_FEE_PRICE'):
|
||||
min_gas *= int(config.get('ETH_MIN_FEE_PRICE'))
|
||||
|
||||
r = have_gas_minimum(chain_spec, gas_provider, min_gas, session=session)
|
||||
|
||||
session.close()
|
||||
|
||||
if not r:
|
||||
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, min_gas))
|
||||
|
||||
return r
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
o = balance(gas_provider)
|
||||
r = rpc.do(o)
|
||||
try:
|
||||
r = int(r, 16)
|
||||
except TypeError:
|
||||
r = int(r)
|
||||
gas_min = int(config.get('ETH_GAS_GIFTER_MINIMUM_BALANCE'))
|
||||
if r < gas_min:
|
||||
logg.error('EEK! gas gifter has balance {}, below minimum {}'.format(r, gas_min))
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
# external imports
|
||||
from chainlib.chain import ChainSpec
|
||||
|
||||
# local imports
|
||||
from cic_eth.admin.ctrl import check_lock
|
||||
from cic_eth.enum import LockEnum
|
||||
from cic_eth.error import LockedError
|
||||
|
||||
|
||||
def health(*args, **kwargs):
|
||||
config = kwargs['config']
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
|
||||
try:
|
||||
check_lock(None, chain_spec.asdict(), LockEnum.START)
|
||||
except LockedError as e:
|
||||
return False
|
||||
return True
|
||||
@ -16,22 +16,16 @@ class ArgumentParser(BaseArgumentParser):
|
||||
self.add_argument('--redis-port', dest='redis_port', type=int, help='redis host to use for task submission')
|
||||
self.add_argument('--redis-db', dest='redis_db', type=int, help='redis db to use')
|
||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||
self.add_argument('--redis-host-callback', dest='redis_host_callback', type=str, help='redis host to use for callback (defaults to redis host)')
|
||||
self.add_argument('--redis-port-callback', dest='redis_port_callback', type=int, help='redis port to use for callback (defaults to redis port)')
|
||||
self.add_argument('--redis-host-callback', dest='redis_host_callback', default='localhost', type=str, help='redis host to use for callback')
|
||||
self.add_argument('--redis-port-callback', dest='redis_port_callback', default=6379, type=int, help='redis port to use for callback')
|
||||
self.add_argument('--redis-timeout', default=20.0, type=float, help='Redis callback timeout')
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
self.add_argument('--celery-scheme', type=str, help='Celery broker scheme (defaults to "redis")')
|
||||
self.add_argument('--celery-host', type=str, help='Celery broker host (defaults to redis host)')
|
||||
self.add_argument('--celery-port', type=str, help='Celery broker port (defaults to redis port)')
|
||||
self.add_argument('--celery-db', type=int, help='Celery broker db (defaults to redis db)')
|
||||
self.add_argument('--celery-result-scheme', type=str, help='Celery result backend scheme (defaults to celery broker scheme)')
|
||||
self.add_argument('--celery-result-host', type=str, help='Celery result backend host (defaults to celery broker host)')
|
||||
self.add_argument('--celery-result-port', type=str, help='Celery result backend port (defaults to celery broker port)')
|
||||
self.add_argument('--celery-result-db', type=int, help='Celery result backend db (defaults to celery broker db)')
|
||||
self.add_argument('--celery-no-result', action='store_true', help='Disable the Celery results backend')
|
||||
self.add_argument('-q', '--celery-queue', dest='celery_queue', type=str, default='cic-eth', help='Task queue')
|
||||
if local_arg_flags & CICFlag.SYNCER:
|
||||
self.add_argument('--offset', type=int, help='Start block height for initial history sync')
|
||||
self.add_argument('--offset', type=int, default=0, help='Start block height for initial history sync')
|
||||
self.add_argument('--no-history', action='store_true', dest='no_history', help='Skip initial history sync')
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
self.add_argument('-r', '--registry-address', type=str, dest='registry_address', help='CIC registry contract address')
|
||||
|
||||
|
||||
|
||||
|
||||
@ -24,8 +24,8 @@ class CICFlag(enum.IntEnum):
|
||||
# sync - nibble 4
|
||||
SYNCER = 4096
|
||||
|
||||
argflag_local_base = argflag_std_base | Flag.CHAIN_SPEC
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
|
||||
argflag_local_task = CICFlag.CELERY
|
||||
argflag_local_taskcallback = argflag_local_task | CICFlag.REDIS | CICFlag.REDIS_CALLBACK
|
||||
argflag_local_chain = CICFlag.CHAIN
|
||||
argflag_local_sync = CICFlag.SYNCER | CICFlag.CHAIN
|
||||
|
||||
@ -1,18 +1,12 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import urllib.parse
|
||||
import copy
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.cli import (
|
||||
Config as BaseConfig,
|
||||
Flag,
|
||||
)
|
||||
from urlybird.merge import (
|
||||
urlhostmerge,
|
||||
urlmerge,
|
||||
)
|
||||
|
||||
# local imports
|
||||
from .base import CICFlag
|
||||
@ -46,7 +40,6 @@ class Config(BaseConfig):
|
||||
if local_arg_flags & CICFlag.CHAIN:
|
||||
local_args_override['CIC_REGISTRY_ADDRESS'] = getattr(args, 'registry_address')
|
||||
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
local_args_override['CELERY_QUEUE'] = getattr(args, 'celery_queue')
|
||||
|
||||
@ -56,71 +49,15 @@ class Config(BaseConfig):
|
||||
|
||||
config.dict_override(local_args_override, 'local cli args')
|
||||
|
||||
local_celery_args_override = {}
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
hostport = urlhostmerge(
|
||||
None,
|
||||
config.get('REDIS_HOST'),
|
||||
config.get('REDIS_PORT'),
|
||||
)
|
||||
db = getattr(args, 'redis_db', None)
|
||||
if db != None:
|
||||
db = str(db)
|
||||
|
||||
redis_url = (
|
||||
'redis',
|
||||
hostport,
|
||||
db,
|
||||
)
|
||||
|
||||
|
||||
celery_config_url = urllib.parse.urlsplit(config.get('CELERY_BROKER_URL'))
|
||||
hostport = urlhostmerge(
|
||||
celery_config_url[1],
|
||||
getattr(args, 'celery_host', None),
|
||||
getattr(args, 'celery_port', None),
|
||||
)
|
||||
db = getattr(args, 'redis_db', None)
|
||||
if db != None:
|
||||
db = str(db)
|
||||
celery_arg_url = (
|
||||
getattr(args, 'celery_scheme', None),
|
||||
hostport,
|
||||
db,
|
||||
)
|
||||
|
||||
celery_url = urlmerge(redis_url, celery_config_url, celery_arg_url)
|
||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
||||
local_celery_args_override['CELERY_BROKER_URL'] = celery_url_string
|
||||
if not getattr(args, 'celery_no_result'):
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = config.get('CELERY_RESULT_URL')
|
||||
if local_celery_args_override['CELERY_RESULT_URL'] == None:
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = local_celery_args_override['CELERY_BROKER_URL']
|
||||
celery_config_url = urllib.parse.urlsplit(local_celery_args_override['CELERY_RESULT_URL'])
|
||||
hostport = urlhostmerge(
|
||||
celery_config_url[1],
|
||||
getattr(args, 'celery_result_host', None),
|
||||
getattr(args, 'celery_result_port', None),
|
||||
)
|
||||
celery_arg_url = (
|
||||
getattr(args, 'celery_result_scheme', None),
|
||||
hostport,
|
||||
getattr(args, 'celery_result_db', None),
|
||||
)
|
||||
celery_url = urlmerge(celery_config_url, celery_arg_url)
|
||||
logg.debug('celery url {} {}'.format(celery_config_url, celery_url))
|
||||
celery_url_string = urllib.parse.urlunsplit(celery_url)
|
||||
local_celery_args_override['CELERY_RESULT_URL'] = celery_url_string
|
||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||
|
||||
config.dict_override(local_celery_args_override, 'local celery cli args')
|
||||
|
||||
if local_arg_flags & CICFlag.REDIS_CALLBACK:
|
||||
redis_host_callback = getattr(args, 'redis_host_callback', config.get('REDIS_HOST'))
|
||||
redis_port_callback = getattr(args, 'redis_port_callback', config.get('REDIS_PORT'))
|
||||
config.add(redis_host_callback, '_REDIS_HOST_CALLBACK')
|
||||
config.add(redis_port_callback, '_REDIS_PORT_CALLBACK')
|
||||
config.add(getattr(args, 'redis_host_callback'), '_REDIS_HOST_CALLBACK')
|
||||
config.add(getattr(args, 'redis_port_callback'), '_REDIS_PORT_CALLBACK')
|
||||
|
||||
if local_arg_flags & CICFlag.CELERY:
|
||||
config.add(config.true('CELERY_DEBUG'), 'CELERY_DEBUG', exists_ok=True)
|
||||
|
||||
logg.debug('config loaded:\n{}'.format(config))
|
||||
|
||||
return config
|
||||
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
[celery]
|
||||
broker_url =
|
||||
broker_url = redis://localhost:6379
|
||||
result_url =
|
||||
queue = cic-eth
|
||||
debug = 0
|
||||
|
||||
@ -2,5 +2,5 @@
|
||||
registry_address =
|
||||
trust_address =
|
||||
default_token_symbol =
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas,cic_eth.check.start
|
||||
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
|
||||
run_dir = /run
|
||||
|
||||
@ -1,6 +1,2 @@
|
||||
[eth]
|
||||
gas_holder_minimum_units = 180000
|
||||
gas_holder_refill_units = 15
|
||||
gas_holder_refill_threshold = 3
|
||||
gas_gifter_refill_buffer = 3
|
||||
min_fee_price = 1
|
||||
gas_gifter_minimum_balance = 10000000000000000000000
|
||||
|
||||
@ -23,7 +23,7 @@ def upgrade():
|
||||
op.create_table(
|
||||
'lock',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column("address", sa.String, nullable=True),
|
||||
sa.Column("address", sa.String(42), nullable=True),
|
||||
sa.Column('blockchain', sa.String),
|
||||
sa.Column("flags", sa.BIGINT(), nullable=False, default=0),
|
||||
sa.Column("date_created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow),
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
"""Add gas cache
|
||||
|
||||
Revision ID: c91cafc3e0c1
|
||||
Revises: aee12aeb47ec
|
||||
Create Date: 2021-10-28 20:45:34.239865
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'c91cafc3e0c1'
|
||||
down_revision = 'aee12aeb47ec'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'gas_cache',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column("address", sa.String, nullable=False),
|
||||
sa.Column("tx_hash", sa.String, nullable=True),
|
||||
sa.Column("method", sa.String, nullable=True),
|
||||
sa.Column("value", sa.BIGINT(), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('gas_cache')
|
||||
@ -1,27 +0,0 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from sqlalchemy import Column, String, NUMERIC
|
||||
|
||||
# local imports
|
||||
from .base import SessionBase
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GasCache(SessionBase):
|
||||
"""Provides gas budget cache for token operations
|
||||
"""
|
||||
__tablename__ = 'gas_cache'
|
||||
|
||||
address = Column(String())
|
||||
tx_hash = Column(String())
|
||||
method = Column(String())
|
||||
value = Column(NUMERIC())
|
||||
|
||||
def __init__(self, address, method, value, tx_hash):
|
||||
self.address = address
|
||||
self.tx_hash = tx_hash
|
||||
self.method = method
|
||||
self.value = value
|
||||
@ -12,7 +12,7 @@ from cic_eth.error import (
|
||||
IntegrityError,
|
||||
)
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
class Nonce(SessionBase):
|
||||
@ -21,7 +21,7 @@ class Nonce(SessionBase):
|
||||
__tablename__ = 'nonce'
|
||||
|
||||
nonce = Column(Integer)
|
||||
address_hex = Column(String(40))
|
||||
address_hex = Column(String(42))
|
||||
|
||||
|
||||
@staticmethod
|
||||
|
||||
@ -24,22 +24,8 @@ class AccountRole(SessionBase):
|
||||
tag = Column(Text)
|
||||
address_hex = Column(String(42))
|
||||
|
||||
|
||||
@staticmethod
|
||||
def all(session=None):
|
||||
session = SessionBase.bind_session(session)
|
||||
|
||||
pairs = []
|
||||
|
||||
q = session.query(AccountRole.tag, AccountRole.address_hex)
|
||||
for r in q.all():
|
||||
pairs.append((r[1], r[0]),)
|
||||
|
||||
SessionBase.release_session(session)
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
|
||||
# TODO:
|
||||
@staticmethod
|
||||
def get_address(tag, session):
|
||||
"""Get Ethereum address matching the given tag
|
||||
|
||||
@ -69,12 +69,9 @@ class StatusEnum(enum.IntEnum):
|
||||
class LockEnum(enum.IntEnum):
|
||||
"""
|
||||
STICKY: When set, reset is not possible
|
||||
INIT: When set, startup is possible without second level sanity checks (e.g. gas gifter balance)
|
||||
START: When set, startup is not possible, regardless of state
|
||||
CREATE: Disable creation of accounts
|
||||
SEND: Disable sending to network
|
||||
QUEUE: Disable queueing new or modified transactions
|
||||
QUERY: Disable all queue state and transaction queries
|
||||
"""
|
||||
STICKY=1
|
||||
INIT=2
|
||||
@ -82,8 +79,7 @@ class LockEnum(enum.IntEnum):
|
||||
SEND=8
|
||||
QUEUE=16
|
||||
QUERY=32
|
||||
START=int(0x80000000)
|
||||
ALL=int(0x7ffffffe)
|
||||
ALL=int(0xfffffffffffffffe)
|
||||
|
||||
|
||||
def status_str(v, bits_only=False):
|
||||
|
||||
@ -48,6 +48,8 @@ class RoleMissingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
|
||||
class IntegrityError(Exception):
|
||||
"""Exception raised to signal irregularities with deduplication and ordering of tasks
|
||||
|
||||
@ -64,10 +66,8 @@ class LockedError(Exception):
|
||||
|
||||
class SeppukuError(Exception):
|
||||
"""Exception base class for all errors that should cause system shutdown
|
||||
|
||||
"""
|
||||
def __init__(self, message, lockdown=False):
|
||||
self.message = message
|
||||
self.lockdown = lockdown
|
||||
|
||||
|
||||
class SignerError(SeppukuError):
|
||||
@ -85,8 +85,3 @@ class RoleAgencyError(SeppukuError):
|
||||
class YouAreBrokeError(Exception):
|
||||
"""Exception raised when a value transfer is attempted without access to sufficient funds
|
||||
"""
|
||||
|
||||
|
||||
class TrustError(Exception):
|
||||
"""Exception raised when required trust proofs are missing for a request
|
||||
"""
|
||||
|
||||
@ -136,7 +136,7 @@ def register(self, account_address, chain_spec_dict, writer_address=None):
|
||||
# Generate and sign transaction
|
||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||
nonce_oracle = CustodialTaskNonceOracle(writer_address, self.request.root_id, session=session) #, default_nonce)
|
||||
gas_oracle = self.create_gas_oracle(rpc, code_callback=AccountRegistry.gas)
|
||||
gas_oracle = self.create_gas_oracle(rpc, AccountRegistry.gas)
|
||||
account_registry = AccountsIndex(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
||||
(tx_hash_hex, tx_signed_raw_hex) = account_registry.add(account_registry_address, writer_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
||||
rpc_signer.disconnect()
|
||||
@ -192,7 +192,7 @@ def gift(self, account_address, chain_spec_dict):
|
||||
# Generate and sign transaction
|
||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||
nonce_oracle = CustodialTaskNonceOracle(account_address, self.request.root_id, session=session) #, default_nonce)
|
||||
gas_oracle = self.create_gas_oracle(rpc, code_callback=MinterFaucet.gas)
|
||||
gas_oracle = self.create_gas_oracle(rpc, MinterFaucet.gas)
|
||||
faucet = Faucet(chain_spec, signer=rpc_signer, nonce_oracle=nonce_oracle, gas_oracle=gas_oracle)
|
||||
(tx_hash_hex, tx_signed_raw_hex) = faucet.give_to(faucet_address, account_address, account_address, tx_format=TxFormat.RLP_SIGNED)
|
||||
rpc_signer.disconnect()
|
||||
@ -266,46 +266,19 @@ def set_role(self, tag, address, chain_spec_dict):
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def role(self, address, chain_spec_dict):
|
||||
"""Return account role for address and/or role
|
||||
"""Return account role for address
|
||||
|
||||
:param account: Account to check
|
||||
:type account: str, 0x-hex
|
||||
:param chain_spec_dict: Chain spec dict representation
|
||||
:type chain_spec_dict: dict
|
||||
:param chain_str: Chain spec string representation
|
||||
:type chain_str: str
|
||||
:returns: Account, or None if not exists
|
||||
:rtype: Varies
|
||||
"""
|
||||
session = self.create_session()
|
||||
role_tag = AccountRole.role_for(address, session=session)
|
||||
session.close()
|
||||
return [(address, role_tag,)]
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def role_account(self, role_tag, chain_spec_dict):
|
||||
"""Return address for role.
|
||||
|
||||
If the role parameter is None, will return addresses for all roles.
|
||||
|
||||
:param role_tag: Role to match
|
||||
:type role_tag: str
|
||||
:param chain_spec_dict: Chain spec dict representation
|
||||
:type chain_spec_dict: dict
|
||||
:returns: List with a single account/tag pair for a single tag, or a list of account and tag pairs for all tags
|
||||
:rtype: list
|
||||
"""
|
||||
session = self.create_session()
|
||||
|
||||
pairs = None
|
||||
if role_tag != None:
|
||||
addr = AccountRole.get_address(role_tag, session=session)
|
||||
pairs = [(addr, role_tag,)]
|
||||
else:
|
||||
pairs = AccountRole.all(session=session)
|
||||
|
||||
session.close()
|
||||
|
||||
return pairs
|
||||
return role_tag
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=CriticalSQLAlchemyTask)
|
||||
|
||||
@ -10,9 +10,6 @@ from chainlib.eth.tx import (
|
||||
TxFormat,
|
||||
unpack,
|
||||
)
|
||||
from chainlib.eth.contract import (
|
||||
ABIContractEncoder,
|
||||
)
|
||||
from cic_eth_registry import CICRegistry
|
||||
from cic_eth_registry.erc20 import ERC20Token
|
||||
from hexathon import (
|
||||
@ -22,7 +19,6 @@ from hexathon import (
|
||||
from chainqueue.error import NotLocalTxError
|
||||
from eth_erc20 import ERC20
|
||||
from chainqueue.sql.tx import cache_tx_dict
|
||||
from okota.token_index.index import to_identifier
|
||||
|
||||
# local imports
|
||||
from cic_eth.db.models.base import SessionBase
|
||||
@ -34,26 +30,25 @@ from cic_eth.error import (
|
||||
YouAreBrokeError,
|
||||
)
|
||||
from cic_eth.queue.tx import register_tx
|
||||
from cic_eth.eth.gas import create_check_gas_task
|
||||
from cic_eth.eth.util import CacheGasOracle
|
||||
from cic_eth.eth.gas import (
|
||||
create_check_gas_task,
|
||||
MaxGasOracle,
|
||||
)
|
||||
from cic_eth.ext.address import translate_address
|
||||
from cic_eth.task import (
|
||||
CriticalSQLAlchemyTask,
|
||||
CriticalWeb3Task,
|
||||
CriticalSQLAlchemyAndSignerTask,
|
||||
BaseTask,
|
||||
)
|
||||
from cic_eth.eth.nonce import CustodialTaskNonceOracle
|
||||
from cic_eth.encode import tx_normalize
|
||||
from cic_eth.eth.trust import verify_proofs
|
||||
from cic_eth.error import SignerError
|
||||
|
||||
celery_app = celery.current_app
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=CriticalWeb3Task)
|
||||
def balance(self, tokens, holder_address, chain_spec_dict):
|
||||
@celery_app.task(base=CriticalWeb3Task)
|
||||
def balance(tokens, holder_address, chain_spec_dict):
|
||||
"""Return token balances for a list of tokens for given address
|
||||
|
||||
:param tokens: Token addresses
|
||||
@ -72,9 +67,8 @@ def balance(self, tokens, holder_address, chain_spec_dict):
|
||||
for t in tokens:
|
||||
address = t['address']
|
||||
logg.debug('address {} {}'.format(address, holder_address))
|
||||
gas_oracle = self.create_gas_oracle(rpc, min_price=self.min_fee_price)
|
||||
token = ERC20Token(chain_spec, rpc, add_0x(address))
|
||||
c = ERC20(chain_spec, gas_oracle=gas_oracle)
|
||||
c = ERC20(chain_spec)
|
||||
o = c.balance_of(address, holder_address, sender_address=caller_address)
|
||||
r = rpc.do(o)
|
||||
t['balance_network'] = c.parse_balance(r)
|
||||
@ -157,12 +151,8 @@ def transfer_from(self, tokens, holder_address, receiver_address, value, chain_s
|
||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||
|
||||
session = self.create_session()
|
||||
|
||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||
enc = ABIContractEncoder()
|
||||
enc.method('transferFrom')
|
||||
method = enc.get()
|
||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
|
||||
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||
try:
|
||||
(tx_hash_hex, tx_signed_raw_hex) = c.transfer_from(t['address'], spender_address, holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||
@ -232,12 +222,8 @@ def transfer(self, tokens, holder_address, receiver_address, value, chain_spec_d
|
||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||
|
||||
session = self.create_session()
|
||||
|
||||
enc = ABIContractEncoder()
|
||||
enc.method('transfer')
|
||||
method = enc.get()
|
||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session, min_price=self.min_fee_price)
|
||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||
try:
|
||||
(tx_hash_hex, tx_signed_raw_hex) = c.transfer(t['address'], holder_address, receiver_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||
@ -305,12 +291,8 @@ def approve(self, tokens, holder_address, spender_address, value, chain_spec_dic
|
||||
rpc_signer = RPCConnection.connect(chain_spec, 'signer')
|
||||
|
||||
session = self.create_session()
|
||||
|
||||
nonce_oracle = CustodialTaskNonceOracle(holder_address, self.request.root_id, session=session)
|
||||
enc = ABIContractEncoder()
|
||||
enc.method('approve')
|
||||
method = enc.get()
|
||||
gas_oracle = self.create_gas_oracle(rpc, t['address'], method=method, session=session)
|
||||
gas_oracle = self.create_gas_oracle(rpc, MaxGasOracle.gas)
|
||||
c = ERC20(chain_spec, signer=rpc_signer, gas_oracle=gas_oracle, nonce_oracle=nonce_oracle)
|
||||
try:
|
||||
(tx_hash_hex, tx_signed_raw_hex) = c.approve(t['address'], holder_address, spender_address, value, tx_format=TxFormat.RLP_SIGNED)
|
||||
@ -397,8 +379,6 @@ def cache_transfer_data(
|
||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
||||
recipient_address = tx_normalize.wallet_address(tx_data[0])
|
||||
token_value = tx_data[1]
|
||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
||||
destination_token_address = source_token_address
|
||||
|
||||
|
||||
session = SessionBase.create_session()
|
||||
@ -406,8 +386,8 @@ def cache_transfer_data(
|
||||
'hash': tx_hash_hex,
|
||||
'from': sender_address,
|
||||
'to': recipient_address,
|
||||
'source_token': source_token_address,
|
||||
'destination_token': destination_token_address,
|
||||
'source_token': tx['to'],
|
||||
'destination_token': tx['to'],
|
||||
'from_value': token_value,
|
||||
'to_value': token_value,
|
||||
}
|
||||
@ -439,16 +419,14 @@ def cache_transfer_from_data(
|
||||
spender_address = tx_data[0]
|
||||
recipient_address = tx_data[1]
|
||||
token_value = tx_data[2]
|
||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
||||
destination_token_address = source_token_address
|
||||
|
||||
session = SessionBase.create_session()
|
||||
tx_dict = {
|
||||
'hash': tx_hash_hex,
|
||||
'from': tx['from'],
|
||||
'to': recipient_address,
|
||||
'source_token': source_token_address,
|
||||
'destination_token': destination_token_address,
|
||||
'source_token': tx['to'],
|
||||
'destination_token': tx['to'],
|
||||
'from_value': token_value,
|
||||
'to_value': token_value,
|
||||
}
|
||||
@ -480,16 +458,14 @@ def cache_approve_data(
|
||||
sender_address = tx_normalize.wallet_address(tx['from'])
|
||||
recipient_address = tx_normalize.wallet_address(tx_data[0])
|
||||
token_value = tx_data[1]
|
||||
source_token_address = tx_normalize.executable_address(tx['to'])
|
||||
destination_token_address = source_token_address
|
||||
|
||||
session = SessionBase.create_session()
|
||||
tx_dict = {
|
||||
'hash': tx_hash_hex,
|
||||
'from': sender_address,
|
||||
'to': recipient_address,
|
||||
'source_token': source_token_address,
|
||||
'destination_token': destination_token_address,
|
||||
'source_token': tx['to'],
|
||||
'destination_token': tx['to'],
|
||||
'from_value': token_value,
|
||||
'to_value': token_value,
|
||||
}
|
||||
@ -497,69 +473,3 @@ def cache_approve_data(
|
||||
session.close()
|
||||
return (tx_hash_hex, cache_id)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def token_info(self, tokens, chain_spec_dict, proofs=[]):
|
||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
|
||||
i = 0
|
||||
|
||||
for token in tokens:
|
||||
result_data = []
|
||||
token_chain_object = ERC20Token(chain_spec, rpc, add_0x(token['address']))
|
||||
token_chain_object.load(rpc)
|
||||
|
||||
token_symbol_proof_hex = to_identifier(token_chain_object.symbol)
|
||||
token_proofs = [token_symbol_proof_hex]
|
||||
if len(proofs) > 0:
|
||||
token_proofs += proofs[i]
|
||||
|
||||
tokens[i] = {
|
||||
'decimals': token_chain_object.decimals,
|
||||
'name': token_chain_object.name,
|
||||
'symbol': token_chain_object.symbol,
|
||||
'address': tx_normalize.executable_address(token_chain_object.address),
|
||||
'proofs': token_proofs,
|
||||
'converters': tokens[i]['converters'],
|
||||
}
|
||||
i += 1
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def verify_token_info(self, tokens, chain_spec_dict, success_callback, error_callback):
|
||||
queue = self.request.delivery_info.get('routing_key')
|
||||
|
||||
for token in tokens:
|
||||
s = celery.signature(
|
||||
'cic_eth.eth.trust.verify_proofs',
|
||||
[
|
||||
token,
|
||||
token['address'],
|
||||
token['proofs'],
|
||||
chain_spec_dict,
|
||||
success_callback,
|
||||
error_callback,
|
||||
],
|
||||
queue=queue,
|
||||
)
|
||||
|
||||
if success_callback != None:
|
||||
s.link(success_callback)
|
||||
if error_callback != None:
|
||||
s.on_error(error_callback)
|
||||
s.apply_async()
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def default_token(self):
|
||||
return {
|
||||
'symbol': self.default_token_symbol,
|
||||
'address': self.default_token_address,
|
||||
'name': self.default_token_name,
|
||||
'decimals': self.default_token_decimals,
|
||||
}
|
||||
|
||||
@ -41,7 +41,6 @@ from chainqueue.db.models.tx import TxCache
|
||||
from chainqueue.db.models.otx import Otx
|
||||
|
||||
# local imports
|
||||
from cic_eth.db.models.gas_cache import GasCache
|
||||
from cic_eth.db.models.role import AccountRole
|
||||
from cic_eth.db.models.base import SessionBase
|
||||
from cic_eth.error import (
|
||||
@ -66,56 +65,17 @@ from cic_eth.encode import (
|
||||
ZERO_ADDRESS_NORMAL,
|
||||
unpack_normal,
|
||||
)
|
||||
from cic_eth.error import SeppukuError
|
||||
from cic_eth.eth.util import MAXIMUM_FEE_UNITS
|
||||
|
||||
celery_app = celery.current_app
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
MAXIMUM_FEE_UNITS = 8000000
|
||||
|
||||
@celery_app.task(base=CriticalSQLAlchemyTask)
|
||||
def apply_gas_value_cache(address, method, value, tx_hash):
|
||||
return apply_gas_value_cache_local(address, method, value, tx_hash)
|
||||
class MaxGasOracle:
|
||||
|
||||
|
||||
def apply_gas_value_cache_local(address, method, value, tx_hash, session=None):
|
||||
address = tx_normalize.executable_address(address)
|
||||
tx_hash = tx_normalize.tx_hash(tx_hash)
|
||||
value = int(value)
|
||||
|
||||
session = SessionBase.bind_session(session)
|
||||
q = session.query(GasCache)
|
||||
q = q.filter(GasCache.address==address)
|
||||
q = q.filter(GasCache.method==method)
|
||||
o = q.first()
|
||||
|
||||
if o == None:
|
||||
o = GasCache(address, method, value, tx_hash)
|
||||
elif value > o.value:
|
||||
o.value = value
|
||||
o.tx_hash = strip_0x(tx_hash)
|
||||
|
||||
session.add(o)
|
||||
session.commit()
|
||||
|
||||
SessionBase.release_session(session)
|
||||
|
||||
|
||||
def have_gas_minimum(chain_spec, address, min_gas, session=None, rpc=None):
|
||||
if rpc == None:
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
o = balance(add_0x(address))
|
||||
r = rpc.do(o)
|
||||
try:
|
||||
r = int(r)
|
||||
except ValueError:
|
||||
r = strip_0x(r)
|
||||
r = int(r, 16)
|
||||
logg.debug('have gas minimum {} have gas {} minimum is {}'.format(address, r, min_gas))
|
||||
if r < min_gas:
|
||||
return False
|
||||
return True
|
||||
def gas(code=None):
|
||||
return MAXIMUM_FEE_UNITS
|
||||
|
||||
|
||||
def create_check_gas_task(tx_signed_raws_hex, chain_spec, holder_address, gas=None, tx_hashes_hex=None, queue=None):
|
||||
@ -397,13 +357,6 @@ def refill_gas(self, recipient_address, chain_spec_dict):
|
||||
# set up evm RPC connection
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
|
||||
# check the gas balance of the gifter
|
||||
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_refill_amount):
|
||||
raise SeppukuError('Noooooooooooo; gas gifter {} is broke!'.format(gas_provider))
|
||||
|
||||
if not have_gas_minimum(chain_spec, gas_provider, self.safe_gas_gifter_balance):
|
||||
logg.error('Gas gifter {} gas balance is below the safe level to operate!'.format(gas_provider))
|
||||
|
||||
# set up transaction builder
|
||||
nonce_oracle = CustodialTaskNonceOracle(gas_provider, self.request.root_id, session=session)
|
||||
gas_oracle = self.create_gas_oracle(rpc)
|
||||
|
||||
@ -1,77 +0,0 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
from eth_address_declarator import Declarator
|
||||
from chainlib.connection import RPCConnection
|
||||
from chainlib.chain import ChainSpec
|
||||
from cic_eth.db.models.role import AccountRole
|
||||
from cic_eth_registry import CICRegistry
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
from cic_eth.task import BaseTask
|
||||
from cic_eth.error import TrustError
|
||||
|
||||
celery_app = celery.current_app
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def verify_proof(self, chained_input, proof, subject, chain_spec_dict, success_callback, error_callback):
|
||||
proof = strip_0x(proof)
|
||||
|
||||
proofs = []
|
||||
|
||||
logg.debug('proof count {}'.format(len(proofs)))
|
||||
if len(proofs) == 0:
|
||||
logg.debug('error {}'.format(len(proofs)))
|
||||
raise TrustError('foo')
|
||||
|
||||
return (chained_input, (proof, proofs))
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=BaseTask)
|
||||
def verify_proofs(self, chained_input, subject, proofs, chain_spec_dict, success_callback, error_callback):
|
||||
queue = self.request.delivery_info.get('routing_key')
|
||||
|
||||
chain_spec = ChainSpec.from_dict(chain_spec_dict)
|
||||
rpc = RPCConnection.connect(chain_spec, 'default')
|
||||
|
||||
session = self.create_session()
|
||||
sender_address = AccountRole.get_address('DEFAULT', session)
|
||||
|
||||
registry = CICRegistry(chain_spec, rpc)
|
||||
declarator_address = registry.by_name('AddressDeclarator', sender_address=sender_address)
|
||||
|
||||
declarator = Declarator(chain_spec)
|
||||
|
||||
have_proofs = {}
|
||||
|
||||
for proof in proofs:
|
||||
|
||||
proof = strip_0x(proof)
|
||||
|
||||
have_proofs[proof] = []
|
||||
|
||||
for trusted_address in self.trusted_addresses:
|
||||
o = declarator.declaration(declarator_address, trusted_address, subject, sender_address=sender_address)
|
||||
r = rpc.do(o)
|
||||
declarations = declarator.parse_declaration(r)
|
||||
logg.debug('comparing proof {} with declarations for {} by {}: {}'.format(proof, subject, trusted_address, declarations))
|
||||
|
||||
for declaration in declarations:
|
||||
declaration = strip_0x(declaration)
|
||||
if declaration == proof:
|
||||
logg.debug('have token proof {} match for trusted address {}'.format(declaration, trusted_address))
|
||||
have_proofs[proof].append(trusted_address)
|
||||
|
||||
out_proofs = {}
|
||||
for proof in have_proofs.keys():
|
||||
if len(have_proofs[proof]) == 0:
|
||||
logg.error('missing signer for proof {} subject {}'.format(proof, subject))
|
||||
raise TrustError((subject, proof,))
|
||||
out_proofs[proof] = have_proofs[proof]
|
||||
|
||||
return (chained_input, out_proofs)
|
||||
@ -1,54 +0,0 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from chainlib.eth.gas import RPCGasOracle
|
||||
from hexathon import strip_0x
|
||||
|
||||
# local imports
|
||||
from cic_eth.db.models.gas_cache import GasCache
|
||||
from cic_eth.encode import tx_normalize
|
||||
from cic_eth.db.models.base import SessionBase
|
||||
|
||||
MAXIMUM_FEE_UNITS = 8000000
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MaxGasOracle(RPCGasOracle):
|
||||
|
||||
def get_fee_units(self, code=None):
|
||||
return MAXIMUM_FEE_UNITS
|
||||
|
||||
|
||||
class CacheGasOracle(MaxGasOracle):
|
||||
"""Returns a previously recorded value for fee unit expenditure for a contract call, if it exists. Otherwise returns max units.
|
||||
|
||||
:todo: instead of max units, connect a pluggable gas heuristics engine.
|
||||
"""
|
||||
|
||||
def __init__(self, conn, address, method=None, session=None, min_price=None, id_generator=None):
|
||||
super(CacheGasOracle, self).__init__(conn, code_callback=self.get_fee_units, min_price=min_price, id_generator=id_generator)
|
||||
self.value = None
|
||||
self.address = address
|
||||
self.method = method
|
||||
|
||||
address = tx_normalize.executable_address(address)
|
||||
session = SessionBase.bind_session(session)
|
||||
q = session.query(GasCache)
|
||||
q = q.filter(GasCache.address==address)
|
||||
if method != None:
|
||||
method = strip_0x(method)
|
||||
q = q.filter(GasCache.method==method)
|
||||
o = q.first()
|
||||
if o != None:
|
||||
self.value = int(o.value)
|
||||
|
||||
SessionBase.release_session(session)
|
||||
|
||||
|
||||
def get_fee_units(self, code=None):
|
||||
if self.value != None:
|
||||
logg.debug('found stored gas unit value {} for address {} method {}'.format(self.value, self.address, self.method))
|
||||
return self.value
|
||||
return super(CacheGasOracle, self).get_fee_units(code=code)
|
||||
@ -4,21 +4,18 @@ import tempfile
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
# local imports
|
||||
# local impors
|
||||
from cic_eth.task import BaseTask
|
||||
|
||||
#logg = logging.getLogger(__name__)
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def init_celery_tasks(
|
||||
contract_roles,
|
||||
):
|
||||
BaseTask.call_address = contract_roles['DEFAULT']
|
||||
BaseTask.trusted_addresses = [
|
||||
contract_roles['TRUSTED_DECLARATOR'],
|
||||
contract_roles['CONTRACT_DEPLOYER'],
|
||||
]
|
||||
|
||||
|
||||
# celery fixtures
|
||||
@ -41,7 +38,6 @@ def celery_includes():
|
||||
'cic_eth.callbacks.noop',
|
||||
'cic_eth.callbacks.http',
|
||||
'cic_eth.pytest.mock.filter',
|
||||
'cic_eth.pytest.mock.callback',
|
||||
]
|
||||
|
||||
|
||||
|
||||
@ -8,14 +8,15 @@ import confini
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.dirname(os.path.dirname(script_dir))
|
||||
config_dir = os.path.join(root_dir, 'cic_eth', 'data', 'config')
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def load_config():
|
||||
override_config_dir = os.path.join(root_dir, 'config', 'test')
|
||||
conf = confini.Config(config_dir, 'CICTEST', override_dirs=[override_config_dir])
|
||||
config_dir = os.environ.get('CONFINI_DIR')
|
||||
if config_dir == None:
|
||||
config_dir = os.path.join(root_dir, 'config/test')
|
||||
conf = confini.Config(config_dir, 'CICTEST')
|
||||
conf.process()
|
||||
logg.debug('config {}'.format(conf))
|
||||
return conf
|
||||
|
||||
@ -1,2 +1 @@
|
||||
from .filter import *
|
||||
from .callback import *
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
# standard imports
|
||||
import os
|
||||
import logging
|
||||
import mmap
|
||||
|
||||
# standard imports
|
||||
import tempfile
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
|
||||
#logg = logging.getLogger(__name__)
|
||||
logg = logging.getLogger()
|
||||
|
||||
celery_app = celery.current_app
|
||||
|
||||
|
||||
class CallbackTask(celery.Task):
|
||||
|
||||
mmap_path = tempfile.mkdtemp()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, base=CallbackTask)
|
||||
def test_callback(self, a, b, c):
|
||||
s = 'ok'
|
||||
if c > 0:
|
||||
s = 'err'
|
||||
|
||||
fp = os.path.join(self.mmap_path, b)
|
||||
f = open(fp, 'wb+')
|
||||
f.write(b'\x00')
|
||||
f.seek(0)
|
||||
m = mmap.mmap(f.fileno(), length=1)
|
||||
m.write(c.to_bytes(1, 'big'))
|
||||
m.close()
|
||||
f.close()
|
||||
|
||||
logg.debug('test callback ({}): {} {} {}'.format(s, a, b, c))
|
||||
@ -72,7 +72,7 @@ def __balance_incoming_compatible(token_address, receiver_address):
|
||||
status_compare = dead()
|
||||
q = q.filter(Otx.status.op('&')(status_compare)==0)
|
||||
# TODO: this can change the result for the recipient if tx is later obsoleted and resubmission is delayed.
|
||||
#q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
|
||||
q = q.filter(Otx.status.op('&')(StatusBits.IN_NETWORK)==StatusBits.IN_NETWORK)
|
||||
q = q.filter(TxCache.destination_token_address==token_address)
|
||||
delta = 0
|
||||
for r in q.all():
|
||||
|
||||
@ -10,6 +10,7 @@ import datetime
|
||||
|
||||
# external imports
|
||||
import celery
|
||||
from cic_eth_registry import CICRegistry
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.tx import unpack
|
||||
from chainlib.connection import RPCConnection
|
||||
|
||||
@ -3,4 +3,3 @@ from .tx import TxFilter
|
||||
from .gas import GasFilter
|
||||
from .register import RegistrationFilter
|
||||
from .transferauth import TransferAuthFilter
|
||||
from .token import TokenFilter
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
# standard imports
|
||||
import logging
|
||||
|
||||
# external imports
|
||||
from eth_erc20 import ERC20
|
||||
from chainlib.eth.contract import (
|
||||
ABIContractEncoder,
|
||||
ABIContractType,
|
||||
)
|
||||
from chainlib.eth.constant import ZERO_ADDRESS
|
||||
from chainlib.eth.address import is_same_address
|
||||
from chainlib.eth.error import RequestMismatchException
|
||||
from cic_eth_registry import CICRegistry
|
||||
from cic_eth_registry.erc20 import ERC20Token
|
||||
from eth_token_index import TokenUniqueSymbolIndex
|
||||
import celery
|
||||
|
||||
# local imports
|
||||
from .base import SyncFilter
|
||||
|
||||
logg = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TokenFilter(SyncFilter):
|
||||
|
||||
def __init__(self, chain_spec, queue, call_address=ZERO_ADDRESS):
|
||||
self.queue = queue
|
||||
self.chain_spec = chain_spec
|
||||
self.caller_address = call_address
|
||||
|
||||
|
||||
def filter(self, conn, block, tx, db_session=None):
|
||||
if not tx.payload:
|
||||
return (None, None)
|
||||
|
||||
try:
|
||||
r = ERC20.parse_transfer_request(tx.payload)
|
||||
except RequestMismatchException:
|
||||
return (None, None)
|
||||
|
||||
token_address = tx.inputs[0]
|
||||
token = ERC20Token(self.chain_spec, conn, token_address)
|
||||
|
||||
registry = CICRegistry(self.chain_spec, conn)
|
||||
r = registry.by_name(token.symbol, sender_address=self.caller_address)
|
||||
if is_same_address(r, ZERO_ADDRESS):
|
||||
return None
|
||||
|
||||
enc = ABIContractEncoder()
|
||||
enc.method('transfer')
|
||||
method = enc.get()
|
||||
|
||||
s = celery.signature(
|
||||
'cic_eth.eth.gas.apply_gas_value_cache',
|
||||
[
|
||||
token_address,
|
||||
method,
|
||||
tx.gas_used,
|
||||
tx.hash,
|
||||
],
|
||||
queue=self.queue,
|
||||
)
|
||||
return s.apply_async()
|
||||
@ -67,10 +67,7 @@ from cic_eth.registry import (
|
||||
connect_declarator,
|
||||
connect_token_registry,
|
||||
)
|
||||
from cic_eth.task import (
|
||||
BaseTask,
|
||||
CriticalWeb3Task,
|
||||
)
|
||||
from cic_eth.task import BaseTask
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
@ -79,18 +76,18 @@ arg_flags = cic_eth.cli.argflag_std_read
|
||||
local_arg_flags = cic_eth.cli.argflag_local_task
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
argparser.add_argument('--default-token-symbol', dest='default_token_symbol', type=str, help='Symbol of default token to use')
|
||||
argparser.add_argument('--trace-queue-status', default=None, dest='trace_queue_status', action='store_true', help='set to perist all queue entry status changes to storage')
|
||||
argparser.add_argument('--aux-all', action='store_true', help='include tasks from all submodules from the aux module path')
|
||||
argparser.add_argument('--min-fee-price', dest='min_fee_price', type=int, help='set minimum fee price for transactions, in wei')
|
||||
argparser.add_argument('--aux', action='append', type=str, default=[], help='add single submodule from the aux module path')
|
||||
args = argparser.parse_args()
|
||||
|
||||
# process config
|
||||
extra_args = {
|
||||
'default_token_symbol': 'CIC_DEFAULT_TOKEN_SYMBOL',
|
||||
'aux_all': None,
|
||||
'aux': None,
|
||||
'trace_queue_status': 'TASKS_TRACE_QUEUE_STATUS',
|
||||
'min_fee_price': 'ETH_MIN_FEE_PRICE',
|
||||
}
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
@ -190,17 +187,6 @@ elif len(args.aux) > 0:
|
||||
logg.info('aux module {} found in path {}'.format(v, aux_dir))
|
||||
aux.append(v)
|
||||
|
||||
default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
|
||||
defaullt_token_address = None
|
||||
if default_token_symbol:
|
||||
default_token_address = registry.by_name(default_token_symbol)
|
||||
else:
|
||||
default_token_address = registry.by_name('DefaultToken')
|
||||
c = ERC20Token(chain_spec, conn, default_token_address)
|
||||
default_token_symbol = c.symbol
|
||||
logg.info('found default token {} address {}'.format(default_token_symbol, default_token_address))
|
||||
config.add(default_token_symbol, 'CIC_DEFAULT_TOKEN_SYMBOL', exists_ok=True)
|
||||
|
||||
for v in aux:
|
||||
mname = 'cic_eth_aux.' + v
|
||||
mod = importlib.import_module(mname)
|
||||
@ -218,22 +204,12 @@ def main():
|
||||
argv.append('-n')
|
||||
argv.append(config.get('CELERY_QUEUE'))
|
||||
|
||||
# TODO: More elegant way of setting queue-wide settings
|
||||
BaseTask.default_token_symbol = default_token_symbol
|
||||
BaseTask.default_token_address = default_token_address
|
||||
BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
|
||||
BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
|
||||
default_token = ERC20Token(chain_spec, conn, add_0x(BaseTask.default_token_address))
|
||||
default_token.load(conn)
|
||||
BaseTask.default_token_decimals = default_token.decimals
|
||||
BaseTask.default_token_name = default_token.name
|
||||
BaseTask.trusted_addresses = trusted_addresses
|
||||
CriticalWeb3Task.safe_gas_refill_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_UNITS'))
|
||||
CriticalWeb3Task.safe_gas_threshold_amount = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_HOLDER_REFILL_THRESHOLD'))
|
||||
CriticalWeb3Task.safe_gas_gifter_balance = int(config.get('ETH_GAS_HOLDER_MINIMUM_UNITS')) * int(config.get('ETH_GAS_GIFTER_REFILL_BUFFER'))
|
||||
if config.get('ETH_MIN_FEE_PRICE'):
|
||||
BaseTask.min_fee_price = int(config.get('ETH_MIN_FEE_PRICE'))
|
||||
CriticalWeb3Task.safe_gas_threshold_amount *= BaseTask.min_fee_price
|
||||
CriticalWeb3Task.safe_gas_refill_amount *= BaseTask.min_fee_price
|
||||
CriticalWeb3Task.safe_gas_gifter_balance *= BaseTask.min_fee_price
|
||||
|
||||
BaseTask.run_dir = config.get('CIC_RUN_DIR')
|
||||
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))
|
||||
|
||||
@ -36,7 +36,6 @@ from cic_eth.runnable.daemons.filters import (
|
||||
TxFilter,
|
||||
RegistrationFilter,
|
||||
TransferAuthFilter,
|
||||
TokenFilter,
|
||||
)
|
||||
from cic_eth.stat import init_chain_stat
|
||||
from cic_eth.registry import (
|
||||
@ -100,10 +99,10 @@ def main():
|
||||
syncer_backends = SQLBackend.resume(chain_spec, block_offset)
|
||||
|
||||
if len(syncer_backends) == 0:
|
||||
initial_block_start = int(config.get('SYNCER_OFFSET'))
|
||||
initial_block_offset = int(block_offset)
|
||||
initial_block_start = config.get('SYNCER_OFFSET')
|
||||
initial_block_offset = block_offset
|
||||
if config.true('SYNCER_NO_HISTORY'):
|
||||
initial_block_start = initial_block_offset
|
||||
initial_block_start = block_offset
|
||||
initial_block_offset += 1
|
||||
syncer_backends.append(SQLBackend.initial(chain_spec, initial_block_offset, start_block_height=initial_block_start))
|
||||
logg.info('found no backends to resume, adding initial sync from history start {} end {}'.format(initial_block_start, initial_block_offset))
|
||||
@ -155,8 +154,6 @@ def main():
|
||||
|
||||
gas_filter = GasFilter(chain_spec, config.get('CELERY_QUEUE'))
|
||||
|
||||
token_gas_cache_filter = TokenFilter(chain_spec, config.get('CELERY_QUEUE'))
|
||||
|
||||
#transfer_auth_filter = TransferAuthFilter(registry, chain_spec, config.get('_CELERY_QUEUE'))
|
||||
|
||||
i = 0
|
||||
@ -166,7 +163,6 @@ def main():
|
||||
syncer.add_filter(registration_filter)
|
||||
# TODO: the two following filter functions break the filter loop if return uuid. Pro: less code executed. Con: Possibly unintuitive flow break
|
||||
syncer.add_filter(tx_filter)
|
||||
syncer.add_filter(token_gas_cache_filter)
|
||||
#syncer.add_filter(transfer_auth_filter)
|
||||
for cf in callback_filters:
|
||||
syncer.add_filter(cf)
|
||||
|
||||
@ -8,7 +8,6 @@ import re
|
||||
# external imports
|
||||
import cic_eth.cli
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.address import is_address
|
||||
from xdg.BaseDirectory import xdg_config_home
|
||||
|
||||
# local imports
|
||||
@ -22,18 +21,12 @@ logg = logging.getLogger()
|
||||
arg_flags = cic_eth.cli.argflag_std_base | cic_eth.cli.Flag.UNSAFE | cic_eth.cli.Flag.CHAIN_SPEC
|
||||
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser.add_argument('--set', action='store_true', help='sets the given tag')
|
||||
argparser.add_argument('--tag', type=str, help='operate on the given tag')
|
||||
argparser.add_positional('address', required=False, type=str, help='address associated with tag')
|
||||
argparser.add_positional('tag', type=str, help='address tag')
|
||||
argparser.add_positional('address', type=str, help='address')
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
extra_args = {
|
||||
'set': None,
|
||||
'tag': None,
|
||||
'address': None,
|
||||
}
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags)
|
||||
|
||||
celery_app = cic_eth.cli.CeleryApp.from_config(config)
|
||||
|
||||
@ -46,17 +39,7 @@ api = AdminApi(None)
|
||||
|
||||
|
||||
def main():
|
||||
if config.get('_ADDRESS') != None and not is_address(config.get('_ADDRESS')):
|
||||
sys.stderr.write('Invalid address {}'.format(config.get('_ADDRESS')))
|
||||
sys.exit(1)
|
||||
|
||||
if config.get('_SET'):
|
||||
admin_api.tag_account(chain_spec, config.get('_TAG'), config.get('_ADDRESS'))
|
||||
else:
|
||||
t = admin_api.get_tag_account(chain_spec, tag=config.get('_TAG'), address=config.get('_ADDRESS'))
|
||||
r = t.get()
|
||||
for v in r:
|
||||
sys.stdout.write('{}\t{}\n'.format(v[1], v[0]))
|
||||
admin_api.tag_account(args.tag, args.address, chain_spec)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -18,7 +18,7 @@ from cic_eth.api import Api
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger('create_account_script')
|
||||
|
||||
arg_flags = cic_eth.cli.argflag_local_base
|
||||
arg_flags = cic_eth.cli.argflag_std_base
|
||||
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser.add_argument('--token-symbol', dest='token_symbol', type=str, help='Token symbol')
|
||||
|
||||
@ -16,14 +16,9 @@ import confini
|
||||
import celery
|
||||
from chainlib.chain import ChainSpec
|
||||
from chainlib.eth.connection import EthHTTPConnection
|
||||
from hexathon import (
|
||||
add_0x,
|
||||
strip_0x,
|
||||
uniform as hex_uniform,
|
||||
)
|
||||
from hexathon import add_0x
|
||||
|
||||
# local imports
|
||||
import cic_eth.cli
|
||||
from cic_eth.api.admin import AdminApi
|
||||
from cic_eth.db.enum import (
|
||||
StatusEnum,
|
||||
@ -36,35 +31,59 @@ logging.basicConfig(level=logging.WARNING)
|
||||
logg = logging.getLogger()
|
||||
|
||||
default_format = 'terminal'
|
||||
default_config_dir = os.environ.get('CONFINI_DIR', '/usr/local/etc/cic')
|
||||
|
||||
arg_flags = cic_eth.cli.argflag_std_base
|
||||
local_arg_flags = cic_eth.cli.argflag_local_taskcallback
|
||||
argparser = cic_eth.cli.ArgumentParser(arg_flags)
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument('-p', '--provider', dest='p', default='http://localhost:8545', type=str, help='Web3 provider url (http only)')
|
||||
argparser.add_argument('-r', '--registry-address', dest='r', type=str, help='CIC registry address')
|
||||
argparser.add_argument('-f', '--format', dest='f', default=default_format, type=str, help='Output format')
|
||||
argparser.add_argument('--status-raw', dest='status_raw', action='store_true', help='Output status bit enum names only')
|
||||
argparser.add_argument('-c', type=str, default=default_config_dir, help='config root to use')
|
||||
argparser.add_argument('-i', '--chain-spec', dest='i', type=str, help='chain spec')
|
||||
argparser.add_argument('-q', type=str, default='cic-eth', help='celery queue to submit transaction tasks to')
|
||||
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
|
||||
argparser.add_argument('-v', action='store_true', help='Be verbose')
|
||||
argparser.add_argument('-vv', help='be more verbose', action='store_true')
|
||||
argparser.add_argument('query', type=str, help='Transaction, transaction hash, account or "lock"')
|
||||
argparser.process_local_flags(local_arg_flags)
|
||||
args = argparser.parse_args()
|
||||
|
||||
if args.v == True:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
elif args.vv == True:
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
extra_args = {
|
||||
'f': '_FORMAT',
|
||||
'query': '_QUERY',
|
||||
|
||||
config_dir = os.path.join(args.c)
|
||||
os.makedirs(config_dir, 0o777, True)
|
||||
config = confini.Config(config_dir, args.env_prefix)
|
||||
config.process()
|
||||
args_override = {
|
||||
'ETH_PROVIDER': getattr(args, 'p'),
|
||||
'CIC_CHAIN_SPEC': getattr(args, 'i'),
|
||||
'CIC_REGISTRY_ADDRESS': getattr(args, 'r'),
|
||||
}
|
||||
config = cic_eth.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
|
||||
# override args
|
||||
config.dict_override(args_override, 'cli args')
|
||||
config.censor('PASSWORD', 'DATABASE')
|
||||
config.censor('PASSWORD', 'SSL')
|
||||
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
|
||||
|
||||
celery_app = cic_eth.cli.CeleryApp.from_config(config)
|
||||
queue = config.get('CELERY_QUEUE')
|
||||
try:
|
||||
config.add(add_0x(args.query), '_QUERY', True)
|
||||
except:
|
||||
config.add(args.query, '_QUERY', True)
|
||||
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CHAIN_SPEC'))
|
||||
celery_app = celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
|
||||
|
||||
# connect to celery
|
||||
celery_app = cic_eth.cli.CeleryApp.from_config(config)
|
||||
queue = args.q
|
||||
|
||||
# set up rpc
|
||||
rpc = cic_eth.cli.RPC.from_config(config) #, use_signer=True)
|
||||
conn = rpc.get_default()
|
||||
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
|
||||
|
||||
admin_api = AdminApi(conn)
|
||||
rpc = EthHTTPConnection(args.p)
|
||||
|
||||
#registry_address = config.get('CIC_REGISTRY_ADDRESS')
|
||||
|
||||
admin_api = AdminApi(rpc)
|
||||
|
||||
t = admin_api.registry()
|
||||
registry_address = t.get()
|
||||
@ -94,7 +113,7 @@ def render_tx(o, **kwargs):
|
||||
|
||||
for v in o.get('status_log', []):
|
||||
d = datetime.datetime.fromisoformat(v[0])
|
||||
e = status_str(v[1], config.get('_RAW'))
|
||||
e = status_str(v[1], args.status_raw)
|
||||
content += '{}: {}\n'.format(d, e)
|
||||
|
||||
return content
|
||||
@ -135,24 +154,20 @@ def render_lock(o, **kwargs):
|
||||
def main():
|
||||
txs = []
|
||||
renderer = render_tx
|
||||
if len(config.get('_QUERY')) > 66:
|
||||
#registry = connect_registry(rpc, chain_spec, registry_address)
|
||||
#admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), registry=registry, renderer=renderer)
|
||||
admin_api.tx(chain_spec, tx_raw=config.get('_QUERY'), renderer=renderer)
|
||||
elif len(config.get('_QUERY')) > 42:
|
||||
#registry = connect_registry(rpc, chain_spec, registry_address)
|
||||
#admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), registry=registry, renderer=renderer)
|
||||
admin_api.tx(chain_spec, tx_hash=config.get('_QUERY'), renderer=renderer)
|
||||
|
||||
query = config.get('_QUERY')
|
||||
try:
|
||||
query = hex_uniform(strip_0x(query))
|
||||
except TypeError:
|
||||
pass
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if len(query) > 64:
|
||||
admin_api.tx(chain_spec, tx_raw=query, renderer=renderer)
|
||||
elif len(query) > 40:
|
||||
admin_api.tx(chain_spec, tx_hash=query, renderer=renderer)
|
||||
|
||||
elif len(query) == 40:
|
||||
txs = admin_api.account(chain_spec, query, include_recipient=False, renderer=render_account)
|
||||
elif len(config.get('_QUERY')) == 42:
|
||||
#registry = connect_registry(rpc, chain_spec, registry_address)
|
||||
txs = admin_api.account(chain_spec, config.get('_QUERY'), include_recipient=False, renderer=render_account)
|
||||
renderer = render_account
|
||||
elif len(query) >= 4 and query[:4] == 'lock':
|
||||
elif len(config.get('_QUERY')) >= 4 and config.get('_QUERY')[:4] == 'lock':
|
||||
t = admin_api.get_lock()
|
||||
txs = t.get()
|
||||
renderer = render_lock
|
||||
@ -160,7 +175,7 @@ def main():
|
||||
r = renderer(txs)
|
||||
sys.stdout.write(r + '\n')
|
||||
else:
|
||||
raise ValueError('cannot parse argument {}'.format(query))
|
||||
raise ValueError('cannot parse argument {}'.format(config.get('_QUERY')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@ -17,7 +17,6 @@ from cic_eth_registry.error import UnknownContractError
|
||||
# local imports
|
||||
from cic_eth.error import SeppukuError
|
||||
from cic_eth.db.models.base import SessionBase
|
||||
from cic_eth.eth.util import CacheGasOracle, MaxGasOracle
|
||||
|
||||
#logg = logging.getLogger().getChild(__name__)
|
||||
logg = logging.getLogger()
|
||||
@ -25,46 +24,18 @@ logg = logging.getLogger()
|
||||
celery_app = celery.current_app
|
||||
|
||||
|
||||
|
||||
class BaseTask(celery.Task):
|
||||
|
||||
session_func = SessionBase.create_session
|
||||
call_address = ZERO_ADDRESS
|
||||
trusted_addresses = []
|
||||
min_fee_price = 1
|
||||
min_fee_limit = 30000
|
||||
create_nonce_oracle = RPCNonceOracle
|
||||
create_gas_oracle = RPCGasOracle
|
||||
default_token_address = None
|
||||
default_token_symbol = None
|
||||
default_token_name = None
|
||||
default_token_decimals = None
|
||||
run_dir = '/run'
|
||||
|
||||
|
||||
def create_gas_oracle(self, conn, address=None, *args, **kwargs):
|
||||
x = None
|
||||
if address is None:
|
||||
x = RPCGasOracle(
|
||||
conn,
|
||||
code_callback=kwargs.get('code_callback', self.get_min_fee_limit),
|
||||
min_price=self.min_fee_price,
|
||||
id_generator=kwargs.get('id_generator'),
|
||||
)
|
||||
else:
|
||||
|
||||
x = MaxGasOracle(conn)
|
||||
x.code_callback = x.get_fee_units
|
||||
|
||||
return x
|
||||
|
||||
|
||||
def get_min_fee_limit(self, code):
|
||||
return self.min_fee_limit
|
||||
|
||||
|
||||
def get_min_fee_limit(self, code):
|
||||
return self.min_fee_limit
|
||||
|
||||
|
||||
def create_session(self):
|
||||
return BaseTask.session_func()
|
||||
|
||||
@ -87,7 +58,7 @@ class BaseTask(celery.Task):
|
||||
)
|
||||
s.apply_async()
|
||||
|
||||
|
||||
|
||||
class CriticalTask(BaseTask):
|
||||
retry_jitter = True
|
||||
retry_backoff = True
|
||||
@ -99,25 +70,26 @@ class CriticalSQLAlchemyTask(CriticalTask):
|
||||
sqlalchemy.exc.DatabaseError,
|
||||
sqlalchemy.exc.TimeoutError,
|
||||
sqlalchemy.exc.ResourceClosedError,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class CriticalWeb3Task(CriticalTask):
|
||||
autoretry_for = (
|
||||
ConnectionError,
|
||||
)
|
||||
safe_gas_threshold_amount = 60000 * 3
|
||||
safe_gas_refill_amount = safe_gas_threshold_amount * 5
|
||||
safe_gas_gifter_balance = safe_gas_threshold_amount * 5 * 100
|
||||
safe_gas_threshold_amount = 2000000000 * 60000 * 3
|
||||
safe_gas_refill_amount = safe_gas_threshold_amount * 5
|
||||
|
||||
|
||||
class CriticalSQLAlchemyAndWeb3Task(CriticalWeb3Task):
|
||||
class CriticalSQLAlchemyAndWeb3Task(CriticalTask):
|
||||
autoretry_for = (
|
||||
sqlalchemy.exc.DatabaseError,
|
||||
sqlalchemy.exc.TimeoutError,
|
||||
ConnectionError,
|
||||
sqlalchemy.exc.ResourceClosedError,
|
||||
)
|
||||
safe_gas_threshold_amount = 2000000000 * 60000 * 3
|
||||
safe_gas_refill_amount = safe_gas_threshold_amount * 5
|
||||
|
||||
|
||||
class CriticalSQLAlchemyAndSignerTask(CriticalTask):
|
||||
@ -125,12 +97,15 @@ class CriticalSQLAlchemyAndSignerTask(CriticalTask):
|
||||
sqlalchemy.exc.DatabaseError,
|
||||
sqlalchemy.exc.TimeoutError,
|
||||
sqlalchemy.exc.ResourceClosedError,
|
||||
)
|
||||
)
|
||||
|
||||
class CriticalWeb3AndSignerTask(CriticalWeb3Task):
|
||||
class CriticalWeb3AndSignerTask(CriticalTask):
|
||||
autoretry_for = (
|
||||
ConnectionError,
|
||||
)
|
||||
safe_gas_threshold_amount = 2000000000 * 60000 * 3
|
||||
safe_gas_refill_amount = safe_gas_threshold_amount * 5
|
||||
|
||||
|
||||
@celery_app.task()
|
||||
def check_health(self):
|
||||
|
||||
@ -10,7 +10,7 @@ version = (
|
||||
0,
|
||||
12,
|
||||
4,
|
||||
'alpha.14',
|
||||
'alpha.8',
|
||||
)
|
||||
|
||||
version_object = semver.VersionInfo(
|
||||
|
||||
2
apps/cic-eth/config/test/accounts.ini
Normal file
2
apps/cic-eth/config/test/accounts.ini
Normal file
@ -0,0 +1,2 @@
|
||||
[accounts]
|
||||
writer_address =
|
||||
2
apps/cic-eth/config/test/bancor.ini
Normal file
2
apps/cic-eth/config/test/bancor.ini
Normal file
@ -0,0 +1,2 @@
|
||||
[bancor]
|
||||
dir = tests/testdata/bancor
|
||||
@ -1,3 +1,5 @@
|
||||
[celery]
|
||||
broker_url = filesystem://
|
||||
result_url = filesystem://
|
||||
#broker_url = redis://
|
||||
#result_url = redis://
|
||||
|
||||
4
apps/cic-eth/config/test/cic.ini
Normal file
4
apps/cic-eth/config/test/cic.ini
Normal file
@ -0,0 +1,4 @@
|
||||
[cic]
|
||||
registry_address =
|
||||
chain_spec =
|
||||
trust_address =
|
||||
2
apps/cic-eth/config/test/dispatcher.ini
Normal file
2
apps/cic-eth/config/test/dispatcher.ini
Normal file
@ -0,0 +1,2 @@
|
||||
[dispatcher]
|
||||
loop_interval = 0.1
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user