kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge "origin/master" into "lash/improve-cache"

Browse Source
This commit is contained in:
philip 2021-12-20 11:14:04 +03:00
parent 9050d331cd
commit d6346bb87b
Signed by untrusted user: mango-habanero
GPG Key ID: B00CE9034DA19FB7
18 changed files with 115 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
README.md
View File

@ -1,41 +1,19 @@
# cic-internal-integration # Community Inclusion Currency Stack (CIC Stack)
A custodial evm wallet for executing transactions via USSD
## Getting started ## Getting started
This repo uses docker-compose and docker buildkit. Set the following environment variables to get started: This repo uses docker-compose and docker buildkit. Set the following environment variables to get started:
``` ```
export COMPOSE_DOCKER_CLI_BUILD=1 export COMPOSE_DOCKER_CLI_BUILD=1
export DOCKER_BUILDKIT=1 export DOCKER_BUILDKIT=1
``` ```
start services, database, redis and local ethereum node To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md)
```
docker-compose up -d
```
Run app/contract-migration to deploy contracts ## Documentation
```
RUN_MASK=3 docker-compose up contract-migration
```
stop cluster [https://docs.grassecon.org/cic_stack/](https://docs.grassecon.org/cic_stack/)
```
docker-compose down
```
stop cluster and delete data
```
docker-compose down -v --remove-orphans
```
rebuild an images
```
docker-compose up --build <service_name>
```
to delete the buildkit cache
```
docker builder prune --filter type=exec.cachemount
```

2
apps/cic-cache/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt -r test_requirements.txt

22
apps/cic-cache/setup.py
View File

@ -1,18 +1,19 @@
from setuptools import setup from setuptools import setup
#import configparser # import configparser
import os import os
#import time
#from cic_cache.version import ( # import time
# from cic_cache.version import (
# version_object, # version_object,
# version_string # version_string
# ) # )
# #
#class PleaseCommitFirstError(Exception): # class PleaseCommitFirstError(Exception):
# pass # pass
# #
#def git_hash(): # def git_hash():
# import subprocess # import subprocess
# git_diff = subprocess.run(['git', 'diff'], capture_output=True) # git_diff = subprocess.run(['git', 'diff'], capture_output=True)
# if len(git_diff.stdout) > 0: # if len(git_diff.stdout) > 0:
@ -21,18 +22,18 @@ import os
# git_hash_brief = git_hash.stdout.decode('utf-8')[:8] # git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
# return git_hash_brief # return git_hash_brief
# #
#version_string = str(version_object) # version_string = str(version_object)
# #
#try: # try:
# version_git = git_hash() # version_git = git_hash()
# version_string += '+build.{}'.format(version_git) # version_string += '+build.{}'.format(version_git)
#except FileNotFoundError: # except FileNotFoundError:
# time_string_pair = str(time.time()).split('.') # time_string_pair = str(time.time()).split('.')
# version_string += '+build.{}{:<09d}'.format( # version_string += '+build.{}{:<09d}'.format(
# time_string_pair[0], # time_string_pair[0],
# int(time_string_pair[1]), # int(time_string_pair[1]),
# ) # )
#print('final version string will be {}'.format(version_string)) # print('final version string will be {}'.format(version_string))
requirements = [] requirements = []
f = open('requirements.txt', 'r') f = open('requirements.txt', 'r')
@ -52,9 +53,8 @@ while True:
test_requirements.append(l.rstrip()) test_requirements.append(l.rstrip())
f.close() f.close()
setup( setup(
# version=version_string, # version=version_string,
install_requires=requirements, install_requires=requirements,
tests_require=test_requirements, tests_require=test_requirements,
) )

18
apps/cic-eth/docker/Dockerfile
View File

@ -11,13 +11,6 @@ ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS="" ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple ARG PIP_INDEX_URL=https://pypi.org/simple
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \
--pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a7
COPY *requirements.txt ./ COPY *requirements.txt ./
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \ pip install --index-url $PIP_INDEX_URL \
@ -25,8 +18,8 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt \ -r requirements.txt \
-r services_requirements.txt \ -r services_requirements.txt \
-r admin_requirements.txt -r admin_requirements.txt
COPY . . COPY . .
RUN python setup.py install RUN python setup.py install
@ -40,8 +33,6 @@ RUN chmod 755 *.sh
# # they can all be overridden by environment variables # # they can all be overridden by environment variables
# # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package) # # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
#COPY config/ /usr/local/etc/cic-eth/ #COPY config/ /usr/local/etc/cic-eth/
COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/
# TODO this kind of code sharing across projects should be discouraged...can we make util a library? # TODO this kind of code sharing across projects should be discouraged...can we make util a library?
#COPY util/liveness/health.sh /usr/local/bin/health.sh #COPY util/liveness/health.sh /usr/local/bin/health.sh
@ -66,9 +57,8 @@ ENTRYPOINT []
## # they can all be overridden by environment variables ## # they can all be overridden by environment variables
## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package) ## # to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
#COPY config/ /usr/local/etc/cic-eth/ #COPY config/ /usr/local/etc/cic-eth/
#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/
#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ #COPY scripts/ scripts/
#COPY scripts/ scripts/
# #
## TODO this kind of code sharing across projects should be discouraged...can we make util a library? ## TODO this kind of code sharing across projects should be discouraged...can we make util a library?
##COPY util/liveness/health.sh /usr/local/bin/health.sh ##COPY util/liveness/health.sh /usr/local/bin/health.sh

2
apps/cic-eth/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r admin_requirements.txt \ -r admin_requirements.txt \
-r services_requirements.txt \ -r services_requirements.txt \
-r test_requirements.txt -r test_requirements.txt

2
apps/cic-eth/tests/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r admin_requirements.txt -r admin_requirements.txt
-r services_requirements.txt -r services_requirements.txt
-r test_requirements.txt -r test_requirements.txt

2
apps/cic-notify/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt -r test_requirements.txt

9
apps/cic-ussd/docker/Dockerfile
View File

@ -14,19 +14,12 @@ ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS="" ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple ARG PIP_INDEX_URL=https://pypi.org/simple
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \
--pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
cic-eth-aux-erc20-demurrage-token~=0.0.2a7
COPY *requirements.txt ./ COPY *requirements.txt ./
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \ pip install --index-url $PIP_INDEX_URL \
--pre \ --pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt -r requirements.txt
COPY . . COPY . .

2
apps/cic-ussd/docker/run_tests.sh
View File

@ -3,7 +3,7 @@
set -e set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt -r test_requirements.txt

13
apps/contract-migration/README.md
View File

@ -27,6 +27,7 @@ In the current version of the scripts, two token types may be deployed; [`giftab
This step may be run multiple times, as long as the token symbol is different from all previously deployed tokens. This step may be run multiple times, as long as the token symbol is different from all previously deployed tokens.
## 4. Initialize custodial engine. ## 4. Initialize custodial engine.
Adds system accounts to the custodial engine, and unlocks the initialization seal. After this step, the custodial system is ready to use. Adds system accounts to the custodial engine, and unlocks the initialization seal. After this step, the custodial system is ready to use.
@ -35,10 +36,12 @@ Adds system accounts to the custodial engine, and unlocks the initialization sea
## Services dependency graph ## Services dependency graph
1. evm 1. evm
2. bootstrap runlevel 1 2. bootstrap runlevel 1 - deploy global contracts (RUN_MASK=1 docker-compose up bootstrap)
3. bootstrap runlevel 2 3. bootstrap runlevel 2 - deploy instance contracts (RUN_MASK=2 docker-compose up bootstrap)
4. bootstrap runlevel 3 4. bootstrap runlevel 4 - deploy token (RUN_MASK=4 docker-compose up bootstrap)
5. redis 5. redis
6. postgres 6. postgres
7. cic-eth-tasker 7. cic-eth-tasker
8. boostrap runlevel 4 8. boostrap runlevel 8 - deploy custodial contracts (RUN_MASK=8 docker-compose up bootstrap)
9. boostrap runlevel 16 - data seeding for development (RUN_MASK=16 docker-compose up bootstrap)
10. bring up the remainig services (docker-compose up -d)

13
apps/contract-migration/docker/Dockerfile
View File

@ -19,23 +19,14 @@ COPY requirements.txt .
#RUN apt-get install libffi-dev #RUN apt-get install libffi-dev
ARG pip_index_url=https://pypi.org/simple ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433" ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
ARG EXTRA_PIP_ARGS="" ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple" ARG PIP_INDEX_URL="https://pypi.org/simple"
ARG pip_trusted_host=pypi.org ARG pip_trusted_host=pypi.org
RUN pip install --index-url $PIP_INDEX_URL \ RUN pip install --index-url $PIP_INDEX_URL \
--pre \ --pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt -r requirements.txt
COPY override_requirements.txt .
RUN pip install --index-url $PIP_INDEX_URL \
--pre \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
--force-reinstall \
--no-cache \
-r override_requirements.txt
RUN pip freeze RUN pip freeze

93
apps/data-seeding/README.md
View File

@ -6,24 +6,22 @@ This folder contains tools to generate and import test data.
Three sets of tools are available, sorted by respective subdirectories. Three sets of tools are available, sorted by respective subdirectories.
* **eth**: Import using sovereign wallets. - **eth**: Import using sovereign wallets.
* **cic_eth**: Import using the `cic_eth` custodial engine. - **cic_eth**: Import using the `cic_eth` custodial engine.
* **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`) - **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`)
Each of the modules include two main scripts: Each of the modules include two main scripts:
* **import_users.py**: Registers all created accounts in the network - **import_users.py**: Registers all created accounts in the network
* **import_balance.py**: Transfer an opening balance using an external keystore wallet - **import_balance.py**: Transfer an opening balance using an external keystore wallet
The balance script will sync with the blockchain, processing transactions and triggering actions when it finds. In its current version it does not keep track of any other state, so it will run indefinitly and needs You the Human to decide when it has done what it needs to do. The balance script will sync with the blockchain, processing transactions and triggering actions when it finds. In its current version it does not keep track of any other state, so it will run indefinitly and needs You the Human to decide when it has done what it needs to do.
In addition the following common tools are available: In addition the following common tools are available:
* **create_import_users.py**: User creation script - **create_import_users.py**: User creation script
* **verify.py**: Import verification script - **verify.py**: Import verification script
* **cic_meta**: Metadata imports - **cic_meta**: Metadata imports
## REQUIREMENTS ## REQUIREMENTS
@ -36,14 +34,12 @@ source .venv/bin/activate
Install all requirements from the `requirements.txt` file: Install all requirements from the `requirements.txt` file:
`pip install --extra-index-url https://pip.grassrootseconomics.net:8433 -r requirements.txt` `pip install --extra-index-url https://pip.grassrootseconomics.net -r requirements.txt`
If you are importing metadata, also do ye olde: If you are importing metadata, also do ye olde:
`npm install` `npm install`
## HOW TO USE ## HOW TO USE
### Step 1 - Data creation ### Step 1 - Data creation
@ -60,7 +56,6 @@ If you want to use a `import_balance.py` script to add to the user's balance fro
`python create_import_users.py --gift-threshold <max_units_to_send> [--dir <datadir>] <number_of_users>` `python create_import_users.py --gift-threshold <max_units_to_send> [--dir <datadir>] <number_of_users>`
### Step 2 - Services ### Step 2 - Services
Unless you know what you are doing, start with a clean slate, and execute (in the repository root): Unless you know what you are doing, start with a clean slate, and execute (in the repository root):
@ -72,46 +67,46 @@ Then go through, in sequence:
#### Base requirements #### Base requirements
If you are importing using `eth` and _not_ importing metadata, then the only service you need running in the cluster is: If you are importing using `eth` and _not_ importing metadata, then the only service you need running in the cluster is:
* eth
- eth
In all other cases you will _also_ need: In all other cases you will _also_ need:
* postgres
* redis
- postgres
- redis
#### EVM provisions #### EVM provisions
This step is needed in *all* cases. This step is needed in _all_ cases.
`RUN_MASK=1 docker-compose up contract-migration` `RUN_MASK=1 docker-compose up contract-migration`
After this step is run, you can find top-level ethereum addresses (like the cic registry address, which you will need below) in `<repository_root>/service-configs/.env` After this step is run, you can find top-level ethereum addresses (like the cic registry address, which you will need below) in `<repository_root>/service-configs/.env`
#### Custodial provisions #### Custodial provisions
This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd` This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
`RUN_MASK=2 docker-compose up contract-migration` `RUN_MASK=2 docker-compose up contract-migration`
#### Custodial services #### Custodial services
If importing using `cic_eth` or `cic_ussd` also run: If importing using `cic_eth` or `cic_ussd` also run:
* cic-eth-tasker
* cic-eth-dispatcher - cic-eth-tasker
* cic-eth-tracker - cic-eth-dispatcher
* cic-eth-retrier - cic-eth-tracker
- cic-eth-retrier
If importing using `cic_ussd` also run: If importing using `cic_ussd` also run:
* cic-user-tasker
* cic-user-ussd-server - cic-user-tasker
* cic-notify-tasker - cic-user-ussd-server
- cic-notify-tasker
If metadata is to be imported, also run: If metadata is to be imported, also run:
* cic-meta-server
- cic-meta-server
### Step 3 - User imports ### Step 3 - User imports
@ -125,10 +120,7 @@ All external balance transactions are saved in raw wire format in `<datadir>/txs
If the contract migrations have been executed with the default "giftable" token contract, then the `token_symbol` in the `import_balance` scripts should be set to `GFT`. If the contract migrations have been executed with the default "giftable" token contract, then the `token_symbol` in the `import_balance` scripts should be set to `GFT`.
#### Alternative 1 - Sovereign wallet import - `eth`
#### Alternative 1 - Sovereign wallet import - `eth`
First, make a note of the **block height** before running anything: First, make a note of the **block height** before running anything:
@ -142,9 +134,7 @@ After the script completes, keystore files for all generated accouts will be fou
Then run: Then run:
`python eth/import_balance.py -v -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>` `python eth/import_balance.py -v -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
#### Alternative 2 - Custodial engine import - `cic_eth` #### Alternative 2 - Custodial engine import - `cic_eth`
@ -158,7 +148,6 @@ In another terminal:
The `redis_hostname_in_docker` value is the hostname required to reach the redis server from within the docker cluster, and should be `redis` if you left the docker-compose unchanged. The `import_users` script will receive the address of each newly created custodial account on a redis subscription fed by a callback task in the `cic_eth` account creation task chain. The `redis_hostname_in_docker` value is the hostname required to reach the redis server from within the docker cluster, and should be `redis` if you left the docker-compose unchanged. The `import_users` script will receive the address of each newly created custodial account on a redis subscription fed by a callback task in the `cic_eth` account creation task chain.
#### Alternative 3 - USSD import - `cic_ussd` #### Alternative 3 - USSD import - `cic_ussd`
If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`. If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`.
@ -177,13 +166,10 @@ In the event that you are running the command in a local environment you may wan
`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out` `python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
### Step 4 - Metadata import (optional) ### Step 4 - Metadata import (optional)
The metadata import scripts can be run at any time after step 1 has been completed. The metadata import scripts can be run at any time after step 1 has been completed.
#### Importing user metadata #### Importing user metadata
To import the main user metadata structs, run: To import the main user metadata structs, run:
@ -194,22 +180,18 @@ Monitors a folder for output from the `import_users.py` script, adding the metad
If _number of users_ is omitted the script will run until manually interrupted. If _number of users_ is omitted the script will run until manually interrupted.
#### Importing phone pointer #### Importing phone pointer
`node cic_meta/import_meta_phone.js <datadir> <number_of_users>` `node cic_meta/import_meta_phone.js <datadir> <number_of_users>`
If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing. If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
### Importing preferences metadata ### Importing preferences metadata
`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>` `node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported. If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
##### Importing pins and ussd data (optional) ##### Importing pins and ussd data (optional)
Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps: Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
@ -228,26 +210,26 @@ Once the creation of the pins file is complete, proceed to import the pins and u
`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>` `python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
- To import ussd data: - To import ussd data:
`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>` `python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log. The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file. The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
### Step 5 - Verify ### Step 5 - Verify
`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>` `python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>`
Included checks: Included checks:
* Private key is in cic-eth keystore
* Address is in accounts index - Private key is in cic-eth keystore
* Address has gas balance - Address is in accounts index
* Address has triggered the token faucet - Address has gas balance
* Address has token balance matching the gift threshold - Address has triggered the token faucet
* Personal metadata can be retrieved and has exact match - Address has token balance matching the gift threshold
* Phone pointer metadata can be retrieved and matches address - Personal metadata can be retrieved and has exact match
* USSD menu response is initial state after registration - Phone pointer metadata can be retrieved and matches address
- USSD menu response is initial state after registration
Checks can be selectively included and excluded. See `--help` for details. Checks can be selectively included and excluded. See `--help` for details.
@ -255,7 +237,6 @@ Will output one line for each check, with name of check and number of errors fou
Should exit with code 0 if all input data is found in the respective services. Should exit with code 0 if all input data is found in the respective services.
## KNOWN ISSUES ## KNOWN ISSUES
- If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount. - If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount.

53
apps/data-seeding/cic_eth/import_balance.py
View File

@ -1,48 +1,37 @@
# standard imports # standard imports
import argparse
import json
import logging
import os import os
import sys import sys
import logging
import time
import argparse
import sys
import re
import hashlib
import csv
import json
# external imports # external imports
import confini import confini
from hexathon import ( from chainlib.chain import ChainSpec
strip_0x,
add_0x,
)
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver.head import HeadSyncer
from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.block import (
block_latest,
)
from chainlib.hash import keccak256_string_to_hex
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
from chainlib.eth.block import (
block_latest,
)
from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.error import (
RequestMismatchException,
)
from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.gas import OverrideGasOracle
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.tx import TxFactory from chainlib.hash import keccak256_string_to_hex
from chainlib.jsonrpc import JSONRPCRequest from chainsyncer.backend.memory import MemBackend
from chainlib.eth.error import ( from chainsyncer.driver.head import HeadSyncer
EthException,
RequestMismatchException,
)
from chainlib.chain import ChainSpec
from chainlib.eth.constant import ZERO_ADDRESS
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore
from cic_types.models.person import Person
from eth_erc20 import ERC20
from cic_eth.cli.chain import chain_interface from cic_eth.cli.chain import chain_interface
from cic_types.models.person import Person
from eth_accounts_index import AccountsIndex from eth_accounts_index import AccountsIndex
from eth_contract_registry import Registry from eth_contract_registry import Registry
from eth_erc20 import ERC20
from eth_token_index import TokenUniqueSymbolIndex from eth_token_index import TokenUniqueSymbolIndex
from funga.eth.keystore.dict import DictKeystore
from funga.eth.signer import EIP155Signer
from hexathon import (
strip_0x,
)
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()

4
apps/data-seeding/cic_eth/traffic/common/signer.py
View File

@ -2,8 +2,8 @@
import logging import logging
# external imports # external imports
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from funga.eth.signer import EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore from funga.eth.keystore.dict import DictKeystore
logg = logging.getLogger(__name__) logg = logging.getLogger(__name__)

4
apps/data-seeding/cic_ussd/import_balance.py
View File

@ -9,8 +9,8 @@ from chainlib.chain import ChainSpec
from chainlib.eth.address import to_checksum_address from chainlib.eth.address import to_checksum_address
from chainlib.eth.connection import EthHTTPConnection from chainlib.eth.connection import EthHTTPConnection
from confini import Config from confini import Config
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from funga.eth.signer import EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore from funga.eth.keystore.dict import DictKeystore
# local imports # local imports
from import_util import BalanceProcessor, get_celery_worker_status from import_util import BalanceProcessor, get_celery_worker_status

38
apps/data-seeding/docker/Dockerfile
View File

@ -1,46 +1,30 @@
ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" # syntax = docker/dockerfile:1.2
FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45
FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
WORKDIR /root WORKDIR /root
RUN mkdir -vp /usr/local/etc/cic RUN mkdir -vp /usr/local/etc/cic
ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org} COPY package.json \
RUN npm config set snyk=false package-lock.json \
#RUN npm config set registry={NPM_REPOSITORY} ./
RUN npm config set registry=${NPM_REPOSITORY}
# copy the dependencies
COPY package.json package-lock.json ./
RUN --mount=type=cache,mode=0755,target=/root/.npm \
npm set cache /root/.npm && \
npm cache verify && \
npm ci --verbose
#RUN npm ci --production --verbose RUN npm ci --production
#RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install
COPY common/ cic_ussd/common/ COPY common/ cic_ussd/common/
COPY requirements.txt . COPY requirements.txt .
COPY config/ config COPY config/ config
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_ARGS="" ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple ARG PIP_INDEX_URL=https://pypi.org/simple
RUN pip install --index-url $PIP_INDEX_URL \ RUN pip install --index-url $PIP_INDEX_URL \
--pre \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt
-r requirements.txt
COPY . .
#RUN pip install --index-url $PIP_INDEX_URL \
# --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
# -r requirements.txt
COPY . .
ENTRYPOINT [ ] ENTRYPOINT [ ]

4
apps/data-seeding/eth/import_balance.py
View File

@ -33,8 +33,8 @@ from chainlib.eth.error import (
RequestMismatchException, RequestMismatchException,
) )
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer from funga.eth.signer import EIP155Signer
from crypto_dev_signer.keystore.dict import DictKeystore from funga.eth.keystore.dict import DictKeystore
from cic_types.models.person import Person from cic_types.models.person import Person
from eth_erc20 import ERC20 from eth_erc20 import ERC20
from cic_eth.cli.chain import chain_interface from cic_eth.cli.chain import chain_interface

6
apps/data-seeding/eth/import_users.py
View File

@ -27,9 +27,9 @@ from cic_types.processor import generate_metadata_pointer
from cic_types import MetadataPointer from cic_types import MetadataPointer
from eth_accounts_index.registry import AccountRegistry from eth_accounts_index.registry import AccountRegistry
from eth_contract_registry import Registry from eth_contract_registry import Registry
from crypto_dev_signer.keystore.dict import DictKeystore from funga.eth.keystore.dict import DictKeystore
from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer from funga.eth.signer.defaultsigner import EIP155Signer
from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict from funga.eth.keystore.keyfile import to_dict as to_keyfile_dict
# local imports # local imports
from common.dirs import initialize_dirs from common.dirs import initialize_dirs