kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

remove port from EXTRA_PIP_INDEX_URL

Browse Source
This commit is contained in:
William Luke 2021-12-08 15:53:54 +03:00
parent 2ead1b0e51
commit 6df51f12e1
16 changed files with 70 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/docker/Dockerfile
View File

@ -4,7 +4,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
COPY requirements.txt .
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"

2
apps/cic-cache/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt

2
apps/cic-eth/docker/Dockerfile
View File

@ -7,7 +7,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
# TODO can we take all the requirements out of setup.py and just do a pip install -r requirements.txt && python setup.py
#COPY cic-eth/requirements.txt .
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple

2
apps/cic-eth/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r admin_requirements.txt \
-r services_requirements.txt \
-r test_requirements.txt

2
apps/cic-eth/tests/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple
-r admin_requirements.txt
-r services_requirements.txt
-r test_requirements.txt

2
apps/cic-notify/docker/Dockerfile
View File

@ -6,7 +6,7 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
RUN apt-get install libffi-dev -y
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple

2
apps/cic-notify/docker/run_tests.sh
View File

@ -2,7 +2,7 @@
set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt

2
apps/cic-signer/Dockerfile
View File

@ -8,7 +8,7 @@ RUN apt-get install libffi-dev -y
COPY requirements.txt .
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \

2
apps/cic-ussd/config/pip.ini
View File

@ -1,5 +1,5 @@
[PIP]
extra_index_host = pip.grassrootseconomics.net
extra_index_port = 8433
extra_index_port =
extra_index_path = /
extra_index_proto = https

2
apps/cic-ussd/config/test/pip.ini
View File

@ -1,5 +1,5 @@
[PIP]
extra_index_host = pip.grassrootseconomics.net
extra_index_port = 8433
extra_index_port =
extra_index_path = /
extra_index_proto = https

2
apps/cic-ussd/docker/Dockerfile
View File

@ -10,7 +10,7 @@ RUN mkdir -vp pgp/keys
RUN mkdir -vp cic-ussd
RUN mkdir -vp data
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple

2
apps/cic-ussd/docker/run_tests.sh
View File

@ -3,7 +3,7 @@
set -e
pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \
pip install --extra-index-url https://pip.grassrootseconomics.net \
--extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \
-r test_requirements.txt

2
apps/contract-migration/docker/Dockerfile
View File

@ -19,7 +19,7 @@ COPY requirements.txt .
#RUN apt-get install libffi-dev
ARG pip_index_url=https://pypi.org/simple
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"
ARG pip_trusted_host=pypi.org

87
apps/data-seeding/README.md
View File

@ -6,24 +6,22 @@ This folder contains tools to generate and import test data.
Three sets of tools are available, sorted by respective subdirectories.
* **eth**: Import using sovereign wallets.
* **cic_eth**: Import using the `cic_eth` custodial engine.
* **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`)
- **eth**: Import using sovereign wallets.
- **cic_eth**: Import using the `cic_eth` custodial engine.
- **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`)
Each of the modules include two main scripts:
* **import_users.py**: Registers all created accounts in the network
* **import_balance.py**: Transfer an opening balance using an external keystore wallet
- **import_users.py**: Registers all created accounts in the network
- **import_balance.py**: Transfer an opening balance using an external keystore wallet
The balance script will sync with the blockchain, processing transactions and triggering actions when it finds. In its current version it does not keep track of any other state, so it will run indefinitly and needs You the Human to decide when it has done what it needs to do.
In addition the following common tools are available:
* **create_import_users.py**: User creation script
* **verify.py**: Import verification script
* **cic_meta**: Metadata imports
- **create_import_users.py**: User creation script
- **verify.py**: Import verification script
- **cic_meta**: Metadata imports
## REQUIREMENTS
@ -36,14 +34,12 @@ source .venv/bin/activate
Install all requirements from the `requirements.txt` file:
`pip install --extra-index-url https://pip.grassrootseconomics.net:8433 -r requirements.txt`
`pip install --extra-index-url https://pip.grassrootseconomics.net -r requirements.txt`
If you are importing metadata, also do ye olde:
`npm install`
## HOW TO USE
### Step 1 - Data creation
@ -60,7 +56,6 @@ If you want to use a `import_balance.py` script to add to the user's balance fro
`python create_import_users.py --gift-threshold <max_units_to_send> [--dir <datadir>] <number_of_users>`
### Step 2 - Services
Unless you know what you are doing, start with a clean slate, and execute (in the repository root):
@ -72,46 +67,46 @@ Then go through, in sequence:
#### Base requirements
If you are importing using `eth` and _not_ importing metadata, then the only service you need running in the cluster is:
* eth
- eth
In all other cases you will _also_ need:
* postgres
* redis
- postgres
- redis
#### EVM provisions
This step is needed in *all* cases.
This step is needed in _all_ cases.
`RUN_MASK=1 docker-compose up contract-migration`
After this step is run, you can find top-level ethereum addresses (like the cic registry address, which you will need below) in `<repository_root>/service-configs/.env`
#### Custodial provisions
This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd`
`RUN_MASK=2 docker-compose up contract-migration`
#### Custodial services
If importing using `cic_eth` or `cic_ussd` also run:
* cic-eth-tasker
* cic-eth-dispatcher
* cic-eth-tracker
* cic-eth-retrier
- cic-eth-tasker
- cic-eth-dispatcher
- cic-eth-tracker
- cic-eth-retrier
If importing using `cic_ussd` also run:
* cic-user-tasker
* cic-user-ussd-server
* cic-notify-tasker
- cic-user-tasker
- cic-user-ussd-server
- cic-notify-tasker
If metadata is to be imported, also run:
* cic-meta-server
- cic-meta-server
### Step 3 - User imports
@ -125,11 +120,8 @@ All external balance transactions are saved in raw wire format in `<datadir>/txs
If the contract migrations have been executed with the default "giftable" token contract, then the `token_symbol` in the `import_balance` scripts should be set to `GFT`.
#### Alternative 1 - Sovereign wallet import - `eth`
First, make a note of the **block height** before running anything:
`eth-info -p http://localhost:63545`
@ -144,8 +136,6 @@ Then run:
`python eth/import_balance.py -v -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> --offset <block_height_at_start> -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c <datadir>`
#### Alternative 2 - Custodial engine import - `cic_eth`
Run in sequence, in first terminal:
@ -158,7 +148,6 @@ In another terminal:
The `redis_hostname_in_docker` value is the hostname required to reach the redis server from within the docker cluster, and should be `redis` if you left the docker-compose unchanged. The `import_users` script will receive the address of each newly created custodial account on a redis subscription fed by a callback task in the `cic_eth` account creation task chain.
#### Alternative 3 - USSD import - `cic_ussd`
If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`.
@ -177,13 +166,10 @@ In the event that you are running the command in a local environment you may wan
`python cic_ussd/import_users.py -v --ussd-host <user_ussd_server_host> --ussd-port <user_ussd_server_port> --ussd-no-ssl -c config out`
### Step 4 - Metadata import (optional)
The metadata import scripts can be run at any time after step 1 has been completed.
#### Importing user metadata
To import the main user metadata structs, run:
@ -194,22 +180,18 @@ Monitors a folder for output from the `import_users.py` script, adding the metad
If _number of users_ is omitted the script will run until manually interrupted.
#### Importing phone pointer
`node cic_meta/import_meta_phone.js <datadir> <number_of_users>`
If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing.
### Importing preferences metadata
`node cic_meta/import_meta_preferences.js <datadir> <number_of_users>`
If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported.
##### Importing pins and ussd data (optional)
Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps:
@ -228,26 +210,26 @@ Once the creation of the pins file is complete, proceed to import the pins and u
`python cic_ussd/import_pins.py -c config -v pinsdir <path to pin export dir tree>`
- To import ussd data:
`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
`python cic_ussd/import_ussd_data.py -c config -v userdir <path to the users export dir tree>`
The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log.
The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file.
### Step 5 - Verify
`python verify.py -v -c config -r <cic_registry_address> -p <eth_provider> --token-symbol <token_symbol> <datadir>`
Included checks:
* Private key is in cic-eth keystore
* Address is in accounts index
* Address has gas balance
* Address has triggered the token faucet
* Address has token balance matching the gift threshold
* Personal metadata can be retrieved and has exact match
* Phone pointer metadata can be retrieved and matches address
* USSD menu response is initial state after registration
- Private key is in cic-eth keystore
- Address is in accounts index
- Address has gas balance
- Address has triggered the token faucet
- Address has token balance matching the gift threshold
- Personal metadata can be retrieved and has exact match
- Phone pointer metadata can be retrieved and matches address
- USSD menu response is initial state after registration
Checks can be selectively included and excluded. See `--help` for details.
@ -255,7 +237,6 @@ Will output one line for each check, with name of check and number of errors fou
Should exit with code 0 if all input data is found in the respective services.
## KNOWN ISSUES
- If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount.

2
apps/data-seeding/docker/Dockerfile
View File

@ -17,7 +17,7 @@ COPY common/ cic_ussd/common/
COPY requirements.txt .
COPY config/ config
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL=https://pypi.org/simple

28
docker-compose.yml
View File

@ -47,7 +47,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DEV_DATA_DIR: ${DEV_DATA_DIR:-/tmp/cic/config}
@ -100,7 +100,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -130,7 +130,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
@ -181,7 +181,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
REDIS_PORT: 6379
@ -216,7 +216,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
@ -264,7 +264,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
@ -308,7 +308,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
RPC_PROVIDER: ${RPC_PROVIDER:-http://evm:8545}
@ -355,7 +355,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
@ -403,7 +403,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
CIC_REGISTRY_ADDRESS: $CIC_REGISTRY_ADDRESS
@ -451,7 +451,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -570,7 +570,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -600,7 +600,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -640,7 +640,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}
@ -671,7 +671,7 @@ services:
args:
DOCKER_REGISTRY: ${DEV_DOCKER_REGISTRY:-registry.gitlab.com/grassrootseconomics}
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.org/simple}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net:8433}
EXTRA_PIP_INDEX_URL: ${EXTRA_PIP_INDEX_URL:-https://pip.grassrootseconomics.net}
EXTRA_PIP_ARGS: $EXTRA_PIP_ARGS
environment:
DATABASE_HOST: ${DATABASE_HOST:-postgres}