From d6346bb87b8f5646263d8cc139f0489f6ea16277 Mon Sep 17 00:00:00 2001 From: philip Date: Mon, 20 Dec 2021 11:14:04 +0300 Subject: [PATCH] Merge "origin/master" into "lash/improve-cache" --- README.md | 34 ++----- apps/cic-cache/docker/run_tests.sh | 2 +- apps/cic-cache/setup.py | 22 ++--- apps/cic-eth/docker/Dockerfile | 18 +--- apps/cic-eth/docker/run_tests.sh | 2 +- apps/cic-eth/tests/run_tests.sh | 2 +- apps/cic-notify/docker/run_tests.sh | 2 +- apps/cic-ussd/docker/Dockerfile | 9 +- apps/cic-ussd/docker/run_tests.sh | 2 +- apps/contract-migration/README.md | 13 ++- apps/contract-migration/docker/Dockerfile | 13 +-- apps/data-seeding/README.md | 93 ++++++++----------- apps/data-seeding/cic_eth/import_balance.py | 53 +++++------ .../cic_eth/traffic/common/signer.py | 4 +- apps/data-seeding/cic_ussd/import_balance.py | 4 +- apps/data-seeding/docker/Dockerfile | 38 +++----- apps/data-seeding/eth/import_balance.py | 4 +- apps/data-seeding/eth/import_users.py | 6 +- 18 files changed, 115 insertions(+), 206 deletions(-) diff --git a/README.md b/README.md index a7966dc9..bc5ffdea 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,19 @@ -# cic-internal-integration +# Community Inclusion Currency Stack (CIC Stack) + +A custodial evm wallet for executing transactions via USSD ## Getting started This repo uses docker-compose and docker buildkit. Set the following environment variables to get started: - ``` export COMPOSE_DOCKER_CLI_BUILD=1 export DOCKER_BUILDKIT=1 ``` -start services, database, redis and local ethereum node -``` -docker-compose up -d -``` +To get started see [./apps/contract-migration/README.md](./apps/contract-migration/README.md) -Run app/contract-migration to deploy contracts -``` -RUN_MASK=3 docker-compose up contract-migration -``` +## Documentation -stop cluster -``` -docker-compose down -``` +[https://docs.grassecon.org/cic_stack/](https://docs.grassecon.org/cic_stack/) -stop cluster and delete data -``` -docker-compose down -v --remove-orphans -``` - -rebuild an images -``` -docker-compose up --build -``` - -to delete the buildkit cache -``` -docker builder prune --filter type=exec.cachemount -``` diff --git a/apps/cic-cache/docker/run_tests.sh b/apps/cic-cache/docker/run_tests.sh index 62c1db72..6b26aeef 100644 --- a/apps/cic-cache/docker/run_tests.sh +++ b/apps/cic-cache/docker/run_tests.sh @@ -2,7 +2,7 @@ set -e -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +pip install --extra-index-url https://pip.grassrootseconomics.net \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ -r test_requirements.txt diff --git a/apps/cic-cache/setup.py b/apps/cic-cache/setup.py index 048a9220..81d76924 100644 --- a/apps/cic-cache/setup.py +++ b/apps/cic-cache/setup.py @@ -1,18 +1,19 @@ from setuptools import setup -#import configparser +# import configparser import os -#import time -#from cic_cache.version import ( +# import time + +# from cic_cache.version import ( # version_object, # version_string # ) # -#class PleaseCommitFirstError(Exception): +# class PleaseCommitFirstError(Exception): # pass # -#def git_hash(): +# def git_hash(): # import subprocess # git_diff = subprocess.run(['git', 'diff'], capture_output=True) # if len(git_diff.stdout) > 0: @@ -21,18 +22,18 @@ import os # git_hash_brief = git_hash.stdout.decode('utf-8')[:8] # return git_hash_brief # -#version_string = str(version_object) +# version_string = str(version_object) # -#try: +# try: # version_git = git_hash() # version_string += '+build.{}'.format(version_git) -#except FileNotFoundError: +# except FileNotFoundError: # time_string_pair = str(time.time()).split('.') # version_string += '+build.{}{:<09d}'.format( # time_string_pair[0], # int(time_string_pair[1]), # ) -#print('final version string will be {}'.format(version_string)) +# print('final version string will be {}'.format(version_string)) requirements = [] f = open('requirements.txt', 'r') @@ -52,9 +53,8 @@ while True: test_requirements.append(l.rstrip()) f.close() - setup( -# version=version_string, + # version=version_string, install_requires=requirements, tests_require=test_requirements, ) diff --git a/apps/cic-eth/docker/Dockerfile b/apps/cic-eth/docker/Dockerfile index 7e4fe418..d9f74c41 100644 --- a/apps/cic-eth/docker/Dockerfile +++ b/apps/cic-eth/docker/Dockerfile @@ -11,13 +11,6 @@ ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL=https://pypi.org/simple -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url $PIP_INDEX_URL \ - --pre \ - --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - cic-eth-aux-erc20-demurrage-token~=0.0.2a7 - - COPY *requirements.txt ./ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url $PIP_INDEX_URL \ @@ -25,8 +18,8 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -r requirements.txt \ -r services_requirements.txt \ - -r admin_requirements.txt - + -r admin_requirements.txt + COPY . . RUN python setup.py install @@ -40,8 +33,6 @@ RUN chmod 755 *.sh # # they can all be overridden by environment variables # # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) #COPY config/ /usr/local/etc/cic-eth/ -COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ -COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ # TODO this kind of code sharing across projects should be discouraged...can we make util a library? #COPY util/liveness/health.sh /usr/local/bin/health.sh @@ -66,9 +57,8 @@ ENTRYPOINT [] ## # they can all be overridden by environment variables ## # to generate a list of environment variables from configuration, use: confini-dump -z (executable provided by confini package) #COPY config/ /usr/local/etc/cic-eth/ -#COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ -#COPY crypto_dev_signer_config/ /usr/local/etc/crypto-dev-signer/ -#COPY scripts/ scripts/ +COPY cic_eth/db/migrations/ /usr/local/share/cic-eth/alembic/ +#COPY scripts/ scripts/ # ## TODO this kind of code sharing across projects should be discouraged...can we make util a library? ##COPY util/liveness/health.sh /usr/local/bin/health.sh diff --git a/apps/cic-eth/docker/run_tests.sh b/apps/cic-eth/docker/run_tests.sh index ef434d5c..96f692bd 100644 --- a/apps/cic-eth/docker/run_tests.sh +++ b/apps/cic-eth/docker/run_tests.sh @@ -2,7 +2,7 @@ set -e -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ +pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ -r admin_requirements.txt \ -r services_requirements.txt \ -r test_requirements.txt diff --git a/apps/cic-eth/tests/run_tests.sh b/apps/cic-eth/tests/run_tests.sh index 30500432..751f42cd 100644 --- a/apps/cic-eth/tests/run_tests.sh +++ b/apps/cic-eth/tests/run_tests.sh @@ -2,7 +2,7 @@ set -e -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple +pip install --extra-index-url https://pip.grassrootseconomics.net --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple -r admin_requirements.txt -r services_requirements.txt -r test_requirements.txt diff --git a/apps/cic-notify/docker/run_tests.sh b/apps/cic-notify/docker/run_tests.sh index 1c2e8678..03a066e2 100644 --- a/apps/cic-notify/docker/run_tests.sh +++ b/apps/cic-notify/docker/run_tests.sh @@ -2,7 +2,7 @@ set -e -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +pip install --extra-index-url https://pip.grassrootseconomics.net \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ -r test_requirements.txt diff --git a/apps/cic-ussd/docker/Dockerfile b/apps/cic-ussd/docker/Dockerfile index 5b9bf36f..634c4299 100644 --- a/apps/cic-ussd/docker/Dockerfile +++ b/apps/cic-ussd/docker/Dockerfile @@ -14,19 +14,12 @@ ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL=https://pypi.org/simple -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ - pip install --index-url $PIP_INDEX_URL \ - --pre \ - --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - cic-eth-aux-erc20-demurrage-token~=0.0.2a7 - - COPY *requirements.txt ./ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \ pip install --index-url $PIP_INDEX_URL \ --pre \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt + -r requirements.txt COPY . . diff --git a/apps/cic-ussd/docker/run_tests.sh b/apps/cic-ussd/docker/run_tests.sh index ab673411..d766ccd0 100644 --- a/apps/cic-ussd/docker/run_tests.sh +++ b/apps/cic-ussd/docker/run_tests.sh @@ -3,7 +3,7 @@ set -e -pip install --extra-index-url https://pip.grassrootseconomics.net:8433 \ +pip install --extra-index-url https://pip.grassrootseconomics.net \ --extra-index-url https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple \ -r test_requirements.txt diff --git a/apps/contract-migration/README.md b/apps/contract-migration/README.md index e3e72d6a..fd3a523f 100644 --- a/apps/contract-migration/README.md +++ b/apps/contract-migration/README.md @@ -27,6 +27,7 @@ In the current version of the scripts, two token types may be deployed; [`giftab This step may be run multiple times, as long as the token symbol is different from all previously deployed tokens. + ## 4. Initialize custodial engine. Adds system accounts to the custodial engine, and unlocks the initialization seal. After this step, the custodial system is ready to use. @@ -35,10 +36,12 @@ Adds system accounts to the custodial engine, and unlocks the initialization sea ## Services dependency graph 1. evm -2. bootstrap runlevel 1 -3. bootstrap runlevel 2 -4. bootstrap runlevel 3 +2. bootstrap runlevel 1 - deploy global contracts (RUN_MASK=1 docker-compose up bootstrap) +3. bootstrap runlevel 2 - deploy instance contracts (RUN_MASK=2 docker-compose up bootstrap) +4. bootstrap runlevel 4 - deploy token (RUN_MASK=4 docker-compose up bootstrap) 5. redis 6. postgres -7. cic-eth-tasker -8. boostrap runlevel 4 +7. cic-eth-tasker +8. boostrap runlevel 8 - deploy custodial contracts (RUN_MASK=8 docker-compose up bootstrap) +9. boostrap runlevel 16 - data seeding for development (RUN_MASK=16 docker-compose up bootstrap) +10. bring up the remainig services (docker-compose up -d) diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index cbc32124..35b22752 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -19,23 +19,14 @@ COPY requirements.txt . #RUN apt-get install libffi-dev ARG pip_index_url=https://pypi.org/simple -ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net:8433" +ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net" ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL="https://pypi.org/simple" ARG pip_trusted_host=pypi.org RUN pip install --index-url $PIP_INDEX_URL \ --pre \ --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt - -COPY override_requirements.txt . - -RUN pip install --index-url $PIP_INDEX_URL \ - --pre \ - --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - --force-reinstall \ - --no-cache \ - -r override_requirements.txt + -r requirements.txt RUN pip freeze diff --git a/apps/data-seeding/README.md b/apps/data-seeding/README.md index 959bd4fe..46c17722 100644 --- a/apps/data-seeding/README.md +++ b/apps/data-seeding/README.md @@ -6,24 +6,22 @@ This folder contains tools to generate and import test data. Three sets of tools are available, sorted by respective subdirectories. -* **eth**: Import using sovereign wallets. -* **cic_eth**: Import using the `cic_eth` custodial engine. -* **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`) +- **eth**: Import using sovereign wallets. +- **cic_eth**: Import using the `cic_eth` custodial engine. +- **cic_ussd**: Import using the `cic_ussd` interface (backed by `cic_eth`) Each of the modules include two main scripts: -* **import_users.py**: Registers all created accounts in the network -* **import_balance.py**: Transfer an opening balance using an external keystore wallet +- **import_users.py**: Registers all created accounts in the network +- **import_balance.py**: Transfer an opening balance using an external keystore wallet The balance script will sync with the blockchain, processing transactions and triggering actions when it finds. In its current version it does not keep track of any other state, so it will run indefinitly and needs You the Human to decide when it has done what it needs to do. - In addition the following common tools are available: -* **create_import_users.py**: User creation script -* **verify.py**: Import verification script -* **cic_meta**: Metadata imports - +- **create_import_users.py**: User creation script +- **verify.py**: Import verification script +- **cic_meta**: Metadata imports ## REQUIREMENTS @@ -36,14 +34,12 @@ source .venv/bin/activate Install all requirements from the `requirements.txt` file: -`pip install --extra-index-url https://pip.grassrootseconomics.net:8433 -r requirements.txt` - +`pip install --extra-index-url https://pip.grassrootseconomics.net -r requirements.txt` If you are importing metadata, also do ye olde: `npm install` - ## HOW TO USE ### Step 1 - Data creation @@ -60,7 +56,6 @@ If you want to use a `import_balance.py` script to add to the user's balance fro `python create_import_users.py --gift-threshold [--dir ] ` - ### Step 2 - Services Unless you know what you are doing, start with a clean slate, and execute (in the repository root): @@ -72,46 +67,46 @@ Then go through, in sequence: #### Base requirements If you are importing using `eth` and _not_ importing metadata, then the only service you need running in the cluster is: -* eth + +- eth In all other cases you will _also_ need: -* postgres -* redis +- postgres +- redis #### EVM provisions -This step is needed in *all* cases. +This step is needed in _all_ cases. `RUN_MASK=1 docker-compose up contract-migration` After this step is run, you can find top-level ethereum addresses (like the cic registry address, which you will need below) in `/service-configs/.env` - #### Custodial provisions This step is _only_ needed if you are importing using `cic_eth` or `cic_ussd` `RUN_MASK=2 docker-compose up contract-migration` - #### Custodial services If importing using `cic_eth` or `cic_ussd` also run: -* cic-eth-tasker -* cic-eth-dispatcher -* cic-eth-tracker -* cic-eth-retrier + +- cic-eth-tasker +- cic-eth-dispatcher +- cic-eth-tracker +- cic-eth-retrier If importing using `cic_ussd` also run: -* cic-user-tasker -* cic-user-ussd-server -* cic-notify-tasker + +- cic-user-tasker +- cic-user-ussd-server +- cic-notify-tasker If metadata is to be imported, also run: -* cic-meta-server - +- cic-meta-server ### Step 3 - User imports @@ -125,10 +120,7 @@ All external balance transactions are saved in raw wire format in `/txs If the contract migrations have been executed with the default "giftable" token contract, then the `token_symbol` in the `import_balance` scripts should be set to `GFT`. - - -#### Alternative 1 - Sovereign wallet import - `eth` - +#### Alternative 1 - Sovereign wallet import - `eth` First, make a note of the **block height** before running anything: @@ -142,9 +134,7 @@ After the script completes, keystore files for all generated accouts will be fou Then run: -`python eth/import_balance.py -v -r -p --token-symbol --offset -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` - - +`python eth/import_balance.py -v -r -p --token-symbol --offset -y ../keystore/UTC--2021-01-08T17-18-44.521011372Z--eb3907ecad74a0013c259d5874ae7f22dcbcc95c ` #### Alternative 2 - Custodial engine import - `cic_eth` @@ -158,7 +148,6 @@ In another terminal: The `redis_hostname_in_docker` value is the hostname required to reach the redis server from within the docker cluster, and should be `redis` if you left the docker-compose unchanged. The `import_users` script will receive the address of each newly created custodial account on a redis subscription fed by a callback task in the `cic_eth` account creation task chain. - #### Alternative 3 - USSD import - `cic_ussd` If you have previously run the `cic_ussd` import incompletely, it could be a good idea to purge the queue. If you have left docker-compose unchanged, `redis_url` should be `redis://localhost:63379`. @@ -177,13 +166,10 @@ In the event that you are running the command in a local environment you may wan `python cic_ussd/import_users.py -v --ussd-host --ussd-port --ussd-no-ssl -c config out` - - ### Step 4 - Metadata import (optional) The metadata import scripts can be run at any time after step 1 has been completed. - #### Importing user metadata To import the main user metadata structs, run: @@ -194,22 +180,18 @@ Monitors a folder for output from the `import_users.py` script, adding the metad If _number of users_ is omitted the script will run until manually interrupted. - - #### Importing phone pointer `node cic_meta/import_meta_phone.js ` If you imported using `cic_ussd`, the phone pointer is _already added_ and this script will do nothing. - ### Importing preferences metadata `node cic_meta/import_meta_preferences.js ` If you used the `cic_ussd/import_user.py` script to import your users, preferences metadata is generated and will be imported. - ##### Importing pins and ussd data (optional) Once the user imports are complete the next step should be importing the user's pins and auxiliary ussd data. This can be done in 3 steps: @@ -228,26 +210,26 @@ Once the creation of the pins file is complete, proceed to import the pins and u `python cic_ussd/import_pins.py -c config -v pinsdir ` - To import ussd data: -`python cic_ussd/import_ussd_data.py -c config -v userdir ` + `python cic_ussd/import_ussd_data.py -c config -v userdir ` The balance script is a celery task worker, and will not exit by itself in its current version. However, after it's done doing its job, you will find "reached nonce ... exiting" among the last lines of the log. The connection parameters for the `cic-ussd-server` is currently _hardcoded_ in the `import_users.py` script file. - ### Step 5 - Verify -`python verify.py -v -c config -r -p --token-symbol ` +`python verify.py -v -c config -r -p --token-symbol ` Included checks: -* Private key is in cic-eth keystore -* Address is in accounts index -* Address has gas balance -* Address has triggered the token faucet -* Address has token balance matching the gift threshold -* Personal metadata can be retrieved and has exact match -* Phone pointer metadata can be retrieved and matches address -* USSD menu response is initial state after registration + +- Private key is in cic-eth keystore +- Address is in accounts index +- Address has gas balance +- Address has triggered the token faucet +- Address has token balance matching the gift threshold +- Personal metadata can be retrieved and has exact match +- Phone pointer metadata can be retrieved and matches address +- USSD menu response is initial state after registration Checks can be selectively included and excluded. See `--help` for details. @@ -255,7 +237,6 @@ Will output one line for each check, with name of check and number of errors fou Should exit with code 0 if all input data is found in the respective services. - ## KNOWN ISSUES - If the faucet disbursement is set to a non-zero amount, the balances will be off. The verify script needs to be improved to check the faucet amount. diff --git a/apps/data-seeding/cic_eth/import_balance.py b/apps/data-seeding/cic_eth/import_balance.py index 28104341..4d471234 100644 --- a/apps/data-seeding/cic_eth/import_balance.py +++ b/apps/data-seeding/cic_eth/import_balance.py @@ -1,48 +1,37 @@ # standard imports +import argparse +import json +import logging import os import sys -import logging -import time -import argparse -import sys -import re -import hashlib -import csv -import json # external imports import confini -from hexathon import ( - strip_0x, - add_0x, - ) -from chainsyncer.backend.memory import MemBackend -from chainsyncer.driver.head import HeadSyncer -from chainlib.eth.connection import EthHTTPConnection -from chainlib.eth.block import ( - block_latest, - ) -from chainlib.hash import keccak256_string_to_hex +from chainlib.chain import ChainSpec from chainlib.eth.address import to_checksum_address +from chainlib.eth.block import ( + block_latest, +) +from chainlib.eth.connection import EthHTTPConnection +from chainlib.eth.error import ( + RequestMismatchException, +) from chainlib.eth.gas import OverrideGasOracle from chainlib.eth.nonce import RPCNonceOracle -from chainlib.eth.tx import TxFactory -from chainlib.jsonrpc import JSONRPCRequest -from chainlib.eth.error import ( - EthException, - RequestMismatchException, - ) -from chainlib.chain import ChainSpec -from chainlib.eth.constant import ZERO_ADDRESS -from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore.dict import DictKeystore -from cic_types.models.person import Person -from eth_erc20 import ERC20 +from chainlib.hash import keccak256_string_to_hex +from chainsyncer.backend.memory import MemBackend +from chainsyncer.driver.head import HeadSyncer from cic_eth.cli.chain import chain_interface +from cic_types.models.person import Person from eth_accounts_index import AccountsIndex from eth_contract_registry import Registry +from eth_erc20 import ERC20 from eth_token_index import TokenUniqueSymbolIndex - +from funga.eth.keystore.dict import DictKeystore +from funga.eth.signer import EIP155Signer +from hexathon import ( + strip_0x, +) logging.basicConfig(level=logging.WARNING) logg = logging.getLogger() diff --git a/apps/data-seeding/cic_eth/traffic/common/signer.py b/apps/data-seeding/cic_eth/traffic/common/signer.py index fa47b79b..319f6c5b 100644 --- a/apps/data-seeding/cic_eth/traffic/common/signer.py +++ b/apps/data-seeding/cic_eth/traffic/common/signer.py @@ -2,8 +2,8 @@ import logging # external imports -from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore.dict import DictKeystore +from funga.eth.signer import EIP155Signer +from funga.eth.keystore.dict import DictKeystore logg = logging.getLogger(__name__) diff --git a/apps/data-seeding/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py index bff5ed89..3db012e8 100644 --- a/apps/data-seeding/cic_ussd/import_balance.py +++ b/apps/data-seeding/cic_ussd/import_balance.py @@ -9,8 +9,8 @@ from chainlib.chain import ChainSpec from chainlib.eth.address import to_checksum_address from chainlib.eth.connection import EthHTTPConnection from confini import Config -from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore.dict import DictKeystore +from funga.eth.signer import EIP155Signer +from funga.eth.keystore.dict import DictKeystore # local imports from import_util import BalanceProcessor, get_celery_worker_status diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile index f2d09df6..cd6493cf 100644 --- a/apps/data-seeding/docker/Dockerfile +++ b/apps/data-seeding/docker/Dockerfile @@ -1,46 +1,30 @@ -ARG DOCKER_REGISTRY="registry.gitlab.com/grassrootseconomics" - -FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2 +# syntax = docker/dockerfile:1.2 +FROM registry.gitlab.com/grassrootseconomics/cic-base-images:python-3.8.6-dev-5ab8bf45 WORKDIR /root RUN mkdir -vp /usr/local/etc/cic -ARG NPM_REPOSITORY=${NPM_REPOSITORY:-https://registry.npmjs.org} -RUN npm config set snyk=false -#RUN npm config set registry={NPM_REPOSITORY} -RUN npm config set registry=${NPM_REPOSITORY} - -# copy the dependencies -COPY package.json package-lock.json ./ -RUN --mount=type=cache,mode=0755,target=/root/.npm \ - npm set cache /root/.npm && \ - npm cache verify && \ - npm ci --verbose +COPY package.json \ + package-lock.json \ + ./ -#RUN npm ci --production --verbose +RUN npm ci --production #RUN --mount=type=cache,mode=0755,target=/root/node_modules npm install COPY common/ cic_ussd/common/ COPY requirements.txt . COPY config/ config - -ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net:8433 +ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net ARG EXTRA_PIP_ARGS="" ARG PIP_INDEX_URL=https://pypi.org/simple -RUN pip install --index-url $PIP_INDEX_URL \ - --pre \ - --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ - -r requirements.txt +RUN pip install --index-url $PIP_INDEX_URL \ + --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ + -r requirements.txt - -#RUN pip install --index-url $PIP_INDEX_URL \ -# --extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \ -# -r requirements.txt - -COPY . . +COPY . . ENTRYPOINT [ ] diff --git a/apps/data-seeding/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py index 1a9080a9..3c103301 100644 --- a/apps/data-seeding/eth/import_balance.py +++ b/apps/data-seeding/eth/import_balance.py @@ -33,8 +33,8 @@ from chainlib.eth.error import ( RequestMismatchException, ) from chainlib.chain import ChainSpec -from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore.dict import DictKeystore +from funga.eth.signer import EIP155Signer +from funga.eth.keystore.dict import DictKeystore from cic_types.models.person import Person from eth_erc20 import ERC20 from cic_eth.cli.chain import chain_interface diff --git a/apps/data-seeding/eth/import_users.py b/apps/data-seeding/eth/import_users.py index f7a54969..dd462f79 100644 --- a/apps/data-seeding/eth/import_users.py +++ b/apps/data-seeding/eth/import_users.py @@ -27,9 +27,9 @@ from cic_types.processor import generate_metadata_pointer from cic_types import MetadataPointer from eth_accounts_index.registry import AccountRegistry from eth_contract_registry import Registry -from crypto_dev_signer.keystore.dict import DictKeystore -from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer -from crypto_dev_signer.keystore.keyfile import to_dict as to_keyfile_dict +from funga.eth.keystore.dict import DictKeystore +from funga.eth.signer.defaultsigner import EIP155Signer +from funga.eth.keystore.keyfile import to_dict as to_keyfile_dict # local imports from common.dirs import initialize_dirs