diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1af9a43..52f8ffd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ include: - local: 'apps/cic-notify/.gitlab-ci.yml' - local: 'apps/cic-meta/.gitlab-ci.yml' - local: 'apps/cic-cache/.gitlab-ci.yml' + - local: 'apps/data-seeding/.gitlab-ci.yml' stages: - build diff --git a/apps/contract-migration/docker/Dockerfile b/apps/contract-migration/docker/Dockerfile index d91e638b..19ba1c13 100644 --- a/apps/contract-migration/docker/Dockerfile +++ b/apps/contract-migration/docker/Dockerfile @@ -87,7 +87,6 @@ COPY contract-migration/testdata/pgp testdata/pgp COPY contract-migration/sarafu_declaration.json sarafu_declaration.json COPY contract-migration/keystore keystore COPY contract-migration/envlist . -COPY contract-migration/scripts scripts/ # A shared output dir for environment configs RUN mkdir -p /tmp/cic/config diff --git a/apps/data-seeding/.gitlab-ci.yml b/apps/data-seeding/.gitlab-ci.yml new file mode 100644 index 00000000..9f4eaaf6 --- /dev/null +++ b/apps/data-seeding/.gitlab-ci.yml @@ -0,0 +1,21 @@ +.data_seeding_variables: + variables: + APP_NAME: data-seeding + DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile + +.data_seeding_changes_target: + rules: + - changes: + - $CONTEXT/$APP_NAME/* + +build-mr-data-seeding: + extends: + - .data_seeding_changes_target + - .py_build_merge_request + - .data_seeding_variables + +build-push-data-seeding: + extends: + - .py_build_push + - .data_seeding_variables + diff --git a/apps/contract-migration/scripts/README.md b/apps/data-seeding/README.md similarity index 100% rename from apps/contract-migration/scripts/README.md rename to apps/data-seeding/README.md diff --git a/apps/contract-migration/scripts/cic_eth/import_balance.py b/apps/data-seeding/cic_eth/import_balance.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/import_balance.py rename to apps/data-seeding/cic_eth/import_balance.py diff --git a/apps/contract-migration/scripts/cic_eth/import_users.py b/apps/data-seeding/cic_eth/import_users.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/import_users.py rename to apps/data-seeding/cic_eth/import_users.py diff --git a/apps/contract-migration/scripts/cic_eth/paths.py b/apps/data-seeding/cic_eth/paths.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/paths.py rename to apps/data-seeding/cic_eth/paths.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/cmd/__init__.py b/apps/data-seeding/cic_eth/traffic/cmd/__init__.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/cmd/__init__.py rename to apps/data-seeding/cic_eth/traffic/cmd/__init__.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py b/apps/data-seeding/cic_eth/traffic/cmd/traffic.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/cmd/traffic.py rename to apps/data-seeding/cic_eth/traffic/cmd/traffic.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/__init__.py b/apps/data-seeding/cic_eth/traffic/common/__init__.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/__init__.py rename to apps/data-seeding/cic_eth/traffic/common/__init__.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/argparse.py b/apps/data-seeding/cic_eth/traffic/common/argparse.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/argparse.py rename to apps/data-seeding/cic_eth/traffic/common/argparse.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/config.py b/apps/data-seeding/cic_eth/traffic/common/config.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/config.py rename to apps/data-seeding/cic_eth/traffic/common/config.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/log.py b/apps/data-seeding/cic_eth/traffic/common/log.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/log.py rename to apps/data-seeding/cic_eth/traffic/common/log.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/registry.py b/apps/data-seeding/cic_eth/traffic/common/registry.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/registry.py rename to apps/data-seeding/cic_eth/traffic/common/registry.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/rpc.py b/apps/data-seeding/cic_eth/traffic/common/rpc.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/rpc.py rename to apps/data-seeding/cic_eth/traffic/common/rpc.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/common/signer.py b/apps/data-seeding/cic_eth/traffic/common/signer.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/common/signer.py rename to apps/data-seeding/cic_eth/traffic/common/signer.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/account.py b/apps/data-seeding/cic_eth/traffic/local/account.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/local/account.py rename to apps/data-seeding/cic_eth/traffic/local/account.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/noop.py b/apps/data-seeding/cic_eth/traffic/local/noop.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/local/noop.py rename to apps/data-seeding/cic_eth/traffic/local/noop.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py b/apps/data-seeding/cic_eth/traffic/local/transfer.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/local/transfer.py rename to apps/data-seeding/cic_eth/traffic/local/transfer.py diff --git a/apps/contract-migration/scripts/cic_eth/traffic/traffic.py b/apps/data-seeding/cic_eth/traffic/traffic.py similarity index 100% rename from apps/contract-migration/scripts/cic_eth/traffic/traffic.py rename to apps/data-seeding/cic_eth/traffic/traffic.py diff --git a/apps/contract-migration/scripts/cic_meta/import_meta.js b/apps/data-seeding/cic_meta/import_meta.js similarity index 100% rename from apps/contract-migration/scripts/cic_meta/import_meta.js rename to apps/data-seeding/cic_meta/import_meta.js diff --git a/apps/contract-migration/scripts/cic_meta/import_meta_custom.js b/apps/data-seeding/cic_meta/import_meta_custom.js similarity index 100% rename from apps/contract-migration/scripts/cic_meta/import_meta_custom.js rename to apps/data-seeding/cic_meta/import_meta_custom.js diff --git a/apps/contract-migration/scripts/cic_meta/import_meta_phone.js b/apps/data-seeding/cic_meta/import_meta_phone.js similarity index 100% rename from apps/contract-migration/scripts/cic_meta/import_meta_phone.js rename to apps/data-seeding/cic_meta/import_meta_phone.js diff --git a/apps/contract-migration/scripts/cic_ussd/import_balance.py b/apps/data-seeding/cic_ussd/import_balance.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_balance.py rename to apps/data-seeding/cic_ussd/import_balance.py diff --git a/apps/contract-migration/scripts/cic_ussd/import_pins.py b/apps/data-seeding/cic_ussd/import_pins.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_pins.py rename to apps/data-seeding/cic_ussd/import_pins.py diff --git a/apps/contract-migration/scripts/cic_ussd/import_task.py b/apps/data-seeding/cic_ussd/import_task.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_task.py rename to apps/data-seeding/cic_ussd/import_task.py diff --git a/apps/contract-migration/scripts/cic_ussd/import_users.py b/apps/data-seeding/cic_ussd/import_users.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_users.py rename to apps/data-seeding/cic_ussd/import_users.py diff --git a/apps/contract-migration/scripts/cic_ussd/import_ussd_data.py b/apps/data-seeding/cic_ussd/import_ussd_data.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_ussd_data.py rename to apps/data-seeding/cic_ussd/import_ussd_data.py diff --git a/apps/contract-migration/scripts/cic_ussd/import_util.py b/apps/data-seeding/cic_ussd/import_util.py similarity index 100% rename from apps/contract-migration/scripts/cic_ussd/import_util.py rename to apps/data-seeding/cic_ussd/import_util.py diff --git a/apps/contract-migration/scripts/config/app.ini b/apps/data-seeding/config/app.ini similarity index 100% rename from apps/contract-migration/scripts/config/app.ini rename to apps/data-seeding/config/app.ini diff --git a/apps/contract-migration/scripts/config/celery.ini b/apps/data-seeding/config/celery.ini similarity index 100% rename from apps/contract-migration/scripts/config/celery.ini rename to apps/data-seeding/config/celery.ini diff --git a/apps/contract-migration/scripts/config/cic.ini b/apps/data-seeding/config/cic.ini similarity index 100% rename from apps/contract-migration/scripts/config/cic.ini rename to apps/data-seeding/config/cic.ini diff --git a/apps/contract-migration/scripts/config/database.ini b/apps/data-seeding/config/database.ini similarity index 100% rename from apps/contract-migration/scripts/config/database.ini rename to apps/data-seeding/config/database.ini diff --git a/apps/contract-migration/scripts/config/eth.ini b/apps/data-seeding/config/eth.ini similarity index 100% rename from apps/contract-migration/scripts/config/eth.ini rename to apps/data-seeding/config/eth.ini diff --git a/apps/contract-migration/scripts/config/meta.ini b/apps/data-seeding/config/meta.ini similarity index 100% rename from apps/contract-migration/scripts/config/meta.ini rename to apps/data-seeding/config/meta.ini diff --git a/apps/contract-migration/scripts/config/pgp.ini b/apps/data-seeding/config/pgp.ini similarity index 100% rename from apps/contract-migration/scripts/config/pgp.ini rename to apps/data-seeding/config/pgp.ini diff --git a/apps/contract-migration/scripts/config/redis.ini b/apps/data-seeding/config/redis.ini similarity index 100% rename from apps/contract-migration/scripts/config/redis.ini rename to apps/data-seeding/config/redis.ini diff --git a/apps/contract-migration/scripts/config/traffic.ini b/apps/data-seeding/config/traffic.ini similarity index 100% rename from apps/contract-migration/scripts/config/traffic.ini rename to apps/data-seeding/config/traffic.ini diff --git a/apps/contract-migration/scripts/create_import_pins.py b/apps/data-seeding/create_import_pins.py similarity index 100% rename from apps/contract-migration/scripts/create_import_pins.py rename to apps/data-seeding/create_import_pins.py diff --git a/apps/contract-migration/scripts/create_import_users.py b/apps/data-seeding/create_import_users.py similarity index 100% rename from apps/contract-migration/scripts/create_import_users.py rename to apps/data-seeding/create_import_users.py diff --git a/apps/data-seeding/docker/Dockerfile b/apps/data-seeding/docker/Dockerfile new file mode 100644 index 00000000..79f9d6a1 --- /dev/null +++ b/apps/data-seeding/docker/Dockerfile @@ -0,0 +1,43 @@ +# syntax = docker/dockerfile:1.2 +FROM python:3.8.6-slim-buster as compile-image + +RUN apt-get update +RUN apt-get install -y --no-install-recommends git gcc g++ libpq-dev gawk jq telnet wget openssl iputils-ping gnupg socat bash procps make python2 cargo + +WORKDIR /root +RUN mkdir -vp /usr/local/etc/cic + +COPY data-seeding/requirements.txt . + +ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433" +RUN pip install --extra-index-url $EXTRA_INDEX_URL -r requirements.txt + +# -------------- begin runtime container ---------------- +FROM python:3.8.6-slim-buster as runtime-image + +RUN apt-get update +RUN apt-get install -y --no-install-recommends gnupg libpq-dev +RUN apt-get install -y jq bash iputils-ping socat telnet dnsutils + +COPY --from=compile-image /usr/local/bin/ /usr/local/bin/ +COPY --from=compile-image /usr/local/etc/cic/ /usr/local/etc/cic/ +COPY --from=compile-image /usr/local/lib/python3.8/site-packages/ \ + /usr/local/lib/python3.8/site-packages/ + +WORKDIR root/ + +ENV EXTRA_INDEX_URL https://pip.grassrootseconomics.net:8433 +# RUN useradd -u 1001 --create-home grassroots +# RUN adduser grassroots sudo && \ +# echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +# WORKDIR /home/grassroots + +COPY data-seeding/ . + +# we copied these from the root build container. +# this is dumb though...I guess the compile image should have the same user +# RUN chown grassroots:grassroots -R /usr/local/lib/python3.8/site-packages/ + +# USER grassroots + +ENTRYPOINT [ ] diff --git a/apps/contract-migration/scripts/eth/import_balance.py b/apps/data-seeding/eth/import_balance.py similarity index 100% rename from apps/contract-migration/scripts/eth/import_balance.py rename to apps/data-seeding/eth/import_balance.py diff --git a/apps/contract-migration/scripts/eth/import_users.py b/apps/data-seeding/eth/import_users.py similarity index 100% rename from apps/contract-migration/scripts/eth/import_users.py rename to apps/data-seeding/eth/import_users.py diff --git a/apps/contract-migration/scripts/package-lock.json b/apps/data-seeding/package-lock.json similarity index 100% rename from apps/contract-migration/scripts/package-lock.json rename to apps/data-seeding/package-lock.json diff --git a/apps/contract-migration/scripts/package.json b/apps/data-seeding/package.json similarity index 100% rename from apps/contract-migration/scripts/package.json rename to apps/data-seeding/package.json diff --git a/apps/contract-migration/scripts/requirements.txt b/apps/data-seeding/requirements.txt similarity index 58% rename from apps/contract-migration/scripts/requirements.txt rename to apps/data-seeding/requirements.txt index f7487c3b..fcc526d4 100644 --- a/apps/contract-migration/scripts/requirements.txt +++ b/apps/data-seeding/requirements.txt @@ -1,5 +1,5 @@ -cic-base[full_graph]==0.1.2b9 +cic-base[full_graph]==0.1.2b11 sarafu-faucet==0.0.3a3 -cic-eth==0.11.0b13 +cic-eth==0.11.0b14 cic-types==0.1.0a11 crypto-dev-signer==0.4.14b3 diff --git a/apps/contract-migration/scripts/verify.py b/apps/data-seeding/verify.py similarity index 100% rename from apps/contract-migration/scripts/verify.py rename to apps/data-seeding/verify.py