kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'lash/free-api' into 'master'

Browse Source 
Minimal dependencies in cic-eth api setup

Closes #75

See merge request grassrootseconomics/cic-internal-integration!210
This commit is contained in:
Louis Holbrook 2021-07-07 07:45:21 +00:00
commit c559bb2fee
25 changed files with 262 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/cic-eth/.gitlab-ci.yml
View File

@ -3,31 +3,29 @@
APP_NAME: cic-eth
DOCKERFILE_PATH: $APP_NAME/docker/Dockerfile
.cic_eth_changes_target:
.cic_eth_mr_changes_target:
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
#changes:
#- $CONTEXT/$APP_NAME/**/*
changes:
- $CONTEXT/$APP_NAME/**/*
when: always
build-mr-cic-eth:
extends:
- .cic_eth_variables
- .cic_eth_changes_target
- .cic_eth_mr_changes_target
- .py_build_target_test
test-mr-cic-eth:
extends:
- .cic_eth_variables
- .cic_eth_changes_target
- .cic_eth_mr_changes_target
stage: test
image: $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
image: $IMAGE_TAG_BASE
script:
- cd apps/$APP_NAME/
- pytest -x --cov=cic_eth --cov-fail-under=90 --cov-report term-missing tests
needs: ["build-mr-cic-eth"]
build-push-cic-eth:
extends:
- .py_build_push
- .cic_eth_variables

5
apps/cic-eth/admin_requirements.txt Normal file
View File

@ -0,0 +1,5 @@
SQLAlchemy==1.3.20
cic-eth-registry~=0.5.6a1
hexathon~=0.0.1a7
chainqueue~=0.0.2b5
eth-erc20==0.0.10a2

1
apps/cic-eth/cic_eth/api/__init__.py
View File

@ -5,4 +5,3 @@
"""
from .api_task import Api
from .api_admin import AdminApi

0
apps/cic-eth/cic_eth/api/api_admin.py → apps/cic-eth/cic_eth/api/admin.py
View File

3
apps/cic-eth/cic_eth/api/api_task.py
View File

@ -8,11 +8,10 @@ import logging
# external imports
import celery
from cic_eth_registry import CICRegistry
from chainlib.chain import ChainSpec
# local imports
from cic_eth.db.enum import LockEnum
from cic_eth.enum import LockEnum
app = celery.current_app

159
apps/cic-eth/cic_eth/db/enum.py
View File

@ -1,158 +1 @@
# standard imports
import enum
@enum.unique
class StatusBits(enum.IntEnum):
"""Individual bit flags that are combined to define the state and legacy of a queued transaction
"""
QUEUED = 0x01 # transaction should be sent to network
IN_NETWORK = 0x08 # transaction is in network
DEFERRED = 0x10 # an attempt to send the transaction to network has failed
GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
LOCAL_ERROR = 0x100 # errors that originate internally from the component
NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
FINAL = 0x1000 # transaction processing has completed
OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
MANUAL = 0x8000 # transaction processing has been manually overridden
@enum.unique
class StatusEnum(enum.IntEnum):
"""
- Inactive, not finalized. (<0)
* PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
* SENDFAIL: The transaction was not received by the node.
* RETRY: The transaction is queued for a new send attempt after previously failing.
* READYSEND: The transaction is queued for its first send attempt
* OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
* WAITFORGAS: The transaction is on hold pending gas funding.
- Active state: (==0)
* SENT: The transaction has been sent to the mempool.
- Inactive, finalized. (>0)
* FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
* CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
* OVERRIDDEN: Transaction has been manually overriden.
* REJECTED: The transaction was rejected by the node.
* REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
* SUCCESS: THe transaction was successfully mined. (Block number will be set)
"""
PENDING = 0
SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
RETRY = StatusBits.QUEUED | StatusBits.DEFERRED
READYSEND = StatusBits.QUEUED
OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
WAITFORGAS = StatusBits.GAS_ISSUES
SENT = StatusBits.IN_NETWORK
FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL
@enum.unique
class LockEnum(enum.IntEnum):
"""
STICKY: When set, reset is not possible
CREATE: Disable creation of accounts
SEND: Disable sending to network
QUEUE: Disable queueing new or modified transactions
"""
STICKY=1
INIT=2
CREATE=4
SEND=8
QUEUE=16
QUERY=32
ALL=int(0xfffffffffffffffe)
def status_str(v, bits_only=False):
"""Render a human-readable string describing the status
If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
:param v: Status bit field
:type v: number
:param bits_only: Only render individual bit labels.
:type bits_only: bool
:returns: Status string
:rtype: str
"""
s = ''
if not bits_only:
try:
s = StatusEnum(v).name
return s
except ValueError:
pass
if v == 0:
return 'NONE'
for i in range(16):
b = (1 << i)
if (b & 0xffff) & v:
n = StatusBits(b).name
if len(s) > 0:
s += ','
s += n
if not bits_only:
s += '*'
return s
def all_errors():
"""Bit mask of all error states
:returns: Error flags
:rtype: number
"""
return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
def is_error_status(v):
"""Check if value is an error state
:param v: Status bit field
:type v: number
:returns: True if error
:rtype: bool
"""
return bool(v & all_errors())
def dead():
"""Bit mask defining whether a transaction is still likely to be processed on the network.
:returns: Bit mask
:rtype: number
"""
return StatusBits.FINAL | StatusBits.OBSOLETE
def is_alive(v):
"""Check if transaction is still likely to be processed on the network.
The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
:returns:
"""
return bool(v & dead() == 0)
from cic_eth.enum import *

158
apps/cic-eth/cic_eth/enum.py Normal file
View File

@ -0,0 +1,158 @@
# standard imports
import enum
@enum.unique
class StatusBits(enum.IntEnum):
"""Individual bit flags that are combined to define the state and legacy of a queued transaction
"""
QUEUED = 0x01 # transaction should be sent to network
IN_NETWORK = 0x08 # transaction is in network
DEFERRED = 0x10 # an attempt to send the transaction to network has failed
GAS_ISSUES = 0x20 # transaction is pending sender account gas funding
LOCAL_ERROR = 0x100 # errors that originate internally from the component
NODE_ERROR = 0x200 # errors originating in the node (invalid RLP input...)
NETWORK_ERROR = 0x400 # errors that originate from the network (REVERT)
UNKNOWN_ERROR = 0x800 # unclassified errors (the should not occur)
FINAL = 0x1000 # transaction processing has completed
OBSOLETE = 0x2000 # transaction has been replaced by a different transaction with higher fee
MANUAL = 0x8000 # transaction processing has been manually overridden
@enum.unique
class StatusEnum(enum.IntEnum):
"""
- Inactive, not finalized. (<0)
* PENDING: The initial state of a newly added transaction record. No action has been performed on this transaction yet.
* SENDFAIL: The transaction was not received by the node.
* RETRY: The transaction is queued for a new send attempt after previously failing.
* READYSEND: The transaction is queued for its first send attempt
* OBSOLETED: A new transaction with the same nonce and higher gas has been sent to network.
* WAITFORGAS: The transaction is on hold pending gas funding.
- Active state: (==0)
* SENT: The transaction has been sent to the mempool.
- Inactive, finalized. (>0)
* FUBAR: Unknown error occurred and transaction is abandoned. Manual intervention needed.
* CANCELLED: The transaction was sent, but was not mined and has disappered from the mempool. This usually follows a transaction being obsoleted.
* OVERRIDDEN: Transaction has been manually overriden.
* REJECTED: The transaction was rejected by the node.
* REVERTED: The transaction was mined, but exception occurred during EVM execution. (Block number will be set)
* SUCCESS: THe transaction was successfully mined. (Block number will be set)
"""
PENDING = 0
SENDFAIL = StatusBits.DEFERRED | StatusBits.LOCAL_ERROR
RETRY = StatusBits.QUEUED | StatusBits.DEFERRED
READYSEND = StatusBits.QUEUED
OBSOLETED = StatusBits.OBSOLETE | StatusBits.IN_NETWORK
WAITFORGAS = StatusBits.GAS_ISSUES
SENT = StatusBits.IN_NETWORK
FUBAR = StatusBits.FINAL | StatusBits.UNKNOWN_ERROR
CANCELLED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.OBSOLETE
OVERRIDDEN = StatusBits.FINAL | StatusBits.OBSOLETE | StatusBits.MANUAL
REJECTED = StatusBits.NODE_ERROR | StatusBits.FINAL
REVERTED = StatusBits.IN_NETWORK | StatusBits.FINAL | StatusBits.NETWORK_ERROR
SUCCESS = StatusBits.IN_NETWORK | StatusBits.FINAL
@enum.unique
class LockEnum(enum.IntEnum):
"""
STICKY: When set, reset is not possible
CREATE: Disable creation of accounts
SEND: Disable sending to network
QUEUE: Disable queueing new or modified transactions
"""
STICKY=1
INIT=2
CREATE=4
SEND=8
QUEUE=16
QUERY=32
ALL=int(0xfffffffffffffffe)
def status_str(v, bits_only=False):
"""Render a human-readable string describing the status
If the bit field exactly matches a StatusEnum value, the StatusEnum label will be returned.
If a StatusEnum cannot be matched, the string will be postfixed with "*", unless explicitly instructed to return bit field labels only.
:param v: Status bit field
:type v: number
:param bits_only: Only render individual bit labels.
:type bits_only: bool
:returns: Status string
:rtype: str
"""
s = ''
if not bits_only:
try:
s = StatusEnum(v).name
return s
except ValueError:
pass
if v == 0:
return 'NONE'
for i in range(16):
b = (1 << i)
if (b & 0xffff) & v:
n = StatusBits(b).name
if len(s) > 0:
s += ','
s += n
if not bits_only:
s += '*'
return s
def all_errors():
"""Bit mask of all error states
:returns: Error flags
:rtype: number
"""
return StatusBits.LOCAL_ERROR | StatusBits.NODE_ERROR | StatusBits.NETWORK_ERROR | StatusBits.UNKNOWN_ERROR
def is_error_status(v):
"""Check if value is an error state
:param v: Status bit field
:type v: number
:returns: True if error
:rtype: bool
"""
return bool(v & all_errors())
def dead():
"""Bit mask defining whether a transaction is still likely to be processed on the network.
:returns: Bit mask
:rtype: number
"""
return StatusBits.FINAL | StatusBits.OBSOLETE
def is_alive(v):
"""Check if transaction is still likely to be processed on the network.
The contingency of "likely" refers to the case a transaction has been obsoleted after sent to the network, but the network still confirms the obsoleted transaction. The return value of this method will not change as a result of this, BUT the state itself will (as the FINAL bit will be set).
:returns:
"""
return bool(v & dead() == 0)

2
apps/cic-eth/cic_eth/runnable/ctrl.py
View File

@ -12,7 +12,7 @@ from chainlib.eth.constant import ZERO_ADDRESS
from chainlib.eth.address import is_checksum_address
# local imports
from cic_eth.api import AdminApi
from cic_eth.api.admin import AdminApi
from cic_eth.db.enum import LockEnum
logging.basicConfig(level=logging.WARNING)

6
apps/cic-eth/cic_eth/runnable/info.py
View File

@ -12,10 +12,8 @@ import confini
import celery
# local imports
from cic_eth.api import (
Api,
AdminApi,
)
from cic_eth.api import Api
from cic_eth.api.admin import AdminApi
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()

2
apps/cic-eth/cic_eth/runnable/resend.py
View File

@ -11,7 +11,7 @@ from chainlib.chain import ChainSpec
from chainlib.eth.connection import EthHTTPConnection
# local imports
from cic_eth.api.api_admin import AdminApi
from cic_eth.api.admin import AdminApi
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()

2
apps/cic-eth/cic_eth/runnable/tag.py
View File

@ -12,7 +12,7 @@ from chainlib.chain import ChainSpec
from xdg.BaseDirectory import xdg_config_home
# local imports
from cic_eth.api import AdminApi
from cic_eth.api.admin import AdminApi
from cic_eth.db import dsn_from_config
from cic_eth.db.models.base import SessionBase

2
apps/cic-eth/cic_eth/runnable/view.py
View File

@ -19,7 +19,7 @@ from chainlib.eth.connection import EthHTTPConnection
from hexathon import add_0x
# local imports
from cic_eth.api import AdminApi
from cic_eth.api.admin import AdminApi
from cic_eth.db.enum import (
StatusEnum,
status_str,

6
apps/cic-eth/cic_eth/version.py
View File

@ -8,9 +8,9 @@ import semver
version = (
0,
11,
1,
'alpha.3',
12,
0,
'alpha.1',
)
version_object = semver.VersionInfo(

12
apps/cic-eth/docker/Dockerfile
View File

@ -8,12 +8,14 @@ RUN apt-get update && \
#RUN python -m venv venv && . venv/bin/activate
ARG pip_extra_index_url_flag='--index https://pypi.org/simple --extra-index-url https://pip.grassrootseconomics.net:8433'
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
RUN /usr/local/bin/python -m pip install --upgrade pip
RUN pip install semver
# TODO use a packaging style that lets us copy requirments only ie. pip-tools
COPY cic-eth/ .
RUN pip install $pip_extra_index_url_flag .
RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL .
# --- TEST IMAGE ---
FROM python:3.8.6-slim-buster as test
@ -32,8 +34,12 @@ COPY --from=compile /usr/local/lib/python3.8/site-packages/ \
# COPY --from=compile /usr/src/cic-eth/ .
# RUN . venv/bin/activate
ARG EXTRA_INDEX_URL="https://pip.grassrootseconomics.net:8433"
ARG GITLAB_PYTHON_REGISTRY="https://gitlab.com/api/v4/projects/27624814/packages/pypi/simple"
COPY cic-eth/test_requirements.txt .
RUN pip install $pip_extra_index_url_flag -r test_requirements.txt
RUN pip install --extra-index-url $GITLAB_PYTHON_REGISTRY \
--extra-index-url $EXTRA_INDEX_URL -r test_requirements.txt
COPY cic-eth .

24
apps/cic-eth/requirements.txt
View File

@ -1,25 +1,3 @@
cic-base==0.1.3a3+build.984b5cff
celery==4.4.7
crypto-dev-signer~=0.4.14b6
confini~=0.3.6rc3
cic-eth-registry~=0.5.6a1
redis==3.5.3
alembic==1.4.2
websockets==8.1
requests~=2.24.0
eth_accounts_index~=0.0.12a1
erc20-transfer-authorization~=0.3.2a1
uWSGI==2.0.19.1
chainlib~=0.0.5a1
semver==2.13.0
websocket-client==0.57.0
moolb~=0.1.1b2
eth-address-index~=0.1.2a1
chainlib-eth~=0.0.5a1
hexathon~=0.0.1a7
chainsyncer[sql]~=0.0.3a3
chainqueue~=0.0.2b5
sarafu-faucet~=0.0.4a1
erc20-faucet~=0.2.2a1
coincurve==15.0.0
potaahto~=0.0.1a2
pycryptodome==3.10.1

7
apps/cic-eth/services_requirements.txt Normal file
View File

@ -0,0 +1,7 @@
chainsyncer[sql]~=0.0.3a3
chainqueue~=0.0.2b5
alembic==1.4.2
confini~=0.3.6rc4
redis==3.5.3
hexathon~=0.0.1a7
pycryptodome==3.10.1

22
apps/cic-eth/setup.cfg
View File

@ -45,16 +45,16 @@ scripts =
[options.entry_points]
console_scripts =
# daemons
cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main
cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main
cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main
cic-eth-retrierd = cic_eth.runnable.daemons.retry:main
cic-eth-taskerd = cic_eth.runnable.daemons.tasker:main [services]
cic-eth-trackerd = cic_eth.runnable.daemons.tracker:main [services]
cic-eth-dispatcherd = cic_eth.runnable.daemons.dispatcher:main [services]
cic-eth-retrierd = cic_eth.runnable.daemons.retry:main [services]
# tools
cic-eth-create = cic_eth.runnable.create:main
cic-eth-inspect = cic_eth.runnable.view:main
cic-eth-ctl = cic_eth.runnable.ctrl:main
cic-eth-info = cic_eth.runnable.info:main
cic-eth-create = cic_eth.runnable.create:main [tools]
cic-eth-inspect = cic_eth.runnable.view:main [tools]
cic-eth-ctl = cic_eth.runnable.ctrl:main [tools]
cic-eth-info = cic_eth.runnable.info:main [tools]
# TODO: Merge this with ctl when subcmds sorted to submodules
cic-eth-tag = cic_eth.runnable.tag:main
cic-eth-resend = cic_eth.runnable.resend:main
cic-eth-transfer = cic_eth.runnable.transfer:main
cic-eth-tag = cic_eth.runnable.tag:main [tools]
cic-eth-resend = cic_eth.runnable.resend:main [tools]
cic-eth-transfer = cic_eth.runnable.transfer:main [tools]

37
apps/cic-eth/setup.py
View File

@ -11,6 +11,41 @@ while True:
requirements.append(l.rstrip())
f.close()
admin_requirements = []
f = open('admin_requirements.txt', 'r')
while True:
l = f.readline()
if l == '':
break
admin_requirements.append(l.rstrip())
f.close()
tools_requirements = []
f = open('tools_requirements.txt', 'r')
while True:
l = f.readline()
if l == '':
break
tools_requirements.append(l.rstrip())
f.close()
services_requirements = []
f = open('services_requirements.txt', 'r')
while True:
l = f.readline()
if l == '':
break
services_requirements.append(l.rstrip())
f.close()
setup(
install_requires=requirements
install_requires=requirements,
extras_require = {
'tools': tools_requirements,
'admin_api': admin_requirements,
'services': services_requirements,
}
)

4
apps/cic-eth/test_requirements.txt
View File

@ -2,6 +2,8 @@ pytest==6.0.1
pytest-celery==0.0.0a1
pytest-mock==3.3.1
pytest-cov==2.10.1
pytest-redis==2.0.0
redis==3.5.3
eth-tester==0.5.0b3
py-evm==0.3.0a20
eth-erc20==0.0.10a2
eth-erc20~=0.0.10a2

2
apps/cic-eth/tests/task/api/test_admin.py
View File

@ -40,7 +40,7 @@ from chainqueue.sql.query import (
)
# local imports
from cic_eth.api import AdminApi
from cic_eth.api.admin import AdminApi
from cic_eth.db.models.role import AccountRole
from cic_eth.db.enum import LockEnum
from cic_eth.error import InitializationError

2
apps/cic-eth/tests/task/api/test_admin_noncritical.py
View File

@ -35,7 +35,7 @@ from eth_erc20 import ERC20
from cic_eth_registry import CICRegistry
# local imports
from cic_eth.api.api_admin import AdminApi
from cic_eth.api.admin import AdminApi
from cic_eth.eth.gas import cache_gas_data
from cic_eth.eth.erc20 import cache_transfer_data

8
apps/cic-eth/tools_requirements.txt Normal file
View File

@ -0,0 +1,8 @@
crypto-dev-signer~=0.4.14b6
chainqueue~=0.0.2b5
confini~=0.3.6rc4
cic-eth-registry~=0.5.6a1
redis==3.5.3
hexathon~=0.0.1a7
pycryptodome==3.10.1
pyxdg==0.27

2
apps/cic-ussd/requirements.txt
View File

@ -1,4 +1,4 @@
cic_base[full_graph]==0.1.3a3+build.984b5cff
cic-eth~=0.11.1a3
cic-eth~=0.12.0a1
cic-notify~=0.4.0a7
cic-types~=0.1.0a11

3
apps/data-seeding/requirements.txt
View File

@ -1,5 +1,4 @@
cic_base[full_graph]==0.1.3a3+build.984b5cff
sarafu-faucet==0.0.4a1
cic-eth==0.11.1a1
cic-eth[tools]==0.12.0a1
cic-types==0.1.0a13
crypto-dev-signer==0.4.14b6

8
ci_templates/.cic-template.yml
View File

@ -21,14 +21,12 @@ variables:
.py_build_target_test:
stage: build
variables:
CI_DEBUG_TRACE: "true"
IMAGE_TAG_BASE: $CI_REGISTRY_IMAGE/$APP_NAME:mr-unittest-$CI_COMMIT_SHORT_SHA
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > "/kaniko/.docker/config.json"
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --tarPath $APP_NAME-test-image.tar --destination $CI_REGISTRY_IMAGE/$APP_NAME-test:latest
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: always
- /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE_PATH $KANIKO_CACHE_ARGS --cache-repo $CI_REGISTRY_IMAGE --target test --destination $IMAGE_TAG_BASE
.py_build_push:
stage: build