grassrootseconomics/cic-stack
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: Improve cic-cache

Browse Source
This commit is contained in:
Louis Holbrook
2022-01-04 16:01:01 +00:00
committed by Philip Wafula
parent b5653a704c
commit 104ff8a76a
74 changed files with 1057 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/MANIFEST.in
View File

@@ -1 +1 @@
include *requirements.txt cic_cache/data/config/*
include *requirements.txt cic_cache/data/config/* cic_cache/db/migrations/default/* cic_cache/db/migrations/default/versions/*

2
apps/cic-cache/cic_cache/data/config/cic.ini
View File

@@ -1,4 +1,4 @@
[cic]
registry_address =
trust_address =
health_modules = cic_eth.check.db,cic_eth.check.redis,cic_eth.check.signer,cic_eth.check.gas
health_modules =

3
apps/cic-cache/cic_cache/data/config/database.ini
View File

@@ -3,7 +3,8 @@ engine =
driver =
host =
port =
name = cic-cache
#name = cic-cache
prefix =
user =
password =
debug = 0

11
apps/cic-cache/cic_cache/db/__init__.py
View File

@@ -9,21 +9,26 @@ from .list import (
tag_transaction,
add_tag,
)
from cic_cache.db.models.base import SessionBase
logg = logging.getLogger()
def dsn_from_config(config):
def dsn_from_config(config, name):
scheme = config.get('DATABASE_ENGINE')
if config.get('DATABASE_DRIVER') != None:
scheme += '+{}'.format(config.get('DATABASE_DRIVER'))
database_name = name
if config.get('DATABASE_PREFIX'):
database_name = '{}_{}'.format(config.get('DATABASE_PREFIX'), database_name)
dsn = ''
if config.get('DATABASE_ENGINE') == 'sqlite':
SessionBase.poolable = False
dsn = '{}:///{}'.format(
scheme,
config.get('DATABASE_NAME'),
database_name,
)
else:
@@ -33,7 +38,7 @@ def dsn_from_config(config):
config.get('DATABASE_PASSWORD'),
config.get('DATABASE_HOST'),
config.get('DATABASE_PORT'),
config.get('DATABASE_NAME'),
database_name,
)
logg.debug('parsed dsn from config: {}'.format(dsn))
return dsn

121
apps/cic-cache/cic_cache/runnable/daemons/query.py
View File

@@ -5,7 +5,11 @@ import re
import base64
# external imports
from hexathon import add_0x
from hexathon import (
add_0x,
strip_0x,
)
from chainlib.encode import TxHexNormalizer
# local imports
from cic_cache.cache import (
@@ -16,27 +20,72 @@ from cic_cache.cache import (
logg = logging.getLogger(__name__)
#logg = logging.getLogger()
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_all_bloom = r'/tx/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_transactions_all_data = r'/txa/(\d+)?/?(\d+)/?'
re_transactions_all_data = r'/txa/?(\d+)?/?(\d+)?/?(\d+)?/?(\d+)?/?'
re_transactions_account_data = r'/txa/user/((0x)?[a-fA-F0-9]+)(/(\d+)(/(\d+))?)?/?'
re_default_limit = r'/defaultlimit/?'
DEFAULT_LIMIT = 100
tx_normalize = TxHexNormalizer()
def parse_query_account(r):
address = strip_0x(r[1])
#address = tx_normalize.wallet_address(address)
limit = DEFAULT_LIMIT
g = r.groups()
if len(g) > 3:
limit = int(r[4])
if limit == 0:
limit = DEFAULT_LIMIT
offset = 0
if len(g) > 4:
offset = int(r[6])
logg.debug('account query is address {} offset {} limit {}'.format(address, offset, limit))
return (address, offset, limit,)
# r is an re.Match
def parse_query_any(r):
limit = DEFAULT_LIMIT
offset = 0
block_offset = None
block_end = None
if r.lastindex != None:
if r.lastindex > 0:
limit = int(r[1])
if r.lastindex > 1:
offset = int(r[2])
if r.lastindex > 2:
block_offset = int(r[3])
if r.lastindex > 3:
block_end = int(r[4])
if block_end < block_offset:
raise ValueError('cart before the horse, dude')
logg.debug('data query is offset {} limit {} block_offset {} block_end {}'.format(offset, limit, block_offset, block_end))
return (offset, limit, block_offset, block_end,)
def process_default_limit(session, env):
r = re.match(re_default_limit, env.get('PATH_INFO'))
if not r:
return None
return ('application/json', str(DEFAULT_LIMIT).encode('utf-8'),)
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match account bloom')
address = r[1]
if r[2] == None:
address = add_0x(address)
offset = 0
if r.lastindex > 2:
offset = r[4]
limit = DEFAULT_LIMIT
if r.lastindex > 4:
limit = r[6]
(address, offset, limit,) = parse_query_account(r)
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
@@ -59,13 +108,9 @@ def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match all bloom')
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
(limit, offset, block_offset, block_end,) = parse_query_any(r)
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
@@ -88,17 +133,16 @@ def process_transactions_all_data(session, env):
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
if not r:
return None
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
return None
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
# return None
logg.debug('match all data')
logg.debug('got data request {}'.format(env))
block_offset = r[1]
block_end = r[2]
if int(r[2]) < int(r[1]):
raise ValueError('cart before the horse, dude')
(offset, limit, block_offset, block_end) = parse_query_any(r)
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(0, 0, block_offset, block_end, oldest=True) # oldest needs to be settable
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, limit, block_offset, block_end, oldest=True) # oldest needs to be settable
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
@@ -113,3 +157,30 @@ def process_transactions_all_data(session, env):
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_account_data(session, env):
r = re.match(re_transactions_account_data, env.get('PATH_INFO'))
if not r:
return None
logg.debug('match account data')
#if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
# return None
(address, offset, limit,) = parse_query_account(r)
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_account_with_data(address, offset, limit)
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
o = {
'low': lowest_block,
'high': highest_block,
'data': tx_cache,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)

17
apps/cic-cache/cic_cache/runnable/daemons/server.py
View File

@@ -12,21 +12,20 @@ import cic_cache.cli
from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import (
process_default_limit,
process_transactions_account_bloom,
process_transactions_account_data,
process_transactions_all_bloom,
process_transactions_all_data,
)
import cic_cache.cli
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
migrationsdir = os.path.join(dbdir, 'migrations')
# process args
arg_flags = cic_cache.cli.argflag_std_base
local_arg_flags = cic_cache.cli.argflag_local_task
arg_flags = cic_cache.cli.argflag_std_read
local_arg_flags = cic_cache.cli.argflag_local_sync | cic_cache.cli.argflag_local_task
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
args = argparser.parse_args()
@@ -35,7 +34,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
@@ -47,9 +46,11 @@ def application(env, start_response):
session = SessionBase.create_session()
for handler in [
process_transactions_account_data,
process_transactions_account_bloom,
process_transactions_all_data,
process_transactions_all_bloom,
process_transactions_account_bloom,
process_default_limit,
]:
r = None
try:

3
apps/cic-cache/cic_cache/runnable/daemons/tasker.py
View File

@@ -3,6 +3,7 @@ import logging
import os
import sys
import argparse
import tempfile
# third-party imports
import celery
@@ -28,7 +29,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn)
# set up celery

2
apps/cic-cache/cic_cache/runnable/daemons/tracker.py
View File

@@ -50,7 +50,7 @@ args = argparser.parse_args()
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags)
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
SessionBase.connect(dsn, debug=config.true('DATABASE_DEBUG'))
# set up rpc

2
apps/cic-cache/cic_cache/version.py
View File

@@ -5,7 +5,7 @@ version = (
0,
2,
1,
'alpha.2',
'alpha.3',
)
version_object = semver.VersionInfo(

3
apps/cic-cache/config/celery.ini
View File

@@ -1,3 +0,0 @@
[celery]
broker_url = redis:///
result_url = redis:///

3
apps/cic-cache/config/cic.ini
View File

@@ -1,3 +0,0 @@
[cic]
registry_address =
trust_address =

0
apps/cic-cache/config/config.ini Normal file
View File

9
apps/cic-cache/config/database.ini
View File

@@ -1,9 +0,0 @@
[database]
NAME=cic_cache
USER=postgres
PASSWORD=
HOST=localhost
PORT=5432
ENGINE=postgresql
DRIVER=psycopg2
DEBUG=0

3
apps/cic-cache/config/docker/celery.ini
View File

@@ -1,3 +0,0 @@
[celery]
broker_url = redis://localhost:63379
result_url = redis://localhost:63379

3
apps/cic-cache/config/docker/cic.ini
View File

@@ -1,3 +0,0 @@
[cic]
registry_address =
trust_address = 0xEb3907eCad74a0013c259D5874AE7f22DcBcC95C

9
apps/cic-cache/config/docker/database.ini
View File

@@ -1,9 +0,0 @@
[database]
NAME=cic_cache
USER=grassroots
PASSWORD=
HOST=localhost
PORT=63432
ENGINE=postgresql
DRIVER=psycopg2
DEBUG=0

4
apps/cic-cache/config/docker/syncer.ini
View File

@@ -1,4 +0,0 @@
[syncer]
loop_interval = 1
offset = 0
no_history = 0

2
apps/cic-cache/config/test/bancor.ini
View File

@@ -1,2 +0,0 @@
[bancor]
dir =

1
apps/cic-cache/config/test/cic.ini
View File

@@ -1,4 +1,3 @@
[cic]
registry_address =
chain_spec =
trust_address =

2
apps/cic-cache/config/test/database.ini
View File

@@ -1,5 +1,5 @@
[database]
NAME=cic-cache-test
PREFIX=cic-cache-test
USER=postgres
PASSWORD=
HOST=localhost

5
apps/cic-cache/config/test/eth.ini
View File

@@ -1,5 +0,0 @@
[eth]
#ws_provider = ws://localhost:8546
#ttp_provider = http://localhost:8545
provider = http://localhost:8545
#chain_id =

597
apps/cic-cache/doc/openapi/server.yml
View File

@@ -1,4 +1,4 @@
openapi: "3.0.3"
openapi: "3.0.2"
info:
title: Grassroots Economics CIC Cache
description: Cache of processed transaction data from Ethereum blockchain and worker queues
@@ -9,17 +9,34 @@ info:
email: will@grassecon.org
license:
name: GPLv3
version: 0.1.0
version: 0.2.0
paths:
/tx/{offset}/{limit}:
description: Bloom filter for batch of latest transactions
/defaultlimit:
summary: The default limit value of result sets.
get:
tags:
- transactions
description:
Retrieve default limit
operationId: limit.default
responses:
200:
description: Limit query successful
content:
application/json:
schema:
$ref: "#/components/schemas/Limit"
/tx:
summary: Bloom filter for batch of latest transactions
description: Generate a bloom filter of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get
operationId: tx.get.latest
responses:
200:
description: Transaction query successful.
@@ -29,27 +46,153 @@ paths:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: offset
in: path
schema:
type: integer
format: int32
- name: limit
in: path
schema:
type: integer
format: int32
/tx/{address}/{offset}/{limit}:
description: Bloom filter for batch of latest transactions by account
/tx/{limit}:
summary: Bloom filter for batch of latest transactions
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get
operationId: tx.get.latest.limit
responses:
200:
description: Transaction query successful. Results are ordered from newest to oldest.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
/tx/{limit}/{offset}:
summary: Bloom filter for batch of latest transactions
description: Generate a bloom filter of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.latest.range
responses:
200:
description: Transaction query successful. Results are ordered from newest to oldest.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
/tx/{limit}/{offset}/{block_offset}:
summary: Bloom filter for batch of transactions since a particular block.
description: Generate a bloom filter of the latest transactions since a particular block in the cache. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.latest.range.block.offset
responses:
200:
description: Transaction query successful. Results are ordered from oldest to newest.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_offset
in: path
required: true
schema:
type: integer
format: int32
/tx/{limit}/{offset}/{block_offset}/{block_end}:
summary: Bloom filter for batch of transactions within a particular block range.
description: Generate a bloom filter of the latest transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.latest.range.block.range
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_end
in: path
required: true
schema:
type: integer
format: int32
/tx/{address}:
summary: Bloom filter for batch of latest transactions by account.
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.user
responses:
200:
description: Transaction query successful.
@@ -58,6 +201,30 @@ paths:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: address
in: path
required: true
schema:
type: string
/tx/{address}/{limit}:
summary: Bloom filter for batch of latest transactions by account.
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.user.limit
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: address
@@ -65,26 +232,317 @@ paths:
required: true
schema:
type: string
- name: offset
in: path
schema:
type: integer
format: int32
- name: limit
in: path
required: true
schema:
type: integer
format: int32
/tx/{address}/{limit}/{offset}:
summary: Bloom filter for batch of latest transactions by account
description: Generate a bloom filter of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: tx.get.user.range
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/BlocksBloom"
parameters:
- name: address
in: path
required: true
schema:
type: string
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
/txa:
summary: Cached data for latest transactions.
description: Return data entries of the latest transactions in the cache. The number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.latest
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
/txa/{limit}:
summary: Cached data for latest transactions.
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.latest.limit
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
/txa/{limit}/{offset}:
summary: Cached data for latest transactions.
description: Return data entries of the latest transactions in the cache. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.latest.range
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
/txa/{limit}/{offset}/{block_offset}:
summary: Cached data for transactions since a particular block.
description: Return cached data entries of transactions since a particular block. The block parameter is inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.latest.range.block.offset
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_offset
in: path
required: true
schema:
type: integer
format: int32
/txa/{limit}/{offset}/{block_offset}/{block_end}:
summary: Cached data for transactions within a particular block range.
description: Return cached data entries of transactions within a particular block range in the cache. The block parameters are inclusive. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.latest.range.block.range
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_offset
in: path
required: true
schema:
type: integer
format: int32
- name: block_end
in: path
required: true
schema:
type: integer
format: int32
/txa/{address}:
summary: Cached data for batch of latest transactions by account.
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.user
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: address
in: path
required: true
schema:
type: string
/txa/{address}/{limit}:
summary: Cached data for batch of latest transactions by account.
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.user.limit
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: address
in: path
required: true
schema:
type: string
- name: limit
in: path
required: true
schema:
type: integer
format: int32
/txa/{address}/{limit}/{offset}:
summary: Cached data for batch of latest transactions by account.
description: Return cached data of the latest transactions where a specific account is the spender or beneficiary. If `limit` is 0, the number of maximum number of transactions returned is returned by the `/defaultlimit` API call.
get:
tags:
- transactions
description:
Retrieve transactions
operationId: txa.get.user.range
responses:
200:
description: Transaction query successful.
content:
application/json:
schema:
$ref: "#/components/schemas/TransactionList"
parameters:
- name: address
in: path
required: true
schema:
type: string
- name: limit
in: path
required: true
schema:
type: integer
format: int32
- name: offset
in: path
required: true
schema:
type: integer
format: int32
components:
schemas:
Limit:
type: integer
format: int32
BlocksBloom:
type: object
properties:
low:
type: int
type: integer
format: int32
description: The lowest block number included in the filter
high:
type: integer
format: int32
description: The highest block number included in the filter
block_filter:
type: string
format: byte
@@ -97,6 +555,89 @@ components:
type: string
description: Hashing algorithm (currently only using sha256)
filter_rounds:
type: int
type: integer
format: int32
description: Number of hash rounds used to create the filter
TransactionList:
type: object
properties:
low:
type: integer
format: int32
description: The lowest block number included in the result set
high:
type: integer
format: int32
description: The highest block number included in the filter
data:
type: array
description: Cached transaction data
items:
$ref: "#/components/schemas/Transaction"
Transaction:
type: object
properties:
block_number:
type: integer
format: int64
description: Block number transaction was included in.
tx_hash:
type: string
description: Transaction hash, in hex.
date_block:
type: integer
format: int32
description: Block timestamp.
sender:
type: string
description: Spender address, in hex.
recipient:
type: string
description: Beneficiary address, in hex.
from_value:
type: integer
format: int64
description: Value deducted from spender's balance.
to_value:
type: integer
format: int64
description: Value added to beneficiary's balance.
source_token:
type: string
description: Network address of token in which `from_value` is denominated.
destination_token:
type: string
description: Network address of token in which `to_value` is denominated.
success:
type: boolean
description: Network consensus state on whether the transaction was successful or not.
tx_type:
type: string
enum:
- erc20.faucet
- faucet.give_to
examples:
data_last:
summary: Get the latest cached transactions, using the server's default limit.
value: "/txa"
data_limit:
summary: Get the last 42 cached transactions.
value: "/txa/42"
data_range:
summary: Get the next 42 cached transactions, starting from the 13th (zero-indexed).
value: "/txa/42/13"
data_range_block_offset:
summary: Get the next 42 cached transactions, starting from block 1337 (inclusive).
value: "/txa/42/0/1337"
data_range_block_offset:
summary: Get the next 42 cached transactions within blocks 1337 and 1453 (inclusive).
value: "/txa/42/0/1337/1453"
data_range_block_range:
summary: Get the next 42 cached transactions after the 13th, within blocks 1337 and 1453 (inclusive).
value: "/txa/42/13/1337/1453"

13
apps/cic-cache/docker/Dockerfile
View File

@@ -4,9 +4,9 @@ FROM $DOCKER_REGISTRY/cic-base-images:python-3.8.6-dev-e8eb2ee2
COPY requirements.txt .
ARG EXTRA_PIP_INDEX_URL="https://pip.grassrootseconomics.net"
ARG EXTRA_PIP_INDEX_URL=https://pip.grassrootseconomics.net
ARG EXTRA_PIP_ARGS=""
ARG PIP_INDEX_URL="https://pypi.org/simple"
ARG PIP_INDEX_URL=https://pypi.org/simple
RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
pip install --index-url $PIP_INDEX_URL \
@@ -14,14 +14,9 @@ RUN --mount=type=cache,mode=0755,target=/root/.cache/pip \
--extra-index-url $EXTRA_PIP_INDEX_URL $EXTRA_PIP_ARGS \
-r requirements.txt
COPY . .
RUN python setup.py install
# ini files in config directory defines the configurable parameters for the application
# they can all be overridden by environment variables
# to generate a list of environment variables from configuration, use: confini-dump -z <dir> (executable provided by confini package)
#COPY config/ /usr/local/etc/cic-cache/
COPY . .
RUN pip install . --extra-index-url $EXTRA_PIP_INDEX_URL
# for db migrations
COPY ./aux/wait-for-it/wait-for-it.sh ./

2
apps/cic-cache/docker/db.sh
View File

@@ -2,5 +2,5 @@
set -e
>&2 echo executing database migration
python scripts/migrate.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
python scripts/migrate_cic_cache.py --migrations-dir /usr/local/share/cic-cache/alembic -vv
set +e

15
apps/cic-cache/requirements.txt
View File

@@ -1,14 +1,15 @@
alembic==1.4.2
confini>=0.3.6rc4,<0.5.0
confini~=0.5.3
uwsgi==2.0.19.1
moolb~=0.1.1b2
cic-eth-registry~=0.6.1a1
moolb~=0.2.0
cic-eth-registry~=0.6.6
SQLAlchemy==1.3.20
semver==2.13.0
psycopg2==2.8.6
celery==4.4.7
redis==3.5.3
chainsyncer[sql]>=0.0.6a3,<0.1.0
erc20-faucet>=0.3.2a2, <0.4.0
chainlib-eth>=0.0.9a14,<0.1.0
eth-address-index>=0.2.3a4,<0.3.0
chainsyncer[sql]~=0.0.7
erc20-faucet~=0.3.2
chainlib-eth~=0.0.15
eth-address-index~=0.2.4
okota~=0.2.5

45
apps/cic-cache/scripts/migrate.py → apps/cic-cache/scripts/migrate_cic_cache.py
View File

@@ -1,54 +1,55 @@
#!/usr/bin/python
#!/usr/bin/python3
# standard imports
import os
import argparse
import logging
import re
# external imports
import alembic
from alembic.config import Config as AlembicConfig
import confini
# local imports
from cic_cache.db import dsn_from_config
import cic_cache.cli
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
# BUG: the dbdir doesn't work after script install
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
rootdir = os.path.dirname(os.path.dirname(os.path.realpath(cic_cache.__file__)))
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
migrationsdir = os.path.join(dbdir, 'migrations')
default_migrations_dir = os.path.join(dbdir, 'migrations')
configdir = os.path.join(rootdir, 'cic_cache', 'data', 'config')
#config_dir = os.path.join('/usr/local/etc/cic-cache')
argparser = argparse.ArgumentParser()
argparser.add_argument('-c', type=str, help='config file')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=migrationsdir, type=str, help='path to alembic migrations directory')
arg_flags = cic_cache.cli.argflag_std_base
local_arg_flags = cic_cache.cli.argflag_local_sync
argparser = cic_cache.cli.ArgumentParser(arg_flags)
argparser.process_local_flags(local_arg_flags)
argparser.add_argument('--reset', action='store_true', help='downgrade before upgrading')
argparser.add_argument('-f', action='store_true', help='force action')
argparser.add_argument('-v', action='store_true', help='be verbose')
argparser.add_argument('-vv', action='store_true', help='be more verbose')
argparser.add_argument('-f', '--force', action='store_true', help='force action')
argparser.add_argument('--migrations-dir', dest='migrations_dir', default=default_migrations_dir, type=str, help='migrations directory')
args = argparser.parse_args()
if args.vv:
logging.getLogger().setLevel(logging.DEBUG)
elif args.v:
logging.getLogger().setLevel(logging.INFO)
extra_args = {
'reset': None,
'force': None,
'migrations_dir': None,
}
# process config
config = cic_cache.cli.Config.from_args(args, arg_flags, local_arg_flags, extra_args=extra_args)
config = confini.Config(configdir, args.env_prefix)
config.process()
config.censor('PASSWORD', 'DATABASE')
config.censor('PASSWORD', 'SSL')
logg.debug('config:\n{}'.format(config))
migrations_dir = os.path.join(args.migrations_dir, config.get('DATABASE_ENGINE'))
migrations_dir = os.path.join(config.get('_MIGRATIONS_DIR'), config.get('DATABASE_ENGINE', 'default'))
if not os.path.isdir(migrations_dir):
logg.debug('migrations dir for engine {} not found, reverting to default'.format(config.get('DATABASE_ENGINE')))
migrations_dir = os.path.join(args.migrations_dir, 'default')
# connect to database
dsn = dsn_from_config(config)
dsn = dsn_from_config(config, 'cic_cache')
logg.info('using migrations dir {}'.format(migrations_dir))

3
apps/cic-cache/setup.cfg
View File

@@ -1,6 +1,7 @@
[metadata]
name = cic-cache
description = CIC Cache API and server
version = 0.3.0a2
author = Louis Holbrook
author_email = dev@holbrook.no
url = https://gitlab.com/grassrootseconomics/cic-eth
@@ -34,7 +35,7 @@ packages =
cic_cache.runnable.daemons
cic_cache.runnable.daemons.filters
scripts =
./scripts/migrate.py
./scripts/migrate_cic_cache.py
[options.entry_points]
console_scripts =

64
apps/cic-cache/setup.py
View File

@@ -1,38 +1,39 @@
from setuptools import setup
import configparser
# import configparser
import os
import time
from cic_cache.version import (
version_object,
version_string
)
# import time
class PleaseCommitFirstError(Exception):
pass
def git_hash():
import subprocess
git_diff = subprocess.run(['git', 'diff'], capture_output=True)
if len(git_diff.stdout) > 0:
raise PleaseCommitFirstError()
git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
return git_hash_brief
version_string = str(version_object)
try:
version_git = git_hash()
version_string += '+build.{}'.format(version_git)
except FileNotFoundError:
time_string_pair = str(time.time()).split('.')
version_string += '+build.{}{:<09d}'.format(
time_string_pair[0],
int(time_string_pair[1]),
)
print('final version string will be {}'.format(version_string))
# from cic_cache.version import (
# version_object,
# version_string
# )
#
# class PleaseCommitFirstError(Exception):
# pass
#
# def git_hash():
# import subprocess
# git_diff = subprocess.run(['git', 'diff'], capture_output=True)
# if len(git_diff.stdout) > 0:
# raise PleaseCommitFirstError()
# git_hash = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True)
# git_hash_brief = git_hash.stdout.decode('utf-8')[:8]
# return git_hash_brief
#
# version_string = str(version_object)
#
# try:
# version_git = git_hash()
# version_string += '+build.{}'.format(version_git)
# except FileNotFoundError:
# time_string_pair = str(time.time()).split('.')
# version_string += '+build.{}{:<09d}'.format(
# time_string_pair[0],
# int(time_string_pair[1]),
# )
# print('final version string will be {}'.format(version_string))
requirements = []
f = open('requirements.txt', 'r')
@@ -52,9 +53,8 @@ while True:
test_requirements.append(l.rstrip())
f.close()
setup(
version=version_string,
# version=version_string,
install_requires=requirements,
tests_require=test_requirements,
)

2
apps/cic-cache/test_requirements.txt
View File

@@ -7,4 +7,4 @@ pytest-celery==0.0.0a1
eth_tester==0.5.0b3
py-evm==0.3.0a20
sarafu-faucet~=0.0.7a1
erc20-transfer-authorization>=0.3.5a1,<0.4.0
erc20-transfer-authorization~=0.3.6

29
apps/cic-cache/tests/conftest.py
View File

@@ -6,6 +6,7 @@ import datetime
# external imports
import pytest
import moolb
from chainlib.encode import TxHexNormalizer
# local imports
from cic_cache import db
@@ -42,6 +43,8 @@ def txs(
list_tokens,
):
tx_normalize = TxHexNormalizer()
session = init_database
tx_number = 13
@@ -54,10 +57,10 @@ def txs(
tx_hash_first,
list_defaults['block'],
tx_number,
list_actors['alice'],
list_actors['bob'],
list_tokens['foo'],
list_tokens['foo'],
tx_normalize.wallet_address(list_actors['alice']),
tx_normalize.wallet_address(list_actors['bob']),
tx_normalize.executable_address(list_tokens['foo']),
tx_normalize.executable_address(list_tokens['foo']),
1024,
2048,
True,
@@ -74,10 +77,10 @@ def txs(
tx_hash_second,
list_defaults['block']-1,
tx_number,
list_actors['diane'],
list_actors['alice'],
list_tokens['foo'],
list_tokens['foo'],
tx_normalize.wallet_address(list_actors['diane']),
tx_normalize.wallet_address(list_actors['alice']),
tx_normalize.executable_address(list_tokens['foo']),
tx_normalize.wallet_address(list_tokens['foo']),
1024,
2048,
False,
@@ -103,6 +106,8 @@ def more_txs(
session = init_database
tx_normalize = TxHexNormalizer()
tx_number = 666
tx_hash = '0x' + os.urandom(32).hex()
tx_signed = '0x' + os.urandom(128).hex()
@@ -115,10 +120,10 @@ def more_txs(
tx_hash,
list_defaults['block']+2,
tx_number,
list_actors['alice'],
list_actors['diane'],
list_tokens['bar'],
list_tokens['bar'],
tx_normalize.wallet_address(list_actors['alice']),
tx_normalize.wallet_address(list_actors['diane']),
tx_normalize.executable_address(list_tokens['bar']),
tx_normalize.executable_address(list_tokens['bar']),
2048,
4096,
False,

3
apps/cic-cache/tests/fixtures_config.py
View File

@@ -14,7 +14,8 @@ logg = logging.getLogger(__file__)
@pytest.fixture(scope='session')
def load_config():
config_dir = os.path.join(root_dir, 'config/test')
conf = confini.Config(config_dir, 'CICTEST')
schema_config_dir = os.path.join(root_dir, 'cic_cache', 'data', 'config')
conf = confini.Config(schema_config_dir, 'CICTEST', override_dirs=config_dir)
conf.process()
logg.debug('config {}'.format(conf))
return conf

8
apps/cic-cache/tests/fixtures_database.py
View File

@@ -24,11 +24,15 @@ def database_engine(
if load_config.get('DATABASE_ENGINE') == 'sqlite':
SessionBase.transactional = False
SessionBase.poolable = False
name = 'cic_cache'
database_name = name
if load_config.get('DATABASE_PREFIX'):
database_name = '{}_{}'.format(load_config.get('DATABASE_PREFIX'), database_name)
try:
os.unlink(load_config.get('DATABASE_NAME'))
os.unlink(database_name)
except FileNotFoundError:
pass
dsn = dsn_from_config(load_config)
dsn = dsn_from_config(load_config, name)
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
return dsn

4
apps/cic-cache/tests/test_api.py
View File

@@ -14,7 +14,7 @@ def test_api_all_data(
):
env = {
'PATH_INFO': '/txa/410000/420000',
'PATH_INFO': '/txa/100/0/410000/420000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}
j = process_transactions_all_data(init_database, env)
@@ -23,7 +23,7 @@ def test_api_all_data(
assert len(o['data']) == 2
env = {
'PATH_INFO': '/txa/420000/410000',
'PATH_INFO': '/txa/100/0/420000/410000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}

47
apps/cic-cache/tests/test_cache.py
View File

@@ -6,6 +6,7 @@ import json
# external imports
import pytest
from chainlib.encode import TxHexNormalizer
# local imports
from cic_cache import db
@@ -62,6 +63,8 @@ def test_cache_ranges(
session = init_database
tx_normalize = TxHexNormalizer()
oldest = list_defaults['block'] - 1
mid = list_defaults['block']
newest = list_defaults['block'] + 2
@@ -100,32 +103,39 @@ def test_cache_ranges(
assert b[1] == mid
# now check when supplying account
b = c.load_transactions_account(list_actors['alice'], 0, 100)
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account(account, 0, 100)
assert b[0] == oldest
assert b[1] == newest
b = c.load_transactions_account(list_actors['bob'], 0, 100)
account = tx_normalize.wallet_address(list_actors['bob'])
b = c.load_transactions_account(account, 0, 100)
assert b[0] == mid
assert b[1] == mid
b = c.load_transactions_account(list_actors['diane'], 0, 100)
account = tx_normalize.wallet_address(list_actors['diane'])
b = c.load_transactions_account(account, 0, 100)
assert b[0] == oldest
assert b[1] == newest
# add block filter to the mix
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
assert b[0] == mid
assert b[1] == newest
b = c.load_transactions_account(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'])
assert b[0] == mid
assert b[1] == newest
b = c.load_transactions_account(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['bob'])
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
assert b[0] == mid
assert b[1] == mid
b = c.load_transactions_account(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['diane'])
b = c.load_transactions_account(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
assert b[0] == oldest
assert b[1] == oldest
@@ -140,6 +150,8 @@ def test_cache_ranges_data(
session = init_database
tx_normalize = TxHexNormalizer()
oldest = list_defaults['block'] - 1
mid = list_defaults['block']
newest = list_defaults['block'] + 2
@@ -203,7 +215,8 @@ def test_cache_ranges_data(
assert b[2][1]['tx_hash'] == more_txs[1]
# now check when supplying account
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100)
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account_with_data(account, 0, 100)
assert b[0] == oldest
assert b[1] == newest
assert len(b[2]) == 3
@@ -211,13 +224,15 @@ def test_cache_ranges_data(
assert b[2][1]['tx_hash'] == more_txs[1]
assert b[2][2]['tx_hash'] == more_txs[2]
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100)
account = tx_normalize.wallet_address(list_actors['bob'])
b = c.load_transactions_account_with_data(account, 0, 100)
assert b[0] == mid
assert b[1] == mid
assert len(b[2]) == 1
assert b[2][0]['tx_hash'] == more_txs[1]
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100)
account = tx_normalize.wallet_address(list_actors['diane'])
b = c.load_transactions_account_with_data(account, 0, 100)
assert b[0] == oldest
assert b[1] == newest
assert len(b[2]) == 2
@@ -225,27 +240,31 @@ def test_cache_ranges_data(
assert b[2][1]['tx_hash'] == more_txs[2]
# add block filter to the mix
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
assert b[0] == mid
assert b[1] == newest
assert len(b[2]) == 2
assert b[2][0]['tx_hash'] == more_txs[0]
assert b[2][1]['tx_hash'] == more_txs[1]
b = c.load_transactions_account_with_data(list_actors['alice'], 0, 100, block_offset=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['alice'])
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'])
assert b[0] == mid
assert b[1] == newest
assert len(b[2]) == 2
assert b[2][0]['tx_hash'] == more_txs[0]
assert b[2][1]['tx_hash'] == more_txs[1]
b = c.load_transactions_account_with_data(list_actors['bob'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['bob'])
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
assert b[0] == mid
assert b[1] == mid
assert len(b[2]) == 1
assert b[2][0]['tx_hash'] == more_txs[1]
b = c.load_transactions_account_with_data(list_actors['diane'], 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
account = tx_normalize.wallet_address(list_actors['diane'])
b = c.load_transactions_account_with_data(account, 0, 100, block_offset=list_defaults['block'] - 1, block_limit=list_defaults['block'])
assert b[0] == oldest
assert b[1] == oldest
assert len(b[2]) == 1

16
apps/cic-cache/tests/test_query.py
View File

@@ -82,7 +82,7 @@ def test_query_regex(
[
('alice', None, None, [(420000, 13), (419999, 42)]),
('alice', None, 1, [(420000, 13)]),
('alice', 1, None, [(419999, 42)]), # 420000 == list_defaults['block']
('alice', 1, 1, [(419999, 42)]), # 420000 == list_defaults['block']
('alice', 2, None, []), # 420000 == list_defaults['block']
],
)
@@ -107,10 +107,11 @@ def test_query_process_txs_account(
path_info = '/tx/user/0x' + strip_0x(actor)
if query_offset != None:
path_info += '/' + str(query_offset)
if query_limit != None:
if query_offset == None:
path_info += '/0'
path_info += '/' + str(query_limit)
if query_limit == None:
query_limit = 100
path_info += '/' + str(query_limit)
if query_offset == None:
path_info += '/0'
env = {
'PATH_INFO': path_info,
}
@@ -192,7 +193,7 @@ def test_query_process_txs_bloom(
@pytest.mark.parametrize(
'query_block_start, query_block_end, query_match_count',
[
(None, 42, 0),
(1, 42, 0),
(420000, 420001, 1),
(419999, 419999, 1), # matches are inclusive
(419999, 420000, 2),
@@ -211,7 +212,7 @@ def test_query_process_txs_data(
query_match_count,
):
path_info = '/txa'
path_info = '/txa/100/0'
if query_block_start != None:
path_info += '/' + str(query_block_start)
if query_block_end != None:
@@ -227,4 +228,5 @@ def test_query_process_txs_data(
assert r != None
o = json.loads(r[1])
logg.debug('oo {}'.format(o))
assert len(o['data']) == query_match_count