Merge branch 'master' into bvander/move-scripts-to-e2e-folder

This commit is contained in:
nolash 2021-05-19 11:58:40 +02:00
commit eefb3ec4ff
17 changed files with 298 additions and 98 deletions

View File

@ -1,22 +1,28 @@
# standard imports # standard imports
import logging import logging
import datetime
# third-party imports # external imports
import moolb import moolb
# local imports # local imports
from cic_cache.db import list_transactions_mined from cic_cache.db.list import (
from cic_cache.db import list_transactions_account_mined list_transactions_mined,
list_transactions_account_mined,
list_transactions_mined_with_data,
)
logg = logging.getLogger() logg = logging.getLogger()
class BloomCache: class Cache:
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
class BloomCache(Cache):
@staticmethod @staticmethod
def __get_filter_size(n): def __get_filter_size(n):
n = 8192 * 8 n = 8192 * 8
@ -87,3 +93,43 @@ class BloomCache:
f_blocktx.add(block + tx) f_blocktx.add(block + tx)
logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block)) logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),) return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
class DataCache(Cache):
def load_transactions_with_data(self, offset, end):
rows = list_transactions_mined_with_data(self.session, offset, end)
tx_cache = []
highest_block = -1;
lowest_block = -1;
date_is_str = None # stick this in startup
for r in rows:
if highest_block == -1:
highest_block = r['block_number']
lowest_block = r['block_number']
tx_type = 'unknown'
if r['value'] != None:
tx_type = '{}.{}'.format(r['domain'], r['value'])
if date_is_str == None:
date_is_str = type(r['date_block']).__name__ == 'str'
o = {
'block_number': r['block_number'],
'tx_hash': r['tx_hash'],
'date_block': r['date_block'],
'sender': r['sender'],
'recipient': r['recipient'],
'from_value': int(r['from_value']),
'to_value': int(r['to_value']),
'source_token': r['source_token'],
'destination_token': r['destination_token'],
'tx_type': tx_type,
}
if date_is_str:
o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
tx_cache.append(o)
return (lowest_block, highest_block, tx_cache)

View File

@ -28,6 +28,26 @@ def list_transactions_mined(
return r return r
def list_transactions_mined_with_data(
session,
offset,
end,
):
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
:param offset: Offset in data set to return transactions from
:type offset: int
:param limit: Max number of transactions to retrieve
:type limit: int
:result: Result set
:rtype: SQLAlchemy.ResultProxy
"""
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
r = session.execute(s)
return r
def list_transactions_account_mined( def list_transactions_account_mined(
session, session,
address, address,

View File

@ -0,0 +1,110 @@
# standard imports
import logging
import json
import re
import base64
# local imports
from cic_cache.cache import (
BloomCache,
DataCache,
)
logg = logging.getLogger(__name__)
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
DEFAULT_LIMIT = 100
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
address = r[1]
if r[2] == None:
address = '0x' + address
offset = DEFAULT_LIMIT
if r.lastindex > 2:
offset = r[3]
limit = 0
if r.lastindex > 3:
limit = r[4]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_data(session, env):
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
if not r:
return None
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
return None
offset = r[1]
end = r[2]
if r[2] < r[1]:
raise ValueError('cart before the horse, dude')
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
o = {
'low': lowest_block,
'high': highest_block,
'data': tx_cache,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)

View File

@ -1,18 +1,20 @@
# standard imports # standard imports
import os import os
import re
import logging import logging
import argparse import argparse
import json
import base64 import base64
# third-party imports # external imports
import confini import confini
# local imports # local imports
from cic_cache import BloomCache
from cic_cache.db import dsn_from_config from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import (
process_transactions_account_bloom,
process_transactions_all_bloom,
process_transactions_all_data,
)
logging.basicConfig(level=logging.WARNING) logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger() logg = logging.getLogger()
@ -44,72 +46,6 @@ logg.debug('config:\n{}'.format(config))
dsn = dsn_from_config(config) dsn = dsn_from_config(config)
SessionBase.connect(dsn, config.true('DATABASE_DEBUG')) SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
DEFAULT_LIMIT = 100
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
address = r[1]
if r[2] == None:
address = '0x' + address
offset = DEFAULT_LIMIT
if r.lastindex > 2:
offset = r[3]
limit = 0
if r.lastindex > 3:
limit = r[4]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
# uwsgi application # uwsgi application
def application(env, start_response): def application(env, start_response):
@ -119,10 +55,16 @@ def application(env, start_response):
session = SessionBase.create_session() session = SessionBase.create_session()
for handler in [ for handler in [
process_transactions_all_data,
process_transactions_all_bloom, process_transactions_all_bloom,
process_transactions_account_bloom, process_transactions_account_bloom,
]: ]:
r = handler(session, env) r = None
try:
r = handler(session, env)
except ValueError as e:
start_response('400 {}'.format(str(e)))
return []
if r != None: if r != None:
(mime_type, content) = r (mime_type, content) = r
break break

View File

@ -88,3 +88,16 @@ def txs(
tx_hash_first, tx_hash_first,
tx_hash_second, tx_hash_second,
] ]
@pytest.fixture(scope='function')
def tag_txs(
init_database,
txs,
):
db.add_tag(init_database, 'taag', domain='test')
init_database.commit()
db.tag_transaction(init_database, txs[1], 'taag', domain='test')

View File

@ -0,0 +1,31 @@
# standard imports
import json
# external imports
import pytest
# local imports
from cic_cache.runnable.daemons.query import process_transactions_all_data
def test_api_all_data(
init_database,
txs,
):
env = {
'PATH_INFO': '/txa/410000/420000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}
j = process_transactions_all_data(init_database, env)
o = json.loads(j[1])
assert len(o['data']) == 2
env = {
'PATH_INFO': '/txa/420000/410000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}
with pytest.raises(ValueError):
j = process_transactions_all_data(init_database, env)

View File

@ -9,6 +9,7 @@ import pytest
# local imports # local imports
from cic_cache import BloomCache from cic_cache import BloomCache
from cic_cache.cache import DataCache
logg = logging.getLogger() logg = logging.getLogger()
@ -33,3 +34,23 @@ def test_cache(
assert b[0] == list_defaults['block'] - 1 assert b[0] == list_defaults['block'] - 1
def test_cache_data(
init_database,
list_defaults,
list_actors,
list_tokens,
txs,
tag_txs,
):
session = init_database
c = DataCache(session)
b = c.load_transactions_with_data(410000, 420000)
assert len(b[2]) == 2
assert b[2][0]['tx_hash'] == txs[1]
assert b[2][1]['tx_type'] == 'unknown'
assert b[2][0]['tx_type'] == 'test.taag'

View File

@ -16,4 +16,6 @@ def default_token(self):
return { return {
'symbol': self.default_token_symbol, 'symbol': self.default_token_symbol,
'address': self.default_token_address, 'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
} }

View File

@ -22,6 +22,7 @@ from chainlib.eth.connection import (
from chainlib.chain import ChainSpec from chainlib.chain import ChainSpec
from chainqueue.db.models.otx import Otx from chainqueue.db.models.otx import Otx
from cic_eth_registry.error import UnknownContractError from cic_eth_registry.error import UnknownContractError
from cic_eth_registry.erc20 import ERC20Token
import liveness.linux import liveness.linux
@ -207,6 +208,11 @@ def main():
BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL') BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol) BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
default_token.load(rpc)
BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name
BaseTask.run_dir = config.get('CIC_RUN_DIR') BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address)) logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

View File

@ -67,6 +67,8 @@ def main():
token_info = t.get() token_info = t.get()
print('Default token symbol: {}'.format(token_info['symbol'])) print('Default token symbol: {}'.format(token_info['symbol']))
print('Default token address: {}'.format(token_info['address'])) print('Default token address: {}'.format(token_info['address']))
logg.debug('Default token name: {}'.format(token_info['name']))
logg.debug('Default token decimals: {}'.format(token_info['decimals']))
if __name__ == '__main__': if __name__ == '__main__':

View File

@ -33,6 +33,8 @@ class BaseTask(celery.Task):
create_gas_oracle = RPCGasOracle create_gas_oracle = RPCGasOracle
default_token_address = None default_token_address = None
default_token_symbol = None default_token_symbol = None
default_token_name = None
default_token_decimals = None
run_dir = '/run' run_dir = '/run'
def create_session(self): def create_session(self):

View File

@ -1,7 +1,7 @@
en: en:
account_successfully_created: |- account_successfully_created: |-
Hello, you have been registered on Sarafu Network! Your balance is %{balance} %{token_symbol}. To use dial *483*46#. For help 0757628885. You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
received_tokens: |- received_tokens: |-
Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}. Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
terms: |- terms: |-
By using the service, you agree to the terms and conditions at https://www.grassrootseconomics.org/terms-and-conditions. By using the service, you agree to the terms and conditions at http://grassecon.org/tos

View File

@ -1,7 +1,7 @@
sw: sw:
account_successfully_created: |- account_successfully_created: |-
Habari, umesajiliwa kwa huduma ya sarafu! Salio lako ni %{token_symbol} %{balance}. Kutumia bonyeza *483*46#. Kwa Usaidizi 0757628885. Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
received_tokens: |- received_tokens: |-
Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}. Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
terms: |- terms: |-
Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo https://www.grassrootseconomics.org/terms-and-conditions. Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

View File

@ -1,29 +1,30 @@
en: en:
kenya: kenya:
initial_language_selection: |- initial_language_selection: |-
CON Welcome to Sarafu CON Welcome to Sarafu Network
1. English 1. English
2. Kiswahili 2. Kiswahili
3. Help 3. Help
initial_pin_entry: |- initial_pin_entry: |-
CON Please enter a PIN to manage your account. CON Please enter a new four number PIN for your account.
0. Back 0. Back
initial_pin_confirmation: |- initial_pin_confirmation: |-
CON Enter your PIN again CON Enter your four number PIN again
0. Back 0. Back
enter_given_name: |- enter_given_name: |-
CON Enter first name CON Enter first name
0. Back 0. Back
enter_family_name: |- enter_family_name: |-
CON Enter last name CON Enter family name
0. Back 0. Back
enter_gender: |- enter_gender: |-
CON Enter gender CON Enter gender
1. Male 1. Male
2. Female 2. Female
3. Other
0. Back 0. Back
enter_location: |- enter_location: |-
CON Enter location CON Enter your location
0. Back 0. Back
enter_products: |- enter_products: |-
CON Please enter a product or service you offer CON Please enter a product or service you offer
@ -83,34 +84,34 @@ en:
Please enter your PIN to confirm. Please enter your PIN to confirm.
0. Back 0. Back
retry: |- retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining. CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back 0. Back
display_metadata_pin_authorization: display_metadata_pin_authorization:
first: |- first: |-
CON Please enter your PIN. CON Please enter your PIN
0. Back 0. Back
retry: |- retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining. CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back 0. Back
account_balances_pin_authorization: account_balances_pin_authorization:
first: |- first: |-
CON Please enter your PIN to view balances. CON Please enter your PIN to view balances
0. Back 0. Back
retry: |- retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining. CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back 0. Back
account_statement_pin_authorization: account_statement_pin_authorization:
first: |- first: |-
CON Please enter your PIN to view statement. CON Please enter your PIN to view statement
0. Back 0. Back
retry: |- retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining. CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back 0. Back
account_balances: |- account_balances: |-
CON Your balances are as follows: CON Your balances are as follows:
balance: %{operational_balance} %{token_symbol} balance: %{operational_balance} %{token_symbol}
taxes: %{tax} %{token_symbol} fees: %{tax} %{token_symbol}
bonsuses: %{bonus} %{token_symbol} rewards: %{bonus} %{token_symbol}
0. Back 0. Back
first_transaction_set: |- first_transaction_set: |-
CON %{first_transaction_set} CON %{first_transaction_set}
@ -140,9 +141,9 @@ en:
exit_pin_blocked: |- exit_pin_blocked: |-
END Your PIN has been blocked. For help, please call %{support_phone}. END Your PIN has been blocked. For help, please call %{support_phone}.
exit_invalid_pin: |- exit_invalid_pin: |-
END The PIN you have entered is Invalid. PIN must consist of 4 digits. For help, call %{support_phone}. END The PIN you have entered is invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
exit_invalid_new_pin: |- exit_invalid_new_pin: |-
END The PIN you have entered is Invalid. PIN must be different from your current PIN. For help, call %{support_phone}. END The PIN you have entered is invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
exit_pin_mismatch: |- exit_pin_mismatch: |-
END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}. END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
exit_invalid_recipient: |- exit_invalid_recipient: |-

View File

@ -171,6 +171,7 @@ if __name__ == '__main__':
logg.error('load error for {}: {}'.format(y, e)) logg.error('load error for {}: {}'.format(y, e))
continue continue
f.close() f.close()
logg.debug('deserializing {} {}'.format(filepath, o))
u = Person.deserialize(o) u = Person.deserialize(o)
new_address = register_eth(i, u) new_address = register_eth(i, u)

View File

@ -105,7 +105,7 @@ def genId(addr, typ):
def genDate(): def genDate():
ts = random.randint(ts_then, ts_now) ts = random.randint(ts_then, ts_now)
return datetime.datetime.fromtimestamp(ts).timestamp() return int(datetime.datetime.fromtimestamp(ts).timestamp())
def genPhone(): def genPhone():
@ -193,6 +193,7 @@ def prepareLocalFilePath(datadir, address):
if __name__ == '__main__': if __name__ == '__main__':
base_dir = os.path.join(user_dir, 'old') base_dir = os.path.join(user_dir, 'old')
ussd_dir = os.path.join(user_dir, 'ussd')
os.makedirs(base_dir, exist_ok=True) os.makedirs(base_dir, exist_ok=True)
fa = open(os.path.join(user_dir, 'balances.csv'), 'w') fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
@ -223,6 +224,8 @@ if __name__ == '__main__':
f = open('{}/{}'.format(d, uid + '.json'), 'w') f = open('{}/{}'.format(d, uid + '.json'), 'w')
json.dump(o.serialize(), f) json.dump(o.serialize(), f)
f.close() f.close()
d = prepareLocalFilePath(ussd_dir, uid)
x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w') x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w')
json.dump(ussd_data, x) json.dump(ussd_data, x)
x.close() x.close()

View File

@ -23,7 +23,7 @@ from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.gas import RPCGasOracle from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle from chainlib.eth.nonce import RPCNonceOracle
from cic_types.processor import generate_metadata_pointer from cic_types.processor import generate_metadata_pointer
from eth_accounts_index import AccountRegistry from eth_accounts_index.registry import AccountRegistry
from eth_contract_registry import Registry from eth_contract_registry import Registry
from crypto_dev_signer.keystore.dict import DictKeystore from crypto_dev_signer.keystore.dict import DictKeystore
from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer