kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' into bvander/move-scripts-to-e2e-folder

Browse Source
This commit is contained in:
nolash 2021-05-19 11:58:40 +02:00
commit eefb3ec4ff
17 changed files with 298 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
apps/cic-cache/cic_cache/cache.py
View File

@ -1,22 +1,28 @@
# standard imports
import logging
import datetime
# third-party imports
# external imports
import moolb
# local imports
from cic_cache.db import list_transactions_mined
from cic_cache.db import list_transactions_account_mined
from cic_cache.db.list import (
list_transactions_mined,
list_transactions_account_mined,
list_transactions_mined_with_data,
)
logg = logging.getLogger()
class BloomCache:
class Cache:
def __init__(self, session):
self.session = session
class BloomCache(Cache):
@staticmethod
def __get_filter_size(n):
n = 8192 * 8
@ -87,3 +93,43 @@ class BloomCache:
f_blocktx.add(block + tx)
logg.debug('added block {} tx {} lo {} hi {}'.format(r[0], r[1], lowest_block, highest_block))
return (lowest_block, highest_block, f_block.to_bytes(), f_blocktx.to_bytes(),)
class DataCache(Cache):
def load_transactions_with_data(self, offset, end):
rows = list_transactions_mined_with_data(self.session, offset, end)
tx_cache = []
highest_block = -1;
lowest_block = -1;
date_is_str = None # stick this in startup
for r in rows:
if highest_block == -1:
highest_block = r['block_number']
lowest_block = r['block_number']
tx_type = 'unknown'
if r['value'] != None:
tx_type = '{}.{}'.format(r['domain'], r['value'])
if date_is_str == None:
date_is_str = type(r['date_block']).__name__ == 'str'
o = {
'block_number': r['block_number'],
'tx_hash': r['tx_hash'],
'date_block': r['date_block'],
'sender': r['sender'],
'recipient': r['recipient'],
'from_value': int(r['from_value']),
'to_value': int(r['to_value']),
'source_token': r['source_token'],
'destination_token': r['destination_token'],
'tx_type': tx_type,
}
if date_is_str:
o['date_block'] = datetime.datetime.fromisoformat(r['date_block'])
tx_cache.append(o)
return (lowest_block, highest_block, tx_cache)

20
apps/cic-cache/cic_cache/db/list.py
View File

@ -28,6 +28,26 @@ def list_transactions_mined(
return r
def list_transactions_mined_with_data(
session,
offset,
end,
):
"""Executes db query to return all confirmed transactions according to the specified offset and limit.
:param offset: Offset in data set to return transactions from
:type offset: int
:param limit: Max number of transactions to retrieve
:type limit: int
:result: Result set
:rtype: SQLAlchemy.ResultProxy
"""
s = "SELECT tx_hash, block_number, date_block, sender, recipient, from_value, to_value, source_token, destination_token, domain, value FROM tx LEFT JOIN tag_tx_link ON tx.id = tag_tx_link.tx_id LEFT JOIN tag ON tag_tx_link.tag_id = tag.id WHERE block_number >= {} AND block_number <= {} ORDER BY block_number ASC, tx_index ASC".format(offset, end)
r = session.execute(s)
return r
def list_transactions_account_mined(
session,
address,

110
apps/cic-cache/cic_cache/runnable/daemons/query.py Normal file
View File

@ -0,0 +1,110 @@
# standard imports
import logging
import json
import re
import base64
# local imports
from cic_cache.cache import (
BloomCache,
DataCache,
)
logg = logging.getLogger(__name__)
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
re_transactions_all_data = r'/txa/(\d+)/(\d+)/?'
DEFAULT_LIMIT = 100
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
address = r[1]
if r[2] == None:
address = '0x' + address
offset = DEFAULT_LIMIT
if r.lastindex > 2:
offset = r[3]
limit = 0
if r.lastindex > 3:
limit = r[4]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_data(session, env):
r = re.match(re_transactions_all_data, env.get('PATH_INFO'))
if not r:
return None
if env.get('HTTP_X_CIC_CACHE_MODE') != 'all':
return None
offset = r[1]
end = r[2]
if r[2] < r[1]:
raise ValueError('cart before the horse, dude')
c = DataCache(session)
(lowest_block, highest_block, tx_cache) = c.load_transactions_with_data(offset, end)
for r in tx_cache:
r['date_block'] = r['date_block'].timestamp()
o = {
'low': lowest_block,
'high': highest_block,
'data': tx_cache,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)

82
apps/cic-cache/cic_cache/runnable/daemons/server.py
View File

@ -1,18 +1,20 @@
# standard imports
import os
import re
import logging
import argparse
import json
import base64
# third-party imports
# external imports
import confini
# local imports
from cic_cache import BloomCache
from cic_cache.db import dsn_from_config
from cic_cache.db.models.base import SessionBase
from cic_cache.runnable.daemons.query import (
process_transactions_account_bloom,
process_transactions_all_bloom,
process_transactions_all_data,
)
logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
@ -44,72 +46,6 @@ logg.debug('config:\n{}'.format(config))
dsn = dsn_from_config(config)
SessionBase.connect(dsn, config.true('DATABASE_DEBUG'))
re_transactions_all_bloom = r'/tx/(\d+)?/?(\d+)/?'
re_transactions_account_bloom = r'/tx/user/((0x)?[a-fA-F0-9]+)/?(\d+)?/?(\d+)/?'
DEFAULT_LIMIT = 100
def process_transactions_account_bloom(session, env):
r = re.match(re_transactions_account_bloom, env.get('PATH_INFO'))
if not r:
return None
address = r[1]
if r[2] == None:
address = '0x' + address
offset = DEFAULT_LIMIT
if r.lastindex > 2:
offset = r[3]
limit = 0
if r.lastindex > 3:
limit = r[4]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions_account(address, offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
def process_transactions_all_bloom(session, env):
r = re.match(re_transactions_all_bloom, env.get('PATH_INFO'))
if not r:
return None
offset = DEFAULT_LIMIT
if r.lastindex > 0:
offset = r[1]
limit = 0
if r.lastindex > 1:
limit = r[2]
c = BloomCache(session)
(lowest_block, highest_block, bloom_filter_block, bloom_filter_tx) = c.load_transactions(offset, limit)
o = {
'alg': 'sha256',
'low': lowest_block,
'high': highest_block,
'block_filter': base64.b64encode(bloom_filter_block).decode('utf-8'),
'blocktx_filter': base64.b64encode(bloom_filter_tx).decode('utf-8'),
'filter_rounds': 3,
}
j = json.dumps(o)
return ('application/json', j.encode('utf-8'),)
# uwsgi application
def application(env, start_response):
@ -119,10 +55,16 @@ def application(env, start_response):
session = SessionBase.create_session()
for handler in [
process_transactions_all_data,
process_transactions_all_bloom,
process_transactions_account_bloom,
]:
r = None
try:
r = handler(session, env)
except ValueError as e:
start_response('400 {}'.format(str(e)))
return []
if r != None:
(mime_type, content) = r
break

13
apps/cic-cache/tests/conftest.py
View File

@ -88,3 +88,16 @@ def txs(
tx_hash_first,
tx_hash_second,
]
@pytest.fixture(scope='function')
def tag_txs(
init_database,
txs,
):
db.add_tag(init_database, 'taag', domain='test')
init_database.commit()
db.tag_transaction(init_database, txs[1], 'taag', domain='test')

31
apps/cic-cache/tests/test_api.py Normal file
View File

@ -0,0 +1,31 @@
# standard imports
import json
# external imports
import pytest
# local imports
from cic_cache.runnable.daemons.query import process_transactions_all_data
def test_api_all_data(
init_database,
txs,
):
env = {
'PATH_INFO': '/txa/410000/420000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}
j = process_transactions_all_data(init_database, env)
o = json.loads(j[1])
assert len(o['data']) == 2
env = {
'PATH_INFO': '/txa/420000/410000',
'HTTP_X_CIC_CACHE_MODE': 'all',
}
with pytest.raises(ValueError):
j = process_transactions_all_data(init_database, env)

21
apps/cic-cache/tests/test_cache.py
View File

@ -9,6 +9,7 @@ import pytest
# local imports
from cic_cache import BloomCache
from cic_cache.cache import DataCache
logg = logging.getLogger()
@ -33,3 +34,23 @@ def test_cache(
assert b[0] == list_defaults['block'] - 1
def test_cache_data(
init_database,
list_defaults,
list_actors,
list_tokens,
txs,
tag_txs,
):
session = init_database
c = DataCache(session)
b = c.load_transactions_with_data(410000, 420000)
assert len(b[2]) == 2
assert b[2][0]['tx_hash'] == txs[1]
assert b[2][1]['tx_type'] == 'unknown'
assert b[2][0]['tx_type'] == 'test.taag'

2
apps/cic-eth/cic_eth/admin/token.py
View File

@ -16,4 +16,6 @@ def default_token(self):
return {
'symbol': self.default_token_symbol,
'address': self.default_token_address,
'name': self.default_token_name,
'decimals': self.default_token_decimals,
}

6
apps/cic-eth/cic_eth/runnable/daemons/tasker.py
View File

@ -22,6 +22,7 @@ from chainlib.eth.connection import (
from chainlib.chain import ChainSpec
from chainqueue.db.models.otx import Otx
from cic_eth_registry.error import UnknownContractError
from cic_eth_registry.erc20 import ERC20Token
import liveness.linux
@ -207,6 +208,11 @@ def main():
BaseTask.default_token_symbol = config.get('CIC_DEFAULT_TOKEN_SYMBOL')
BaseTask.default_token_address = registry.by_name(BaseTask.default_token_symbol)
default_token = ERC20Token(chain_spec, rpc, BaseTask.default_token_address)
default_token.load(rpc)
BaseTask.default_token_decimals = default_token.decimals
BaseTask.default_token_name = default_token.name
BaseTask.run_dir = config.get('CIC_RUN_DIR')
logg.info('default token set to {} {}'.format(BaseTask.default_token_symbol, BaseTask.default_token_address))

2
apps/cic-eth/cic_eth/runnable/info.py
View File

@ -67,6 +67,8 @@ def main():
token_info = t.get()
print('Default token symbol: {}'.format(token_info['symbol']))
print('Default token address: {}'.format(token_info['address']))
logg.debug('Default token name: {}'.format(token_info['name']))
logg.debug('Default token decimals: {}'.format(token_info['decimals']))
if __name__ == '__main__':

2
apps/cic-eth/cic_eth/task.py
View File

@ -33,6 +33,8 @@ class BaseTask(celery.Task):
create_gas_oracle = RPCGasOracle
default_token_address = None
default_token_symbol = None
default_token_name = None
default_token_decimals = None
run_dir = '/run'
def create_session(self):

4
apps/cic-ussd/var/lib/locale/sms.en.yml
View File

@ -1,7 +1,7 @@
en:
account_successfully_created: |-
Hello, you have been registered on Sarafu Network! Your balance is %{balance} %{token_symbol}. To use dial *483*46#. For help 0757628885.
You have been registered on Sarafu Network! To use dial *384*96# on Safaricom and *483*96# on other networks. For help %{support_phone}.
received_tokens: |-
Successfully received %{amount} %{token_symbol} from %{tx_sender_information} %{timestamp}. New balance is %{balance} %{token_symbol}.
terms: |-
By using the service, you agree to the terms and conditions at https://www.grassrootseconomics.org/terms-and-conditions.
By using the service, you agree to the terms and conditions at http://grassecon.org/tos

4
apps/cic-ussd/var/lib/locale/sms.sw.yml
View File

@ -1,7 +1,7 @@
sw:
account_successfully_created: |-
Habari, umesajiliwa kwa huduma ya sarafu! Salio lako ni %{token_symbol} %{balance}. Kutumia bonyeza *483*46#. Kwa Usaidizi 0757628885.
Umesajiliwa kwa huduma ya Sarafu! Kutumia bonyeza *384*96# Safaricom ama *483*46# kwa utandao tofauti. Kwa Usaidizi %{support_phone}.
received_tokens: |-
Umepokea %{amount} %{token_symbol} kutoka kwa %{tx_sender_information} %{timestamp}. Salio la %{token_symbol} ni %{balance}.
terms: |-
Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo https://www.grassrootseconomics.org/terms-and-conditions.
Kwa kutumia hii huduma, umekubali sheria na masharti yafuatayo http://grassecon.org/tos

33
apps/cic-ussd/var/lib/locale/ussd.en.yml
View File

@ -1,29 +1,30 @@
en:
kenya:
initial_language_selection: |-
CON Welcome to Sarafu
CON Welcome to Sarafu Network
1. English
2. Kiswahili
3. Help
initial_pin_entry: |-
CON Please enter a PIN to manage your account.
CON Please enter a new four number PIN for your account.
0. Back
initial_pin_confirmation: |-
CON Enter your PIN again
CON Enter your four number PIN again
0. Back
enter_given_name: |-
CON Enter first name
0. Back
enter_family_name: |-
CON Enter last name
CON Enter family name
0. Back
enter_gender: |-
CON Enter gender
1. Male
2. Female
3. Other
0. Back
enter_location: |-
CON Enter location
CON Enter your location
0. Back
enter_products: |-
CON Please enter a product or service you offer
@ -83,34 +84,34 @@ en:
Please enter your PIN to confirm.
0. Back
retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back
display_metadata_pin_authorization:
first: |-
CON Please enter your PIN.
CON Please enter your PIN
0. Back
retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back
account_balances_pin_authorization:
first: |-
CON Please enter your PIN to view balances.
CON Please enter your PIN to view balances
0. Back
retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back
account_statement_pin_authorization:
first: |-
CON Please enter your PIN to view statement.
CON Please enter your PIN to view statement
0. Back
retry: |-
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining.
CON Please enter your PIN. You have %{remaining_attempts} attempts remaining
0. Back
account_balances: |-
CON Your balances are as follows:
balance: %{operational_balance} %{token_symbol}
taxes: %{tax} %{token_symbol}
bonsuses: %{bonus} %{token_symbol}
fees: %{tax} %{token_symbol}
rewards: %{bonus} %{token_symbol}
0. Back
first_transaction_set: |-
CON %{first_transaction_set}
@ -140,9 +141,9 @@ en:
exit_pin_blocked: |-
END Your PIN has been blocked. For help, please call %{support_phone}.
exit_invalid_pin: |-
END The PIN you have entered is Invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
END The PIN you have entered is invalid. PIN must consist of 4 digits. For help, call %{support_phone}.
exit_invalid_new_pin: |-
END The PIN you have entered is Invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
END The PIN you have entered is invalid. PIN must be different from your current PIN. For help, call %{support_phone}.
exit_pin_mismatch: |-
END The new PIN does not match the one you entered. Please try again. For help, call %{support_phone}.
exit_invalid_recipient: |-

1
apps/data-seeding/cic_eth/import_users.py
View File

@ -171,6 +171,7 @@ if __name__ == '__main__':
logg.error('load error for {}: {}'.format(y, e))
continue
f.close()
logg.debug('deserializing {} {}'.format(filepath, o))
u = Person.deserialize(o)
new_address = register_eth(i, u)

5
apps/data-seeding/create_import_users.py
View File

@ -105,7 +105,7 @@ def genId(addr, typ):
def genDate():
ts = random.randint(ts_then, ts_now)
return datetime.datetime.fromtimestamp(ts).timestamp()
return int(datetime.datetime.fromtimestamp(ts).timestamp())
def genPhone():
@ -193,6 +193,7 @@ def prepareLocalFilePath(datadir, address):
if __name__ == '__main__':
base_dir = os.path.join(user_dir, 'old')
ussd_dir = os.path.join(user_dir, 'ussd')
os.makedirs(base_dir, exist_ok=True)
fa = open(os.path.join(user_dir, 'balances.csv'), 'w')
@ -223,6 +224,8 @@ if __name__ == '__main__':
f = open('{}/{}'.format(d, uid + '.json'), 'w')
json.dump(o.serialize(), f)
f.close()
d = prepareLocalFilePath(ussd_dir, uid)
x = open('{}/{}'.format(d, uid + '_ussd_data.json'), 'w')
json.dump(ussd_data, x)
x.close()

2
apps/data-seeding/eth/import_users.py
View File

@ -23,7 +23,7 @@ from chainlib.eth.connection import EthHTTPConnection
from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from cic_types.processor import generate_metadata_pointer
from eth_accounts_index import AccountRegistry
from eth_accounts_index.registry import AccountRegistry
from eth_contract_registry import Registry
from crypto_dev_signer.keystore.dict import DictKeystore
from crypto_dev_signer.eth.signer.defaultsigner import ReferenceSigner as EIP155Signer