Add 'add tag' method, tags tests
This commit is contained in:
nolash
parent
8db76dc0a8
commit
8d5a787165
GPG Key ID:
21D2E7BB88C2A746
@ -6,3 +6,4 @@ HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=postgresql
|
||||
DRIVER=psycopg2
|
||||
DEBUG=
|
||||
|
||||
@ -6,3 +6,4 @@ HOST=localhost
|
||||
PORT=5432
|
||||
ENGINE=sqlite
|
||||
DRIVER=pysqlite
|
||||
DEBUG=
|
||||
|
||||
@ -2,9 +2,14 @@
|
||||
import logging
|
||||
|
||||
# local imports
|
||||
from .list import list_transactions_mined
|
||||
from .list import list_transactions_account_mined
|
||||
from .list import add_transaction
|
||||
from .list import (
|
||||
list_transactions_mined,
|
||||
list_transactions_account_mined,
|
||||
add_transaction,
|
||||
tag_transaction,
|
||||
add_tag,
|
||||
)
|
||||
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
@ -2,8 +2,9 @@
|
||||
import logging
|
||||
import datetime
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from sqlalchemy import text
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
@ -50,7 +51,8 @@ def list_transactions_account_mined(
|
||||
|
||||
|
||||
def add_transaction(
|
||||
session, tx_hash,
|
||||
session,
|
||||
tx_hash,
|
||||
block_number,
|
||||
tx_index,
|
||||
sender,
|
||||
@ -77,3 +79,49 @@ def add_transaction(
|
||||
date_block,
|
||||
)
|
||||
session.execute(s)
|
||||
|
||||
|
||||
|
||||
def tag_transaction(
|
||||
session,
|
||||
tx_hash,
|
||||
name,
|
||||
domain=None,
|
||||
):
|
||||
|
||||
s = text("SELECT id from tx where tx_hash = :a")
|
||||
r = session.execute(s, {'a': tx_hash}).fetchall()
|
||||
tx_id = r[0].values()[0]
|
||||
|
||||
if tx_id == None:
|
||||
raise ValueError('unknown tx hash {}'.format(tx_hash))
|
||||
|
||||
#s = text("SELECT id from tag where value = :a and domain = :b")
|
||||
if domain == None:
|
||||
s = text("SELECT id from tag where value = :a")
|
||||
else:
|
||||
s = text("SELECT id from tag where value = :a and domain = :b")
|
||||
r = session.execute(s, {'a': name, 'b': domain}).fetchall()
|
||||
tag_id = r[0].values()[0]
|
||||
|
||||
logg.debug('type {} {}'.format(type(tag_id), type(tx_id)))
|
||||
|
||||
if tag_id == None:
|
||||
raise ValueError('unknown tag name {} domain {}'.format(name, domain))
|
||||
|
||||
s = text("INSERT INTO tag_tx_link (tag_id, tx_id) VALUES (:a, :b)")
|
||||
r = session.execute(s, {'a': int(tag_id), 'b': int(tx_id)})
|
||||
|
||||
|
||||
def add_tag(
|
||||
session,
|
||||
name,
|
||||
domain=None,
|
||||
):
|
||||
|
||||
s = None
|
||||
if domain == None:
|
||||
s = text("INSERT INTO tag (value) VALUES (:b)")
|
||||
else:
|
||||
s = text("INSERT INTO tag (domain, value) VALUES (:a, :b)")
|
||||
session.execute(s, {'a': domain, 'b': name})
|
||||
|
||||
@ -0,0 +1,38 @@
|
||||
"""Transaction tags
|
||||
|
||||
Revision ID: aaf2bdce7d6e
|
||||
Revises: 6604de4203e2
|
||||
Create Date: 2021-05-01 09:20:20.775082
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = 'aaf2bdce7d6e'
|
||||
down_revision = '6604de4203e2'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
'tag',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('domain', sa.String(), nullable=True),
|
||||
sa.Column('value', sa.String(), nullable=False),
|
||||
)
|
||||
op.create_index('idx_tag_domain_value', 'tag', ['domain', 'value'], unique=True)
|
||||
|
||||
op.create_table(
|
||||
'tag_tx_link',
|
||||
sa.Column('id', sa.Integer, primary_key=True),
|
||||
sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id'), nullable=False),
|
||||
sa.Column('tx_id', sa.Integer, sa.ForeignKey('tx.id'), nullable=False),
|
||||
)
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('tag_tx_link')
|
||||
op.drop_index('idx_tag_domain_value')
|
||||
op.drop_table('tag')
|
||||
@ -1,2 +1,15 @@
|
||||
class SyncFilter:
|
||||
pass
|
||||
class TagSyncFilter:
|
||||
|
||||
def __init__(self, name, domain=None):
|
||||
self.tag_name = name
|
||||
self.tag_domain = domain
|
||||
|
||||
|
||||
def tag(self):
|
||||
return (self.tag_name, self.tag_domain)
|
||||
|
||||
|
||||
def __str__(self):
|
||||
if tag_domain == None:
|
||||
return self.tag_name
|
||||
return '{}.{}'.format(self.tag_domain, self.tag_name)
|
||||
|
||||
@ -21,9 +21,10 @@ from cic_cache import db as cic_cache_db
|
||||
logg = logging.getLogger().getChild(__name__)
|
||||
|
||||
|
||||
class ERC20TransferFilter(SyncFilter):
|
||||
class ERC20TransferFilter(TagSyncFilter):
|
||||
|
||||
def __init__(self, chain_spec):
|
||||
super(TagSyncFilter, self).__init__('transfer', 'erc20')
|
||||
self.chain_spec = chain_spec
|
||||
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
cic-base~=0.1.2a77
|
||||
cic-base~=0.1.2b6
|
||||
alembic==1.4.2
|
||||
confini~=0.3.6rc3
|
||||
uwsgi==2.0.19.1
|
||||
moolb~=0.1.0
|
||||
cic-eth-registry~=0.5.4a16
|
||||
cic-eth-registry~=0.5.5a1
|
||||
SQLAlchemy==1.3.20
|
||||
semver==2.13.0
|
||||
psycopg2==2.8.6
|
||||
|
||||
@ -84,3 +84,7 @@ def txs(
|
||||
|
||||
session.commit()
|
||||
|
||||
return [
|
||||
tx_hash_first,
|
||||
tx_hash_second,
|
||||
]
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
import os
|
||||
import logging
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import confini
|
||||
|
||||
@ -13,7 +13,7 @@ logg = logging.getLogger(__file__)
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def load_config():
|
||||
config_dir = os.path.join(root_dir, '.config/test')
|
||||
config_dir = os.path.join(root_dir, 'config/test')
|
||||
conf = confini.Config(config_dir, 'CICTEST')
|
||||
conf.process()
|
||||
logg.debug('config {}'.format(conf))
|
||||
|
||||
@ -3,13 +3,16 @@ import os
|
||||
import logging
|
||||
import re
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import sqlparse
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
# local imports
|
||||
from cic_cache.db.models.base import SessionBase
|
||||
from cic_cache.db import dsn_from_config
|
||||
from cic_cache.db import add_tag
|
||||
|
||||
logg = logging.getLogger(__file__)
|
||||
|
||||
@ -26,11 +29,10 @@ def database_engine(
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
dsn = dsn_from_config(load_config)
|
||||
SessionBase.connect(dsn)
|
||||
SessionBase.connect(dsn, debug=load_config.true('DATABASE_DEBUG'))
|
||||
return dsn
|
||||
|
||||
|
||||
# TODO: use alembic instead to migrate db, here we have to keep separate schema than migration script in script/migrate.py
|
||||
@pytest.fixture(scope='function')
|
||||
def init_database(
|
||||
load_config,
|
||||
@ -38,52 +40,23 @@ def init_database(
|
||||
):
|
||||
|
||||
rootdir = os.path.dirname(os.path.dirname(__file__))
|
||||
schemadir = os.path.join(rootdir, 'db', load_config.get('DATABASE_DRIVER'))
|
||||
|
||||
if load_config.get('DATABASE_ENGINE') == 'sqlite':
|
||||
rconn = SessionBase.engine.raw_connection()
|
||||
f = open(os.path.join(schemadir, 'db.sql'))
|
||||
s = f.read()
|
||||
f.close()
|
||||
rconn.executescript(s)
|
||||
|
||||
else:
|
||||
rconn = SessionBase.engine.raw_connection()
|
||||
rcursor = rconn.cursor()
|
||||
|
||||
#rcursor.execute('DROP FUNCTION IF EXISTS public.transaction_list')
|
||||
#rcursor.execute('DROP FUNCTION IF EXISTS public.balances')
|
||||
|
||||
f = open(os.path.join(schemadir, 'db.sql'))
|
||||
s = f.read()
|
||||
f.close()
|
||||
r = re.compile(r'^[A-Z]', re.MULTILINE)
|
||||
for l in sqlparse.parse(s):
|
||||
strl = str(l)
|
||||
# we need to check for empty query lines, as sqlparse doesn't do that on its own (and psycopg complains when it gets them)
|
||||
if not re.search(r, strl):
|
||||
logg.warning('skipping parsed query line {}'.format(strl))
|
||||
continue
|
||||
rcursor.execute(strl)
|
||||
rconn.commit()
|
||||
|
||||
rcursor.execute('SET search_path TO public')
|
||||
|
||||
# this doesn't work when run separately, no idea why
|
||||
# functions have been manually added to original schema from cic-eth
|
||||
# f = open(os.path.join(schemadir, 'proc_transaction_list.sql'))
|
||||
# s = f.read()
|
||||
# f.close()
|
||||
# rcursor.execute(s)
|
||||
#
|
||||
# f = open(os.path.join(schemadir, 'proc_balances.sql'))
|
||||
# s = f.read()
|
||||
# f.close()
|
||||
# rcursor.execute(s)
|
||||
|
||||
rcursor.close()
|
||||
dbdir = os.path.join(rootdir, 'cic_cache', 'db')
|
||||
migrationsdir = os.path.join(dbdir, 'migrations', load_config.get('DATABASE_ENGINE'))
|
||||
if not os.path.isdir(migrationsdir):
|
||||
migrationsdir = os.path.join(dbdir, 'migrations', 'default')
|
||||
logg.info('using migrations directory {}'.format(migrationsdir))
|
||||
|
||||
session = SessionBase.create_session()
|
||||
|
||||
ac = AlembicConfig(os.path.join(migrationsdir, 'alembic.ini'))
|
||||
ac.set_main_option('sqlalchemy.url', database_engine)
|
||||
ac.set_main_option('script_location', migrationsdir)
|
||||
|
||||
alembic.command.downgrade(ac, 'base')
|
||||
alembic.command.upgrade(ac, 'head')
|
||||
|
||||
session.commit()
|
||||
|
||||
yield session
|
||||
session.commit()
|
||||
session.close()
|
||||
@ -116,3 +89,14 @@ def list_defaults(
|
||||
return {
|
||||
'block': 420000,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def tags(
|
||||
init_database,
|
||||
):
|
||||
|
||||
add_tag(init_database, 'foo')
|
||||
add_tag(init_database, 'baz', domain='bar')
|
||||
add_tag(init_database, 'xyzzy', domain='bar')
|
||||
init_database.commit()
|
||||
|
||||
@ -4,7 +4,7 @@ import datetime
|
||||
import logging
|
||||
import json
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
|
||||
37
apps/cic-cache/tests/test_tag.py
Normal file
37
apps/cic-cache/tests/test_tag.py
Normal file
@ -0,0 +1,37 @@
|
||||
import os
|
||||
import datetime
|
||||
import logging
|
||||
import json
|
||||
|
||||
# external imports
|
||||
import pytest
|
||||
|
||||
# local imports
|
||||
from cic_cache.db import tag_transaction
|
||||
|
||||
logg = logging.getLogger()
|
||||
|
||||
|
||||
def test_cache(
|
||||
init_database,
|
||||
list_defaults,
|
||||
list_actors,
|
||||
list_tokens,
|
||||
txs,
|
||||
tags,
|
||||
):
|
||||
|
||||
tag_transaction(init_database, txs[0], 'foo')
|
||||
tag_transaction(init_database, txs[0], 'baz', domain='bar')
|
||||
tag_transaction(init_database, txs[1], 'xyzzy', domain='bar')
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.value = 'foo'").fetchall()
|
||||
assert r[0][0] == txs[0]
|
||||
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'baz'").fetchall()
|
||||
assert r[0][0] == txs[0]
|
||||
|
||||
|
||||
r = init_database.execute("SELECT x.tx_hash FROM tag a INNER JOIN tag_tx_link l ON l.tag_id = a.id INNER JOIN tx x ON x.id = l.tx_id WHERE a.domain = 'bar' AND a.value = 'xyzzy'").fetchall()
|
||||
assert r[0][0] == txs[1]
|
||||
@ -2,7 +2,7 @@
|
||||
import os
|
||||
import logging
|
||||
|
||||
# third-party imports
|
||||
# external imports
|
||||
import pytest
|
||||
import alembic
|
||||
from alembic.config import Config as AlembicConfig
|
||||
|
||||
Loading…
Reference in New Issue
Block a user