kamikazechaser/cic-internal-integration
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into cic-eth-unittest

Browse Source
This commit is contained in:
nolash 2021-05-21 18:58:11 +02:00
commit 91b964eefc
33 changed files with 4819 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/cic-cache/config/test/syncer.ini Normal file
View File

@ -0,0 +1,2 @@
[syncer]
loop_interval = 1

2
apps/cic-eth/cic_eth/version.py
View File

@ -10,7 +10,7 @@ version = (
0,
11,
0,
'beta.13',
'beta.14',
)
version_object = semver.VersionInfo(

51
apps/cic-meta/bin/get.js Executable file
View File

@ -0,0 +1,51 @@
#!/usr/bin/env node
const colors = require('colors');
const {Meta} = require("../dist");
let { argv } = require('yargs')
.usage('Usage: $0 -m http://localhost:63380 -n publickeys')
.example(
'$0 -m http://localhost:63380 -n publickeys',
'Fetches the public keys blob from the meta server'
)
.option('m', {
alias: 'metaurl',
describe: 'The URL for the meta service',
demandOption: 'The meta url is required',
type: 'string',
nargs: 1,
})
.option('n', {
alias: 'name',
describe: 'The name of the resource to be fetched from the meta service',
demandOption: 'The name of the resource is required',
type: 'string',
nargs: 1,
})
.option('t', {
alias: 'type',
describe: 'The type of resource to be fetched from the meta service\n' +
'Options: `user`, `phone` and `custom`\n' +
'Defaults to `custom`',
type: 'string',
nargs: 1,
})
.epilog('Grassroots Economics (c) 2021')
.wrap(null);
const metaUrl = argv.m;
const resourceName = argv.n;
let type = argv.t;
if (type === undefined) {
type = 'custom'
}
(async () => {
const identifier = await Meta.getIdentifier(resourceName, type);
console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
const metaResponse = await Meta.get(identifier, metaUrl);
if (typeof metaResponse !== "object") {
console.error(colors.red('Metadata get failed!'));
}
console.log(colors.green(metaResponse));
})();

81
apps/cic-meta/bin/set.js Executable file
View File

@ -0,0 +1,81 @@
#!/usr/bin/env node
const fs = require("fs");
const colors = require('colors');
const {Meta} = require("../dist");
let { argv } = require('yargs')
.usage('Usage: $0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc')
.example(
'$0 -m http://localhost:63380 -k ./privatekeys.asc -n publickeys -r ./publickeys.asc',
'Updates the public keys blob to the meta server'
)
.option('m', {
alias: 'metaurl',
describe: 'The URL for the meta service',
demandOption: 'The meta url is required',
type: 'string',
nargs: 1,
})
.option('k', {
alias: 'privatekey',
describe: 'The PGP private key blob file used to sign the changes to the meta service',
demandOption: 'The private key file is required',
type: 'string',
nargs: 1,
})
.option('n', {
alias: 'name',
describe: 'The name of the resource to be set or updated to the meta service',
demandOption: 'The name of the resource is required',
type: 'string',
nargs: 1,
})
.option('r', {
alias: 'resource',
describe: 'The resource file to be set or updated to the meta service',
demandOption: 'The resource file is required',
type: 'string',
nargs: 1,
})
.option('t', {
alias: 'type',
describe: 'The type of resource to be set or updated to the meta service\n' +
'Options: `user`, `phone` and `custom`\n' +
'Defaults to `custom`',
type: 'string',
nargs: 1,
})
.epilog('Grassroots Economics (c) 2021')
.wrap(null);
const metaUrl = argv.m;
const privateKeyFile = argv.k;
const resourceName = argv.n;
const resourceFile = argv.r;
let type = argv.t;
if (type === undefined) {
type = 'custom'
}
const privateKey = readFile(privateKeyFile);
const resource = readFile(resourceFile);
(async () => {
if (privateKey && resource) {
const identifier = await Meta.getIdentifier(resourceName, type);
console.log(colors.cyan(`Meta server storage identifier: ${identifier}`));
const meta = new Meta(metaUrl, privateKey);
meta.onload = async (status) => {
const response = await meta.set(identifier, resource)
console.log(colors.green(response));
}
}
})();
function readFile(filename) {
if(!fs.existsSync(filename)) {
console.log(colors.red(`File ${filename} not found`));
return;
}
return fs.readFileSync(filename, {encoding: 'utf8', flag: 'r'});
}

4232
apps/cic-meta/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

15
apps/cic-meta/package.json
View File

@ -1,9 +1,14 @@
{
"name": "cic-client-meta",
"version": "0.0.7-alpha.8",
"name": "@cicnet/cic-client-meta",
"version": "0.0.11",
"description": "Signed CRDT metadata graphs for the CIC network",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"bin": {
"meta-set": "bin/set.js",
"meta-get": "bin/get.js"
},
"preferGlobal": true,
"scripts": {
"test": "mocha -r node_modules/node-localstorage/register -r ts-node/register tests/*.ts",
"build": "node_modules/typescript/bin/tsc -d --outDir dist src/index.ts",
@ -11,12 +16,14 @@
"pack": "node_modules/typescript/bin/tsc -d --outDir dist && webpack",
"clean": "rm -rf dist",
"prepare": "npm run build && npm run build-server",
"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts"
"start": "./node_modules/ts-node/dist/bin.js ./scripts/server/server.ts",
"publish": "npm publish --access public"
},
"dependencies": {
"@cicnet/crdt-meta": "^0.0.10",
"@ethereumjs/tx": "^3.0.0-beta.1",
"automerge": "^0.14.1",
"crdt-meta": "0.0.8",
"colors": "^1.4.0",
"ethereumjs-wallet": "^1.0.1",
"ini": "^1.3.8",
"openpgp": "^4.10.8",

2
apps/cic-meta/scripts/dumpconfig.js
View File

@ -1,4 +1,4 @@
import { Config } from 'crdt-meta';
import { Config } from '@cicnet/crdt-meta';
const fs = require('fs');
if (process.argv[2] === undefined) {

2
apps/cic-meta/scripts/server/handlers.ts
View File

@ -1,7 +1,7 @@
import * as Automerge from 'automerge';
import * as pgp from 'openpgp';
import { Envelope, Syncable } from 'crdt-meta';
import { Envelope, Syncable } from '@cicnet/crdt-meta';
function handleNoMergeGet(db, digest, keystore) {

3
apps/cic-meta/scripts/server/server.ts
View File

@ -3,7 +3,8 @@ import * as fs from 'fs';
import * as path from 'path';
import * as handlers from './handlers';
import { PGPKeyStore, PGPSigner, Config, SqliteAdapter, PostgresAdapter } from 'crdt-meta';
import { PGPKeyStore, PGPSigner, Config } from '@cicnet/crdt-meta';
import { SqliteAdapter, PostgresAdapter } from '../../src/db';
import { standardArgs } from './args';

27
apps/cic-meta/src/custom.ts Normal file
View File

@ -0,0 +1,27 @@
import {Addressable, mergeKey, Syncable} from "@cicnet/crdt-meta";
class Custom extends Syncable implements Addressable {
name: string
value: Object
constructor(name:string, v:Object={}) {
super('', v);
Custom.toKey(name).then((cid) => {
this.id = cid;
this.value = v;
});
}
public static async toKey(item:string, identifier: string = ':cic.custom') {
return await mergeKey(Buffer.from(item), Buffer.from(identifier));
}
public key(): string {
return this.id;
}
}
export {
Custom,
}

90
apps/cic-meta/src/db.ts Normal file
View File

@ -0,0 +1,90 @@
import * as pg from 'pg';
import * as sqlite from 'sqlite3';
type DbConfig = {
name: string
host: string
port: number
user: string
password: string
}
interface DbAdapter {
query: (s:string, callback:(e:any, rs:any) => void) => void
close: () => void
}
const re_creatematch = /^(CREATE)/i
const re_getmatch = /^(SELECT)/i;
const re_setmatch = /^(INSERT|UPDATE)/i;
class SqliteAdapter implements DbAdapter {
db: any
constructor(dbConfig:DbConfig, callback?:(any) => void) {
this.db = new sqlite.Database(dbConfig.name); //, callback);
}
public query(s:string, callback:(e:any, rs?:any) => void): void {
const local_callback = (e, rs) => {
let r = undefined;
if (rs !== undefined) {
r = {
rowCount: rs.length,
rows: rs,
}
}
callback(e, r);
};
if (s.match(re_getmatch)) {
this.db.all(s, local_callback);
} else if (s.match(re_setmatch)) {
this.db.run(s, local_callback);
} else if (s.match(re_creatematch)) {
this.db.run(s, callback);
} else {
throw 'unhandled query';
}
}
public close() {
this.db.close();
}
}
class PostgresAdapter implements DbAdapter {
db: any
constructor(dbConfig:DbConfig) {
let o = dbConfig;
o['database'] = o.name;
this.db = new pg.Pool(o);
return this.db;
}
public query(s:string, callback:(e:any, rs:any) => void): void {
this.db.query(s, (e, rs) => {
let r = {
length: rs.rowCount,
}
rs.length = rs.rowCount;
if (e === undefined) {
e = null;
}
console.debug(e, rs);
callback(e, rs);
});
}
public close() {
this.db.end();
}
}
export {
DbConfig,
SqliteAdapter,
PostgresAdapter,
}

2
apps/cic-meta/src/index.ts
View File

@ -1,2 +1,4 @@
export { User } from './user';
export { Phone } from './phone';
export { Custom } from './custom';
export { Meta } from './meta';

126
apps/cic-meta/src/meta.ts Normal file
View File

@ -0,0 +1,126 @@
import {ArgPair, Envelope, Syncable, MutablePgpKeyStore, PGPSigner} from "@cicnet/crdt-meta";
import {User} from "./user";
import {Phone} from "./phone";
import {Custom} from "./custom";
const fetch = require("node-fetch");
const headers = {
'Content-Type': 'application/json;charset=utf-8',
'x-cic-automerge': 'client'
};
const options = {
headers: headers,
};
class Meta {
keystore: MutablePgpKeyStore = new MutablePgpKeyStore();
signer: PGPSigner = new PGPSigner(this.keystore);
metaUrl: string;
private privateKey: string;
onload: (status: boolean) => void;
constructor(metaUrl: string, privateKey: any) {
this.metaUrl = metaUrl;
this.privateKey = privateKey;
this.keystore.loadKeyring().then(() => {
this.keystore.importPrivateKey(privateKey).then(() => this.onload(true));
});
}
async set(identifier: string, data: Object): Promise<any> {
let syncable: Syncable;
const response = await Meta.get(identifier, this.metaUrl);
if (response === `Request to ${this.metaUrl}/${identifier} failed. Connection error.`) {
return response;
} else if (typeof response !== "object" || typeof data !== "object") {
syncable = new Syncable(identifier, data);
const res = await this.updateMeta(syncable, identifier);
return `${res.status}: ${res.statusText}`;
} else {
syncable = await Meta.get(identifier, this.metaUrl);
let update: Array<ArgPair> = [];
for (const prop in data) {
update.push(new ArgPair(prop, data[prop]));
}
syncable.update(update, 'client-branch');
const res = await this.updateMeta(syncable, identifier);
return `${res.status}: ${res.statusText}`;
}
}
async updateMeta(syncable: Syncable, identifier: string): Promise<any> {
const envelope: Envelope = await this.wrap(syncable);
const reqBody: string = envelope.toJSON();
const putOptions = {
method: 'PUT',
headers: headers,
body: reqBody
};
return await fetch(`${this.metaUrl}/${identifier}`, putOptions).then(async response => {
if (response.ok) {
return Promise.resolve({
status: response.status,
statusText: response.statusText + ', Metadata updated successfully!'
});
} else {
return Promise.reject({
status: response.status,
statusText: response.statusText
});
}
});
}
static async get(identifier: string, metaUrl: string): Promise<any> {
const response = await fetch(`${metaUrl}/${identifier}`, options).then(response => {
if (response.ok) {
return (response.json());
} else {
return Promise.reject({
status: response.status,
statusText: response.statusText
});
}
}).catch(error => {
if (error.code === 'ECONNREFUSED') {
return `Request to ${metaUrl}/${identifier} failed. Connection error.`
}
return `${error.status}: ${error.statusText}`;
});
if (typeof response !== "object") {
return response;
}
return Envelope.fromJSON(JSON.stringify(response)).unwrap();
}
static async getIdentifier(name: string, type: string = 'custom'): Promise<string> {
let identifier: string;
type = type.toLowerCase();
if (type === 'user') {
identifier = await User.toKey(name);
} else if (type === 'phone') {
identifier = await Phone.toKey(name);
} else {
identifier = await Custom.toKey(name);
}
return identifier;
}
private wrap(syncable: Syncable): Promise<Envelope> {
return new Promise<Envelope>(async (resolve, reject) => {
syncable.setSigner(this.signer);
syncable.onwrap = async (env) => {
if (env === undefined) {
reject();
return;
}
resolve(env);
};
syncable.sign();
});
}
}
export {
Meta,
}

2
apps/cic-meta/src/phone.ts
View File

@ -1,4 +1,4 @@
import { Syncable, Addressable, mergeKey } from 'crdt-meta';
import { Syncable, Addressable, mergeKey } from '@cicnet/crdt-meta';
class Phone extends Syncable implements Addressable {

2
apps/cic-meta/src/user.ts
View File

@ -1,4 +1,4 @@
import { Syncable, Addressable, toAddressKey } from 'crdt-meta';
import { Syncable, Addressable, toAddressKey } from '@cicnet/crdt-meta';
const keySalt = new TextEncoder().encode(':cic.person');
class User extends Syncable implements Addressable {

3
apps/cic-meta/tests/server.ts
View File

@ -4,7 +4,8 @@ import pgp = require('openpgp');
import sqlite = require('sqlite3');
import * as handlers from '../scripts/server/handlers';
import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer, SqliteAdapter } from 'crdt-meta';
import { Envelope, Syncable, ArgPair, PGPKeyStore, PGPSigner, KeyStore, Signer } from '@cicnet/crdt-meta';
import { SqliteAdapter } from '../src/db';
function createKeystore() {
const pksa = fs.readFileSync(__dirname + '/privatekeys.asc', 'utf-8');

3
apps/cic-meta/tsconfig.json
View File

@ -19,6 +19,7 @@
"include": [
"src/**/*",
"scripts/server/*",
"index.ts"
"index.ts",
"bin"
]
}

76
apps/contract-migration/scripts/cic_eth/traffic/cmd/cache.py Normal file
View File

@ -0,0 +1,76 @@
# external imports
from chainlib.jsonrpc import JSONRPCException
from eth_erc20 import ERC20
from eth_accounts_index import AccountsIndex
from eth_token_index import TokenUniqueSymbolIndex
class ERC20Token:
def __init__(self, chain_spec, address, conn):
self.__address = address
c = ERC20(chain_spec)
o = c.symbol(address)
r = conn.do(o)
self.__symbol = c.parse_symbol(r)
o = c.decimals(address)
r = conn.do(o)
self.__decimals = c.parse_decimals(r)
def symbol(self):
return self.__symbol
def decimals(self):
return self.__decimals
class IndexCache:
def __init__(self, chain_spec, address):
self.address = address
self.chain_spec = chain_spec
def parse(self, r):
return r
def get(self, conn):
entries = []
i = 0
while True:
o = self.o.entry(self.address, i)
try:
r = conn.do(o)
entries.append(self.parse(r, conn))
except JSONRPCException:
return entries
i += 1
class AccountRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(AccountRegistryCache, self).__init__(chain_spec, address)
self.o = AccountsIndex(chain_spec)
self.get_accounts = self.get
def parse(self, r, conn):
return self.o.parse_account(r)
class TokenRegistryCache(IndexCache):
def __init__(self, chain_spec, address):
super(TokenRegistryCache, self).__init__(chain_spec, address)
self.o = TokenUniqueSymbolIndex(chain_spec)
self.get_tokens = self.get
def parse(self, r, conn):
token_address = self.o.parse_entry(r)
return ERC20Token(self.chain_spec, token_address, conn)

23
apps/data-seeding/cic_eth/traffic/cmd/traffic.py
View File

@ -163,9 +163,9 @@ class TrafficProvisioner:
"""Aux parameter template to be passed to the traffic generator module"""
def __init__(self):
self.tokens = self.oracles['token'].get_tokens()
self.accounts = self.oracles['account'].get_accounts()
def __init__(self, conn):
self.tokens = self.oracles['token'].get_tokens(conn)
self.accounts = self.oracles['account'].get_accounts(conn)
self.aux = copy.copy(self.default_aux)
self.__balances = {}
for a in self.accounts:
@ -277,13 +277,14 @@ class TrafficSyncHandler:
:type traffic_router: TrafficRouter
:raises Exception: Any Exception redis may raise on connection attempt.
"""
def __init__(self, config, traffic_router):
def __init__(self, config, traffic_router, conn):
self.traffic_router = traffic_router
self.redis_channel = str(uuid.uuid4())
self.pubsub = self.__connect_redis(self.redis_channel, config)
self.traffic_items = {}
self.config = config
self.init = False
self.conn = conn
# connects to redis
@ -307,7 +308,7 @@ class TrafficSyncHandler:
:param tx_index: Syncer block transaction index at time of call.
:type tx_index: number
"""
traffic_provisioner = TrafficProvisioner()
traffic_provisioner = TrafficProvisioner(self.conn)
traffic_provisioner.add_aux('redis_channel', self.redis_channel)
refresh_accounts = None
@ -343,7 +344,7 @@ class TrafficSyncHandler:
sender = traffic_provisioner.accounts[sender_index]
#balance_full = balances[sender][token_pair[0].symbol()]
if len(sender_indices) == 1:
sender_indices[m] = sender_sender_indices[len(senders)-1]
sender_indices[sender_index] = sender_indices[len(sender_indices)-1]
sender_indices = sender_indices[:len(sender_indices)-1]
balance_full = traffic_provisioner.balance(sender, token_pair[0])
@ -351,7 +352,14 @@ class TrafficSyncHandler:
recipient_index = random.randint(0, len(traffic_provisioner.accounts)-1)
recipient = traffic_provisioner.accounts[recipient_index]
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}')
logg.debug('trigger item {} tokens {} sender {} recipient {} balance {}'.format(
traffic_item,
token_pair,
sender,
recipient,
balance_full,
)
)
(e, t, balance_result,) = traffic_item.method(
token_pair,
sender,
@ -359,7 +367,6 @@ class TrafficSyncHandler:
balance_full,
traffic_provisioner.aux,
block_number,
tx_index,
)
traffic_provisioner.update_balance(sender, token_pair[0], balance_result)
sender_indices.append(recipient_index)

2
apps/data-seeding/cic_eth/traffic/common/registry.py
View File

@ -3,7 +3,7 @@ import logging
import copy
# external imports
from cic_registry import CICRegistry
from cic_registry.registry import Registry
from eth_token_index import TokenUniqueSymbolIndex
from eth_accounts_index import AccountRegistry
from chainlib.chain import ChainSpec

2
apps/data-seeding/cic_eth/traffic/common/signer.py
View File

@ -3,7 +3,7 @@ import logging
# external imports
from crypto_dev_signer.eth.signer import ReferenceSigner as EIP155Signer
from crypto_dev_signer.keystore import DictKeystore
from crypto_dev_signer.keystore.dict import DictKeystore
logg = logging.getLogger(__name__)

2
apps/data-seeding/cic_eth/traffic/local/account.py
View File

@ -11,7 +11,7 @@ queue = 'cic-eth'
name = 'account'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers creation and registration of new account through the custodial cic-eth component.
It expects the following aux parameters to exist:

6
apps/data-seeding/cic_eth/traffic/local/noop.py
View File

@ -5,7 +5,7 @@ logging.basicConfig(level=logging.WARNING)
logg = logging.getLogger()
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Defines the function signature for a traffic generator. The method itself only logs the input parameters.
If the error position in the return tuple is not None, the calling code should consider the generation as failed, and not count it towards the limit of simultaneous traffic items that can be simultaneously in flight.
@ -26,12 +26,10 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
:type aux: dict
:param block_number: Syncer block number position at time of method call
:type block_number: number
:param tx_index: Syncer block transaction index position at time of method call
:type tx_index: number
:raises KeyError: Missing required aux element
:returns: Exception|None, task_id|None and adjusted_sender_balance respectively
:rtype: tuple
"""
logg.debug('running {} {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number, tx_index))
logg.debug('running {} {} {} {} {} {} {}'.format(__name__, token_pair, sender, recipient, sender_balance, aux, block_number))
return (None, None, sender_balance, )

4
apps/data-seeding/cic_eth/traffic/local/transfer.py
View File

@ -12,7 +12,7 @@ queue = 'cic-eth'
name = 'erc20_transfer'
def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_index):
def do(token_pair, sender, recipient, sender_balance, aux, block_number):
"""Triggers an ERC20 token transfer through the custodial cic-eth component, with a randomly chosen amount in integer resolution.
It expects the following aux parameters to exist:
@ -33,7 +33,7 @@ def do(token_pair, sender, recipient, sender_balance, aux, block_number, tx_inde
balance_units = int(sender_balance_value / decimals)
if balance_units <= 0:
return (AttributeError('sender {} has zero balance'), None, 0,)
return (AttributeError('sender {} has zero balance ({} / {})'.format(sender, sender_balance_value, decimals)), None, 0,)
spend_units = random.randint(1, balance_units)
spend_value = spend_units * decimals

70
apps/data-seeding/cic_eth/traffic/traffic.py
View File

@ -8,16 +8,25 @@ import json
# external imports
import redis
import celery
from chainsyncer.backend import MemBackend
from cic_eth_registry.registry import CICRegistry
from chainsyncer.backend.memory import MemBackend
from chainsyncer.driver import HeadSyncer
from chainlib.eth.connection import HTTPConnection
from chainlib.eth.gas import DefaultGasOracle
from chainlib.eth.nonce import DefaultNonceOracle
from chainlib.eth.connection import EthHTTPConnection
from chainlib.chain import ChainSpec
from chainlib.eth.gas import RPCGasOracle
from chainlib.eth.nonce import RPCNonceOracle
from chainlib.eth.block import block_latest
from hexathon import strip_0x
from cic_base import (
argparse,
config,
log,
rpc,
signer as signer_funcs,
)
# local imports
import common
#import common
from cmd.traffic import (
TrafficItem,
TrafficRouter,
@ -25,15 +34,19 @@ from cmd.traffic import (
TrafficSyncHandler,
)
from cmd.traffic import add_args as add_traffic_args
from cmd.cache import (
AccountRegistryCache,
TokenRegistryCache,
)
# common basics
script_dir = os.path.realpath(os.path.dirname(__file__))
logg = common.log.create()
argparser = common.argparse.create(script_dir, common.argparse.full_template)
argparser = common.argparse.add(argparser, add_traffic_args, 'traffic')
args = common.argparse.parse(argparser, logg)
config = common.config.create(args.c, args, args.env_prefix)
logg = log.create()
argparser = argparse.create(script_dir, argparse.full_template)
argparser = argparse.add(argparser, add_traffic_args, 'traffic')
args = argparse.parse(argparser, logg)
config = config.create(args.c, args, args.env_prefix)
# map custom args to local config entries
batchsize = args.batch_size
@ -49,29 +62,32 @@ config.add(args.y, '_KEYSTORE_FILE', True)
config.add(args.q, '_CELERY_QUEUE', True)
common.config.log(config)
logg.debug(config)
chain_spec = ChainSpec.from_chain_str(config.get('CIC_CHAIN_SPEC'))
def main():
# create signer (not currently in use, but needs to be accessible for custom traffic item generators)
(signer_address, signer) = common.signer.from_keystore(config.get('_KEYSTORE_FILE'))
(signer_address, signer) = signer_funcs.from_keystore(config.get('_KEYSTORE_FILE'))
# connect to celery
celery.Celery(broker=config.get('CELERY_BROKER_URL'), backend=config.get('CELERY_RESULT_URL'))
# set up registry
w3 = common.rpc.create(config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
registry = common.registry.init_legacy(config, w3)
rpc.setup(config.get('CIC_CHAIN_SPEC'), config.get('ETH_PROVIDER')) # replace with HTTPConnection when registry has been so refactored
conn = EthHTTPConnection(config.get('ETH_PROVIDER'))
#registry = registry.init_legacy(config, w3)
CICRegistry.address = config.get('CIC_REGISTRY_ADDRESS')
registry = CICRegistry(chain_spec, conn)
# Connect to blockchain with chainlib
conn = HTTPConnection(config.get('ETH_PROVIDER'))
gas_oracle = DefaultGasOracle(conn)
nonce_oracle = DefaultNonceOracle(signer_address, conn)
gas_oracle = RPCGasOracle(conn)
nonce_oracle = RPCNonceOracle(signer_address, conn)
# Set up magic traffic handler
traffic_router = TrafficRouter()
traffic_router.apply_import_dict(config.all(), config)
handler = TrafficSyncHandler(config, traffic_router)
handler = TrafficSyncHandler(config, traffic_router, conn)
# Set up syncer
syncer_backend = MemBackend(config.get('CIC_CHAIN_SPEC'), 0)
@ -80,9 +96,21 @@ def main():
block_offset = int(strip_0x(r), 16) + 1
syncer_backend.set(block_offset, 0)
# get relevant registry entries
token_registry = registry.lookup('TokenRegistry')
logg.info('using token registry {}'.format(token_registry))
token_cache = TokenRegistryCache(chain_spec, token_registry)
account_registry = registry.lookup('AccountRegistry')
logg.info('using account registry {}'.format(account_registry))
account_cache = AccountRegistryCache(chain_spec, account_registry)
# Set up provisioner for common task input data
TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
#TrafficProvisioner.oracles['token']= common.registry.TokenOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
#TrafficProvisioner.oracles['account'] = common.registry.AccountsOracle(w3, config.get('CIC_CHAIN_SPEC'), registry)
TrafficProvisioner.oracles['token'] = token_cache
TrafficProvisioner.oracles['account'] = account_cache
TrafficProvisioner.default_aux = {
'chain_spec': config.get('CIC_CHAIN_SPEC'),
'registry': registry,
@ -92,7 +120,7 @@ def main():
'api_queue': config.get('_CELERY_QUEUE'),
}
syncer = HeadSyncer(syncer_backend, loop_callback=handler.refresh)
syncer = HeadSyncer(syncer_backend, block_callback=handler.refresh)
syncer.add_filter(handler)
syncer.loop(1, conn)

4
apps/data-seeding/cic_meta/import_meta.js
View File

@ -2,8 +2,8 @@ const fs = require('fs');
const path = require('path');
const http = require('http');
const cic = require('cic-client-meta');
const crdt = require('crdt-meta');
const cic = require('@cicnet/cic-client-meta');
const crdt = require('@cicnet/crdt-meta');
//const conf = JSON.parse(fs.readFileSync('./cic.conf'));

36
apps/data-seeding/cic_meta/import_meta_custom.js
View File

@ -2,12 +2,12 @@ const fs = require('fs');
const path = require('path');
const http = require('http');
const cic = require('cic-client-meta');
const vcfp = require('vcard-parser');
const cic = require('@cicnet/cic-client-meta');
const crdt = require('@cicnet/crdt-meta');
//const conf = JSON.parse(fs.readFileSync('./cic.conf'));
const config = new cic.Config('./config');
const config = new crdt.Config('./config');
config.process();
console.log(config);
@ -42,24 +42,18 @@ function sendit(uid, envelope) {
}
function doOne(keystore, filePath, identifier) {
const signer = new cic.PGPSigner(keystore);
const signer = new crdt.PGPSigner(keystore);
const o = JSON.parse(fs.readFileSync(filePath).toString());
//const b = Buffer.from(j['vcard'], 'base64');
//const s = b.toString();
//const o = vcfp.parse(s);
//const phone = o.tel[0].value;
//cic.Phone.toKey(phone).then((uid) => {
//const o = fs.readFileSync(filePath, 'utf-8');
const s = new cic.Syncable(identifier, o);
s.setSigner(signer);
s.onwrap = (env) => {
sendit(identifier, env);
};
s.sign();
//});
cic.Custom.toKey(identifier).then((uid) => {
const s = new crdt.Syncable(uid, o);
s.setSigner(signer);
s.onwrap = (env) => {
sendit(identifier, env);
};
s.sign();
});
}
const privateKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_PRIVATE_KEY_FILE'));
@ -67,7 +61,7 @@ const publicKeyPath = path.join(config.get('PGP_EXPORTS_DIR'), config.get('PGP_P
pk = fs.readFileSync(privateKeyPath);
pubk = fs.readFileSync(publicKeyPath);
new cic.PGPKeyStore(
new crdt.PGPKeyStore(
config.get('PGP_PASSPHRASE'),
pk,
pubk,
@ -94,7 +88,7 @@ function importMetaCustom(keystore) {
err, files = fs.readdirSync(workDir);
} catch {
console.error('source directory not yet ready', workDir);
setTimeout(importMetaPhone, batchDelay, keystore);
setTimeout(importMetaCustom, batchDelay, keystore);
return;
}
let limit = batchSize;
@ -128,7 +122,7 @@ function importMetaCustom(keystore) {
if (batchCount == batchSize) {
console.debug('reached batch size, breathing');
batchCount=0;
setTimeout(importMeta, batchDelay, keystore);
setTimeout(importMetaCustom, batchDelay, keystore);
return;
}
}

4
apps/data-seeding/cic_meta/import_meta_phone.js
View File

@ -2,9 +2,9 @@ const fs = require('fs');
const path = require('path');
const http = require('http');
const cic = require('cic-client-meta');
const cic = require('@cicnet/cic-client-meta');
const vcfp = require('vcard-parser');
const crdt = require('crdt-meta');
const crdt = require('@cicnet/crdt-meta');
//const conf = JSON.parse(fs.readFileSync('./cic.conf'));

2
apps/data-seeding/config/pgp.ini
View File

@ -1,5 +1,5 @@
[pgp]
exports_dir = ../testdata/pgp
exports_dir = ../contract-migration/testdata/pgp
private_key_file = privatekeys_meta.asc
public_key_file = publickeys_meta.asc
passphrase = merman

2
apps/data-seeding/config/traffic.ini
View File

@ -1,4 +1,4 @@
[traffic]
#local.noop_traffic = 2
local.account = 2
#local.account = 2
local.transfer = 2

147
apps/data-seeding/package-lock.json generated
View File

@ -5,10 +5,49 @@
"packages": {
"": {
"dependencies": {
"cic-client-meta": "0.0.7-alpha.6",
"@cicnet/cic-client-meta": "^0.0.11",
"@cicnet/crdt-meta": "^0.0.10",
"vcard-parser": "^1.0.0"
}
},
"node_modules/@cicnet/cic-client-meta": {
"version": "0.0.11",
"resolved": "https://registry.npmjs.org/@cicnet/cic-client-meta/-/cic-client-meta-0.0.11.tgz",
"integrity": "sha512-RL9CPXkWQBQzaqMoldnENC/xqinIMyWNSsejs9+qkFOWbAvC4inLdpjjCaooyvLpIZGHF9cLjxHiAdBMR9L8sQ==",
"dependencies": {
"@cicnet/crdt-meta": "^0.0.10",
"@ethereumjs/tx": "^3.0.0-beta.1",
"automerge": "^0.14.1",
"colors": "^1.4.0",
"ethereumjs-wallet": "^1.0.1",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"pg": "^8.4.2",
"sqlite3": "^5.0.0",
"yargs": "^16.1.0"
},
"bin": {
"meta-get": "bin/get.js",
"meta-set": "bin/set.js"
},
"engines": {
"node": ">=14.16.1"
}
},
"node_modules/@cicnet/crdt-meta": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.10.tgz",
"integrity": "sha512-f+H6BQA2tE718KuNYiNzrDJN4wY00zeuhXM6aPKJUX6nryzX9g2r0yf8iDhkz+Fts1R6M7Riz73MfFEa8fgvsw==",
"dependencies": {
"automerge": "^0.14.2",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"readline-sync": "^1.4.10"
},
"engines": {
"node": ">=14.16.1"
}
},
"node_modules/@ethereumjs/common": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz",
@ -317,24 +356,6 @@
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"node_modules/cic-client-meta": {
"version": "0.0.7-alpha.6",
"resolved": "https://registry.npmjs.org/cic-client-meta/-/cic-client-meta-0.0.7-alpha.6.tgz",
"integrity": "sha512-oIN1aHkPHfsxJKDV6k4f1kX2tcppw3Q+D1b4BoPh0hYjNKNb7gImBMWnGsy8uiD9W6SNYE4sIXyrtct8mvrhsw==",
"dependencies": {
"@ethereumjs/tx": "^3.0.0-beta.1",
"automerge": "^0.14.1",
"ethereumjs-wallet": "^1.0.1",
"ini": "^1.3.5",
"openpgp": "^4.10.8",
"pg": "^8.4.2",
"sqlite3": "^5.0.0",
"yargs": "^16.1.0"
},
"engines": {
"node": "~14.16.1"
}
},
"node_modules/cipher-base": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
@ -418,6 +439,14 @@
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/colors": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==",
"engines": {
"node": ">=0.1.90"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@ -1555,6 +1584,14 @@
"node": ">= 6"
}
},
"node_modules/readline-sync": {
"version": "1.4.10",
"resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
"integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/request": {
"version": "2.88.2",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",
@ -2085,6 +2122,34 @@
}
},
"dependencies": {
"@cicnet/cic-client-meta": {
"version": "0.0.11",
"resolved": "https://registry.npmjs.org/@cicnet/cic-client-meta/-/cic-client-meta-0.0.11.tgz",
"integrity": "sha512-RL9CPXkWQBQzaqMoldnENC/xqinIMyWNSsejs9+qkFOWbAvC4inLdpjjCaooyvLpIZGHF9cLjxHiAdBMR9L8sQ==",
"requires": {
"@cicnet/crdt-meta": "^0.0.10",
"@ethereumjs/tx": "^3.0.0-beta.1",
"automerge": "^0.14.1",
"colors": "^1.4.0",
"ethereumjs-wallet": "^1.0.1",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"pg": "^8.4.2",
"sqlite3": "^5.0.0",
"yargs": "^16.1.0"
}
},
"@cicnet/crdt-meta": {
"version": "0.0.10",
"resolved": "https://registry.npmjs.org/@cicnet/crdt-meta/-/crdt-meta-0.0.10.tgz",
"integrity": "sha512-f+H6BQA2tE718KuNYiNzrDJN4wY00zeuhXM6aPKJUX6nryzX9g2r0yf8iDhkz+Fts1R6M7Riz73MfFEa8fgvsw==",
"requires": {
"automerge": "^0.14.2",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"readline-sync": "^1.4.10"
}
},
"@ethereumjs/common": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@ethereumjs/common/-/common-2.2.0.tgz",
@ -2112,9 +2177,9 @@
}
},
"@types/node": {
"version": "14.14.39",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.39.tgz",
"integrity": "sha512-Qipn7rfTxGEDqZiezH+wxqWYR8vcXq5LRpZrETD19Gs4o8LbklbmqotSUsMU+s5G3PJwMRDfNEYoxrcBwIxOuw=="
"version": "14.14.41",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.41.tgz",
"integrity": "sha512-dueRKfaJL4RTtSa7bWeTK1M+VH+Gns73oCgzvYfHZywRCoPSd8EkXBL0mZ9unPTveBn+D9phZBaxuzpwjWkW0g=="
},
"@types/pbkdf2": {
"version": "3.1.0",
@ -2379,22 +2444,6 @@
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"cic-client-meta": {
"version": "0.0.7-alpha.8",
"resolved": "https://registry.npmjs.org/cic-client-meta/-/cic-client-meta-0.0.7-alpha.8.tgz",
"integrity": "sha512-NtU4b4dptG2gsKXIvAv1xCxxxhrr801tb8+Co1O+VLx+wvxFyPRxqa2f2eN5nrSnFnljNsWWpE6K5bJZb1+Rqw==",
"requires": {
"@ethereumjs/tx": "^3.0.0-beta.1",
"automerge": "^0.14.1",
"crdt-meta": "0.0.8",
"ethereumjs-wallet": "^1.0.1",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"pg": "^8.4.2",
"sqlite3": "^5.0.0",
"yargs": "^16.1.0"
}
},
"cipher-base": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
@ -2462,6 +2511,11 @@
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"colors": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA=="
},
"combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@ -2495,18 +2549,6 @@
"printj": "~1.1.0"
}
},
"crdt-meta": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/crdt-meta/-/crdt-meta-0.0.8.tgz",
"integrity": "sha512-CS0sS0L2QWthz7vmu6vzl3p4kcpJ+IKILBJ4tbgN4A3iNG8wnBeuDIv/z3KFFQjcfuP4QAh6E9LywKUTxtDc3g==",
"requires": {
"automerge": "^0.14.2",
"ini": "^1.3.8",
"openpgp": "^4.10.8",
"pg": "^8.5.1",
"sqlite3": "^5.0.2"
}
},
"create-hash": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
@ -3415,6 +3457,11 @@
"util-deprecate": "^1.0.1"
}
},
"readline-sync": {
"version": "1.4.10",
"resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
"integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw=="
},
"request": {
"version": "2.88.2",
"resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz",

4
apps/data-seeding/package.json
View File

@ -1,7 +1,7 @@
{
"dependencies": {
"cic-client-meta": "^0.0.7-alpha.8",
"crdt-meta": "0.0.8",
"@cicnet/cic-client-meta": "^0.0.11",
"@cicnet/crdt-meta": "^0.0.10",
"vcard-parser": "^1.0.0"
}
}

10
apps/data-seeding/verify.py
View File

@ -72,6 +72,7 @@ argparser.add_argument('--ussd-provider', type=str, dest='ussd_provider', defaul
argparser.add_argument('--skip-custodial', dest='skip_custodial', action='store_true', help='skip all custodial verifications')
argparser.add_argument('--exclude', action='append', type=str, default=[], help='skip specified verification')
argparser.add_argument('--include', action='append', type=str, help='include specified verification')
argparser.add_argument('--token-symbol', default='SRF', type=str, dest='token_symbol', help='Token symbol to use for trnsactions')
argparser.add_argument('-r', '--registry-address', type=str, dest='r', help='CIC Registry address')
argparser.add_argument('--env-prefix', default=os.environ.get('CONFINI_ENV_PREFIX'), dest='env_prefix', type=str, help='environment prefix for variables to overwrite configuration')
argparser.add_argument('-x', '--exit-on-error', dest='x', action='store_true', help='Halt exection on error')
@ -101,6 +102,8 @@ config.censor('PASSWORD', 'SSL')
config.add(args.meta_provider, '_META_PROVIDER', True)
config.add(args.ussd_provider, '_USSD_PROVIDER', True)
token_symbol = args.token_symbol
logg.debug('config loaded from {}:\n{}'.format(config_dir, config))
celery_app = celery.Celery(backend=config.get('CELERY_RESULT_URL'), broker=config.get('CELERY_BROKER_URL'))
@ -273,7 +276,10 @@ class Verifier:
def verify_balance(self, address, balance):
o = self.erc20_tx_factory.balance(self.token_address, address)
r = self.conn.do(o)
actual_balance = int(strip_0x(r), 16)
try:
actual_balance = int(strip_0x(r), 16)
except ValueError:
actual_balance = int(r)
balance = int(balance / 1000000) * 1000000
logg.debug('balance for {}: {}'.format(address, balance))
if balance != actual_balance:
@ -461,7 +467,7 @@ def main():
tx = txf.template(ZERO_ADDRESS, token_index_address)
data = add_0x(registry_addressof_method)
h = hashlib.new('sha256')
h.update(b'SRF')
h.update(token_symbol.encode('utf-8'))
z = h.digest()
data += eth_abi.encode_single('bytes32', z).hex()
txf.set_code(tx, data)