2021-10-11 12:23:08 +02:00
|
|
|
|
# standard imports
|
|
|
|
|
import os
|
2021-10-11 17:39:01 +02:00
|
|
|
|
import sys
|
2021-10-15 12:48:14 +02:00
|
|
|
|
import logging
|
2021-10-19 14:18:47 +02:00
|
|
|
|
import urllib.request
|
2021-10-15 12:48:14 +02:00
|
|
|
|
|
|
|
|
|
logg = logging.getLogger(__name__)
|
2021-10-11 12:23:08 +02:00
|
|
|
|
|
|
|
|
|
|
2021-10-11 17:39:01 +02:00
|
|
|
|
class OutputWriter:
|
|
|
|
|
|
2021-10-12 08:39:20 +02:00
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
|
pass
|
|
|
|
|
|
2021-10-11 17:39:01 +02:00
|
|
|
|
def write(self, k, v):
|
|
|
|
|
raise NotImplementedError()
|
|
|
|
|
|
|
|
|
|
|
2021-10-12 08:39:20 +02:00
|
|
|
|
class StdoutWriter(OutputWriter):
|
2021-10-11 17:39:01 +02:00
|
|
|
|
|
|
|
|
|
def write(self, k, v):
|
|
|
|
|
sys.stdout.write('{}\t{}\n'.format(k, v))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class KVWriter(OutputWriter):
|
2021-10-11 12:23:08 +02:00
|
|
|
|
|
2021-10-12 08:39:20 +02:00
|
|
|
|
def __init__(self, path=None, *args, **kwargs):
|
2021-10-19 11:18:22 +02:00
|
|
|
|
try:
|
|
|
|
|
os.stat(path)
|
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
os.makedirs(path)
|
2021-10-11 12:23:08 +02:00
|
|
|
|
self.path = path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write(self, k, v):
|
2021-10-11 15:28:09 +02:00
|
|
|
|
fp = os.path.join(self.path, str(k))
|
2021-10-15 12:48:14 +02:00
|
|
|
|
logg.debug('path write {} {}'.format(fp, str(v)))
|
2021-10-11 17:39:01 +02:00
|
|
|
|
f = open(fp, 'wb')
|
2021-10-11 12:23:08 +02:00
|
|
|
|
f.write(v)
|
|
|
|
|
f.close()
|
2021-10-19 14:18:47 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class HTTPWriter(OutputWriter):
|
|
|
|
|
|
|
|
|
|
def __init__(self, path=None, *args, **kwargs):
|
|
|
|
|
super(HTTPWriter, self).__init__(*args, **kwargs)
|
|
|
|
|
self.path = path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write(self, k, v):
|
2021-10-22 16:42:38 +02:00
|
|
|
|
path = self.path
|
|
|
|
|
if k != None:
|
|
|
|
|
path = os.path.join(path, k)
|
|
|
|
|
logg.debug('http writer post {}'.format(path))
|
|
|
|
|
rq = urllib.request.Request(path, method='POST', data=v)
|
2021-10-19 14:18:47 +02:00
|
|
|
|
r = urllib.request.urlopen(rq)
|
2021-10-22 19:41:01 +02:00
|
|
|
|
logg.info('http writer submitted at {}'.format(r.read()))
|
2021-10-22 16:42:38 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class KeyedWriter(OutputWriter):
|
|
|
|
|
|
|
|
|
|
def __init__(self, writer_keyed, writer_immutable):
|
|
|
|
|
self.writer_keyed = writer_keyed
|
|
|
|
|
self.writer_immutable = writer_immutable
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def write(self, k, v):
|
|
|
|
|
logg.debug('writing keywriter {} {}'.format(k, v))
|
|
|
|
|
if isinstance(v, str):
|
|
|
|
|
v = v.encode('utf-8')
|
|
|
|
|
if self.writer_keyed != None:
|
|
|
|
|
self.writer_keyed.write(k, v)
|
|
|
|
|
if self.writer_immutable != None:
|
|
|
|
|
self.writer_immutable.write(None, v)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class KeyedWriterFactory:
|
|
|
|
|
|
|
|
|
|
def __init__(self, key_writer_constructor, immutable_writer_constructor, *args, **kwargs):
|
|
|
|
|
self.key_writer_constructor = key_writer_constructor
|
|
|
|
|
self.immutable_writer_constructor = immutable_writer_constructor
|
|
|
|
|
self.x = {}
|
|
|
|
|
for k in kwargs.keys():
|
|
|
|
|
logg.debug('adding key {} t keyed writer factory'.format(k))
|
|
|
|
|
self.x[k] = kwargs[k]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def new(self, path=None, *args, **kwargs):
|
|
|
|
|
writer_keyed = None
|
|
|
|
|
writer_immutable = None
|
|
|
|
|
if self.key_writer_constructor != None:
|
|
|
|
|
writer_keyed = self.key_writer_constructor(path, **self.x)
|
|
|
|
|
if self.immutable_writer_constructor != None:
|
|
|
|
|
writer_immutable = self.immutable_writer_constructor(path, **self.x)
|
|
|
|
|
return KeyedWriter(writer_keyed, writer_immutable)
|