cic-cli/cic/writers.py

122 lines
3.7 KiB
Python
Raw Normal View History

# standard imports
2022-03-01 09:17:17 +01:00
import base64
2022-04-26 13:03:38 +02:00
import json
2022-03-01 09:17:17 +01:00
import logging
import os
import sys
2021-10-19 14:18:47 +02:00
import urllib.request
2022-04-26 13:03:38 +02:00
from typing import Type, Union
2022-03-01 09:17:17 +01:00
2022-04-26 13:03:38 +02:00
from cic_types.ext.metadata import MetadataPointer, MetadataRequestsHandler
2021-10-15 12:48:14 +02:00
logg = logging.getLogger(__name__)
class OutputWriter:
2021-10-12 08:39:20 +02:00
def __init__(self, *args, **kwargs):
pass
def write(self, k, v):
raise NotImplementedError()
2021-10-12 08:39:20 +02:00
class StdoutWriter(OutputWriter):
def write(self, k, v):
2022-03-01 10:30:19 +01:00
sys.stdout.write(f"{k}\t{v}\n")
class KVWriter(OutputWriter):
2022-04-26 13:39:49 +02:00
def __init__(self, path=None, *args, **kwargs):
2021-10-19 11:18:22 +02:00
try:
os.stat(path)
except FileNotFoundError:
os.makedirs(path)
self.path = path
2022-04-26 13:03:38 +02:00
super().__init__(*args, **kwargs)
def write(self, k, v):
fp = os.path.join(self.path, str(k))
2022-03-01 10:30:19 +01:00
logg.debug(f"path write {fp} {str(v)}")
2022-03-01 09:17:17 +01:00
f = open(fp, "wb")
f.write(v)
f.close()
2021-10-19 14:18:47 +02:00
class HTTPWriter(OutputWriter):
2022-04-26 13:39:49 +02:00
def __init__(self, path=None, *args, **kwargs):
2021-10-19 14:18:47 +02:00
super(HTTPWriter, self).__init__(*args, **kwargs)
self.path = path
def write(self, k, v):
path = self.path
2022-03-01 10:30:19 +01:00
if k is not None:
path = os.path.join(path, k)
2022-03-01 09:17:17 +01:00
logg.debug(f"http writer post {path} \n key: {k}, value: {v}")
rq = urllib.request.Request(path, method="POST", data=v)
2021-10-19 14:18:47 +02:00
r = urllib.request.urlopen(rq)
2022-03-01 10:30:19 +01:00
logg.info(f"http writer submitted at {r.read()}")
class KeyedWriter(OutputWriter):
def __init__(self, writer_keyed, writer_immutable):
self.writer_keyed = writer_keyed
self.writer_immutable = writer_immutable
2022-04-26 13:03:38 +02:00
super().__init__()
2022-03-01 10:30:19 +01:00
def write(self, k, v):
logg.debug(f"writing keywriter key: {k} value: {v}")
2022-03-01 14:37:25 +01:00
if isinstance(v, str):
v = v.encode("utf-8")
2022-03-01 10:30:19 +01:00
if self.writer_keyed is not None:
self.writer_keyed.write(k, v)
if self.writer_immutable is not None:
self.writer_immutable.write(None, v)
class KeyedWriterFactory:
2022-03-01 09:17:17 +01:00
def __init__(
2022-04-26 13:03:38 +02:00
self, key_writer_constructor, immutable_writer_constructor, *_args, **kwargs
2022-03-01 09:17:17 +01:00
):
self.key_writer_constructor = key_writer_constructor
self.immutable_writer_constructor = immutable_writer_constructor
self.x = {}
2022-03-01 10:30:19 +01:00
for k, v in kwargs.items():
logg.debug(f"adding key {k} t keyed writer factory")
self.x[k] = v
2022-04-26 13:39:49 +02:00
def new(self, path=None, *_args, **_kwargs):
writer_keyed = None
writer_immutable = None
2022-03-01 10:30:19 +01:00
if self.key_writer_constructor is not None:
writer_keyed = self.key_writer_constructor(path, **self.x)
2022-03-01 10:30:19 +01:00
if self.immutable_writer_constructor is not None:
writer_immutable = self.immutable_writer_constructor(path, **self.x)
return KeyedWriter(writer_keyed, writer_immutable)
2022-03-01 09:17:17 +01:00
class MetadataWriter(OutputWriter):
"""Custom writer for publishing data under immutable content-addressed pointers in the cic-meta storage backend.
Data that is not utf-8 will be converted to base64 before publishing.
Implements cic.writers.OutputWriter
"""
def write(self, k, v):
rq = MetadataRequestsHandler(MetadataPointer.NONE, bytes.fromhex(k))
try:
v = v.decode("utf-8")
v = json.loads(v)
logg.debug(f"metadatawriter bindecode {k} {v}")
except UnicodeDecodeError:
v = base64.b64encode(v).decode("utf-8")
v = json.loads(json.dumps(v, separators=(",", ":")))
logg.debug(f"metadatawriter b64encode {k} {v}")
r = rq.create(v)
logg.info(f"metadata submitted at {k}")
return r
2022-04-26 13:03:38 +02:00
WritersType = Union[
Type[OutputWriter], Type[KeyedWriter], Type[MetadataWriter], Type[OutputWriter]
]