commit a73ffc19935b3420264fccb87d9b0458c1015a0e Author: timotheyca Date: Mon Jul 19 12:40:59 2021 +0300 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b96173 --- /dev/null +++ b/.gitignore @@ -0,0 +1,225 @@ +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + + + + + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + + + + +# Other +/dev.py +/*.db diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..73f69e0 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..69f4ba6 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,51 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..21c84a0 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..20fe4ca --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/ptvp35.iml b/.idea/ptvp35.iml new file mode 100644 index 0000000..74d515a --- /dev/null +++ b/.idea/ptvp35.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/ptvp35/__init__.py b/ptvp35/__init__.py new file mode 100644 index 0000000..4efb0a1 --- /dev/null +++ b/ptvp35/__init__.py @@ -0,0 +1,237 @@ +import asyncio +import json +import pathlib +import pickle +from io import StringIO +from typing import Any, Optional, IO, Type + + +class Request: + def set_result(self, result): + pass + + def set_exception(self, exception): + pass + + +class KVRequest(Request): + def __init__(self, key: Any, value: Any, future: Optional[asyncio.Future]): + self.key = key + self.value = value + self.future = future + + def free(self): + return type(self)(self.key, self.value, None) + + def line(self) -> str: + raise NotImplementedError + + @classmethod + def fromline(cls, line: str) -> 'KVRequest': + raise NotImplementedError + + def set_result(self, result): + self.future and self.future.set_result(result) + + def set_exception(self, exception): + self.future and self.future.set_exception(exception) + + +class DumpRequest(Request): + pass + + +class UnkownRequestType(TypeError): + pass + + +class KVPickle(KVRequest): + def line(self) -> str: + return pickle.dumps(self.free()).hex() + "\n" + + @classmethod + def fromline(cls, line: str) -> 'KVPickle': + return pickle.loads(bytes.fromhex(line.strip())) + + +class KVJson(KVRequest): + def line(self) -> str: + return json.dumps({'key': self.key, 'value': self.value}) + "\n" + + @classmethod + def fromline(cls, line: str) -> 'KVJson': + d = json.loads(line) + return KVJson(d['key'], d['value'], None) + + +class Db: + __mmdb: Optional[dict] + __loop: Optional[asyncio.AbstractEventLoop] + __queue: Optional[asyncio.Queue] + __file: Optional[IO[str]] + __buffer: Optional[StringIO] + __task: Optional[asyncio.Future] + + def __init__(self, path: str, *, kvrequest_type: Type[KVRequest], buffersize=1048576): + self.kvrequest_type = kvrequest_type + self.buffersize = buffersize + self.__path = pathlib.Path(path) + self.__task = None + + def io2db(self, io: IO[str], db: dict) -> int: + size = 0 + for line in io: + request = self.kvrequest_type.fromline(line) + db[request.key] = request.value + size += len(line) + return size + + def db2io(self, db: dict, io: IO[str]) -> int: + size = 0 + for key, value in db.items(): + size += io.write(self.kvrequest_type(key, value, None).line()) + return size + + def get(self, key: Any, default: Any): + return self.__mmdb.get(key, default) + + async def set(self, key: Any, value: Any): + self.__mmdb[key] = value + future = self.__loop.create_future() + self.__queue.put_nowait(self.kvrequest_type(key, value, future)) + await future + + def set_nowait(self, key: Any, value: Any): + self.__mmdb[key] = value + self.__queue.put_nowait(self.kvrequest_type(key, value, None)) + + async def _dump_buffer_or_request_so(self): + if self.__buffer.tell() >= self.buffersize: + await self._dump_buffer() + else: + await self.__queue.put(DumpRequest()) + + async def _write(self, line: str): + self.__buffer.write(line) + await self._dump_buffer_or_request_so() + + def _clear_buffer(self): + self.__buffer = StringIO() + + def _compress_buffer(self) -> StringIO: + self.__buffer.seek(0) + bufferdb = {} + self.io2db(self.__buffer, bufferdb) + buffer = StringIO() + self.db2io(bufferdb, buffer) + return buffer + + async def _dump_compressed_buffer(self): + buffer = self._compress_buffer() + await self.__loop.run_in_executor(None, self.__file.write, buffer.getvalue()) + + async def _do_dump_buffer(self): + await self._dump_compressed_buffer() + self._clear_buffer() + + async def _reload_if_oversized(self): + if self.__file.tell() > 2 * self.__initial_size: + await self.__loop.run_in_executor(None, self._reload) + + async def _dump_buffer(self): + if self.__buffer.tell(): + await self._do_dump_buffer() + await self._reload_if_oversized() + + async def _handle_request(self, request: Request): + if isinstance(request, self.kvrequest_type): + await self._write(request.line()) + elif isinstance(request, DumpRequest): + await self._dump_buffer() + else: + raise UnkownRequestType + request.set_result(None) + + async def _background_cycle(self): + request: Request = await self.__queue.get() + try: + await self._handle_request(request) + except Exception as e: + request.set_exception(e) + finally: + self.__queue.task_done() + + async def _background_task(self): + while True: + await self._background_cycle() + + def _rebuild_file(self, db: {}): + self.__path.touch() + with open(self.__path) as file: + self.io2db(file, db) + with open(self.__path, "w") as file: + self.__initial_size = self.db2io(db, file) + + def _reload(self): + self.__file.close() + self._rebuild_file({}) + self.__file = open(self.__path, "a") + + def _load_from_file(self): + self._rebuild_file(self.__mmdb) + + async def _initialize_mmdb(self): + self.__mmdb = {} + await self.__loop.run_in_executor(None, self._load_from_file) + + async def _initialize_queue(self): + self.__queue = asyncio.Queue() + self.__file = open(self.__path, "a") + self.__buffer = StringIO() + self.__task = self.__loop.create_task(self._background_task()) + + async def _initialize(self): + assert self.__task is None + self.__loop = asyncio.get_event_loop() + await self._initialize_mmdb() + await self._initialize_queue() + + async def __aenter__(self): + await self._initialize() + return self + + async def _aclose(self): + if not self.__task.done(): + await self.__queue.join() + self.__task.cancel() + await self._dump_buffer() + self.__file.close() + with open(self.__path, "w") as file: + self.db2io(self.__mmdb, file) + + def _uninitialize(self): + self.__mmdb = None + self.__loop = None + self.__queue = None + self.__file = None + self.__buffer = None + self.__task = None + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._aclose() + self._uninitialize() + + def cursor(self, **kwargs): + return Cursor(self, **kwargs) + + +class Cursor: + def __init__(self, db: Db, default=None): + self.default = default + self.db = db + + def __getitem__(self, item): + return self.db.get(item, self.default) + + def __setitem__(self, key, value): + self.db.set_nowait(key, value)