From 04e8ba559e30e361b37724aa9795c0615303d0ab Mon Sep 17 00:00:00 2001 From: timofey Date: Sun, 20 Nov 2022 16:00:08 +0000 Subject: [PATCH] docs --- Dockerfile | 15 ++++++++ docs/Makefile | 20 +++++++++++ docs/make.bat | 35 ++++++++++++++++++ docs/source/conf.py | 36 +++++++++++++++++++ docs/source/index.rst | 25 +++++++++++++ docs/source/modules.rst | 7 ++++ docs/source/motivation.rst | 28 +++++++++++++++ docs/source/ptvp35.rst | 10 ++++++ docs/source/usage.rst | 54 ++++++++++++++++++++++++++++ ptvp35/__init__.py | 73 ++++++++++++++++++++++++++++++++++---- 10 files changed, 297 insertions(+), 6 deletions(-) create mode 100644 Dockerfile create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/modules.rst create mode 100644 docs/source/motivation.rst create mode 100644 docs/source/ptvp35.rst create mode 100644 docs/source/usage.rst diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3f89f89 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 +FROM python:3.10 +WORKDIR /app/ +RUN apt-get update +RUN apt-get install -y python3-sphinx node.js +RUN apt-get install -y npm +RUN npm install -g http-server +RUN pip install pydata-sphinx-theme +COPY docs/Makefile Makefile +COPY setup.py setup.py +COPY docs/source source +COPY ptvp35 ptvp35 +RUN make html +WORKDIR /app/build/html/ +CMD [ "http-server", "-p", "80" ] diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..447077b --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,36 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'ptvp35' +copyright = '2022, PARRRATE TNV' +author = 'PARRRATE TNV' +release = '1.0rc2' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx.ext.autodoc', +] + +templates_path = ['_templates'] +exclude_patterns = [] + + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'pydata_sphinx_theme' +html_static_path = ['_static'] + + +import sys +import os.path + +sys.path.insert(0, os.path.abspath('..')) diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..acfa18c --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,25 @@ +.. ptvp35 documentation master file, created by + sphinx-quickstart on Sat Nov 19 20:02:24 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to ptvp35's documentation! +================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + motivation + usage + modules + + + +Indices and tables +================== + +* :doc:`motivation` +* :doc:`usage` +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..5e713dd --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +ptvp35 +====== + +.. toctree:: + :maxdepth: 4 + + ptvp35 diff --git a/docs/source/motivation.rst b/docs/source/motivation.rst new file mode 100644 index 0000000..04245e0 --- /dev/null +++ b/docs/source/motivation.rst @@ -0,0 +1,28 @@ +Motivation +========== + +General structure +----------------- + +* We're making a key-value database. +* Keys and values are python objects. +* While the DB is offline (after a correct shutdown), all its KVPs are stored as lines in one editable file. +* DB should be able to survive powercut at any point (D in ACID). + +AP/LP get +------ + +The central idea for this database is the zero-latency (as far as python goes) reads. Therefore, the DB has all its data stored as a :code:`dict`. + +CAP/CLP set +------- + +The database allows both availability-(zero)latency and consistency (note: it's not ACID consistency, it's CAP consistency and ACID durability) variants for its writes. +All zero-latency writes are also consistent with respect to (zero-latency) reads. + +Consistent get +------ + +Later versions of the database will include remote connections, and thus also consistent reads. +These will probably be implemented as requests (same as writes). +Some connection variants may not even include availability reads/writes. diff --git a/docs/source/ptvp35.rst b/docs/source/ptvp35.rst new file mode 100644 index 0000000..b5728f2 --- /dev/null +++ b/docs/source/ptvp35.rst @@ -0,0 +1,10 @@ +ptvp35 package +============== + +Module contents +--------------- + +.. automodule:: ptvp35 + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..242bfa4 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,54 @@ +Usage +===== + +Installation +------------ + +Default installation option is to use pip+git + +.. code-block:: console + + (.venv) $ pip install git+https://gitea.parrrate.ru/PTV/ptvp35.git + +Basic functionality +------------------- + +.. autoclass:: ptvp35.DbFactory + + :code:`DbFactory` class provides context management for database connections (:code:`DbConnection`) via :code:`async with` statement. + The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**. + Also, that means that each connection start/shutdown is quite time expensive. + These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block. + +.. code-block:: python3 + + import pathlib + from ptvp35 import DbFactory, KVJson + + async def main(): + async with DbFactory(pathlib.Path('example.db', kvfactory=KVJson())) as connection: + await _main(connection) + +.. autoclass:: ptvp35.DbConnection + + .. automethod:: get + + this method is instant. + + .. automethod:: set + + this method may take time to run. + ordering may not be guaranteed (depends on event loop implementation). + + .. automethod:: set_nowait + + this method is instant. + ordering is guaranteed. + + .. automethod:: commit + + this method may take time to run. + respects the ordering of previously called :code:`set_nowait` methods. + will, under most circumstances, also execute later changes. + + .. automethod:: transaction diff --git a/ptvp35/__init__.py b/ptvp35/__init__.py index 6cfec13..e770896 100644 --- a/ptvp35/__init__.py +++ b/ptvp35/__init__.py @@ -9,7 +9,15 @@ from io import StringIO, UnsupportedOperation from typing import Any, Optional, IO, Hashable -__all__ = ('KVRequest', 'KVJson', 'DbConnection', 'DbFactory', 'Db',) +__all__ = ( + 'KVFactory', + 'KVJson', + 'DbConnection', + 'DbFactory', + 'Db', + 'Transaction', + 'FallbackMapping', +) class Request: @@ -37,18 +45,27 @@ class Request: class KVFactory: + """note: unstable signature.""" + __slots__ = () def line(self, key: Any, value: Any, /) -> str: + """line must contain exactly one '\\n' at exactly the end if the line is not empty.""" raise NotImplementedError def fromline(self, line: str, /) -> 'KVRequest': + """inverse of line(). should use free() method to construct the request.""" raise NotImplementedError def request(self, key: Any, value: Any, /, *, future: Optional[asyncio.Future]) -> 'KVRequest': + """form request with Future. +low-level API. +note: unstable signature.""" return KVRequest(key, value, future=future, factory=self) def free(self, key: Any, value: Any, /) -> 'KVRequest': + """result free from Future. +note: unstable signature.""" return self.request(key, value, future=None) @@ -81,6 +98,8 @@ class UnknownRequestType(TypeError): class KVJson(KVFactory): + """note: unstable signature.""" + __slots__ = () def line(self, key: Any, value: Any, /) -> str: @@ -118,6 +137,8 @@ class TransactionRequest(Request): class DbConnection: + """note: unstable constructor signature.""" + __slots__ = ( '__factory', '__path', @@ -182,7 +203,8 @@ class DbConnection: self._queue_error(line).result() def io2db(self, io: IO[str], db: dict, /) -> int: - """there are no guarantees about .error file if error occurs here""" + """there are no guarantees about .error file if an error occurs here. +note: unstable signature.""" size = 0 for line in io: try: @@ -196,6 +218,8 @@ class DbConnection: return size def db2io(self, db: dict, io: IO[str], /) -> int: + """does not handle any errors. +note: unstable signature.""" size = 0 for key, value in db.items(): size += io.write(self.__factory.kvfactory.free(key, value).line()) @@ -211,9 +235,11 @@ class DbConnection: return self.db2io(db, file) def get(self, key: Any, default: Any, /): + """dict-like get with mandatory default parametre.""" return self.__mmdb.get(key, default) async def set(self, key: Any, value: Any, /) -> None: + """set the value and wait until it's written to disk.""" self.__mmdb[key] = value future = self._create_future() self.__queue.put_nowait( @@ -221,6 +247,7 @@ class DbConnection: await future def set_nowait(self, key: Any, value: Any, /) -> None: + """set value and add write-to-disk request to queue.""" self.__mmdb[key] = value self.__queue.put_nowait(self.__factory.kvfactory.free(key, value)) @@ -448,11 +475,15 @@ intended for heavy tasks.""" @classmethod async def create(cls, factory: 'DbFactory', /) -> 'DbConnection': + """connect to the factory. +note: unstable signature.""" dbconnection = DbConnection(factory) await dbconnection._initialize() return dbconnection async def aclose(self, /) -> None: + """close the connection. +note: unstable signature.""" if not self.__task.done(): await self.__queue.join() self.__task.cancel() @@ -473,6 +504,7 @@ intended for heavy tasks.""" self.__not_running = True async def complete_transaction(self, delta: dict, /) -> None: + """hybrid of set() and dict.update().""" if not delta: return buffer = StringIO() @@ -481,13 +513,20 @@ intended for heavy tasks.""" future = self._create_future() self.__queue.put_nowait(TransactionRequest(buffer, future=future)) await future + + def submit_transaction(self, delta: dict, /) -> asyncio.Future | None: + """not implemented. +low-level API.""" + raise NotImplementedError async def commit(self, /) -> None: + """wait until all requests queued before are completed.""" future = self._create_future() self.__queue.put_nowait(DumpRequest(future)) await future def transaction(self, /) -> 'Transaction': + """open new transaction.""" return Transaction(self) @@ -496,23 +535,28 @@ class DbFactory: 'path', 'kvfactory', 'buffersize', - 'db', + '__db', ) def __init__(self, path: pathlib.Path, /, *, kvfactory: KVFactory, buffersize=1048576) -> None: self.path = path + """note: unstable signature.""" self.kvfactory = kvfactory + """note: unstable signature.""" self.buffersize = buffersize + """note: unstable signature.""" async def __aenter__(self) -> DbConnection: - self.db = await DbConnection.create(self) - return self.db + self.__db = await DbConnection.create(self) + return self.__db async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.db.aclose() + await self.__db.aclose() class Db(DbConnection): + """simplified usecase combining the factory and the connection in one class.""" + __slots__ = () def __init__(self, path: str | pathlib.Path, /, *, kvfactory: KVFactory, buffersize=1048576): @@ -532,6 +576,8 @@ class Db(DbConnection): class FallbackMapping: + """note: unstable constructor signature.""" + __slots__ = ( '__delta', '__shadow', @@ -554,13 +600,28 @@ class FallbackMapping: self.__delta[key] = value async def commit(self, /) -> None: + """bulk analog of DbConnection.set method.""" delta = self.__delta.copy() self.__shadow |= delta self.__delta.clear() await self.__connection.complete_transaction(delta) + async def commit_submitted(self, /) -> None: + """not implemented. +commit previously submitted changes.""" + raise NotImplementedError + + def submit(self, /) -> None: + """not implemented. +submit changes. +_nowait analog of commit. +bulk analog of DbConnection.set_nowait method.""" + raise NotImplementedError + class Transaction: + """note: unstable signature.""" + __slots__ = ( '__connection', '__delta',