formatting

.dockerignore

@@ -1 +1,4 @@
 .git*
+__pycache__
+*.egg-info
+build

.gitignore

@@ -222,5 +222,5 @@ cython_debug/
 
 # Other
 /dev.py
-/*.db
-/*.db.*
+*.db
+*.db.*

Dockerfile

@@ -1,41 +1,71 @@
 # syntax=docker/dockerfile:1
-FROM python:3.10
+
+FROM python:3.10 as compile-sphinx5.3.0-base
+
 RUN apt-get update
-RUN apt-get install -y python3-sphinx node.js npm
-RUN npm install -g http-server
-RUN pip install pydata-sphinx-theme
+RUN apt-get install -y python3-sphinx
+RUN pip install Sphinx==5.3.0 pydata-sphinx-theme==0.12.0
 WORKDIR /app/
 RUN pip install git+https://gitea.parrrate.ru/PTV/rainbowadn.git@e9fba7b064902ceedee0dd5578cb47030665a6aa
 
 RUN mkdir /app/docs/
-RUN mkdir /app/docs/build/
-RUN mkdir /app/docs/build/html/
+
+
+FROM compile-sphinx5.3.0-base as compile-legacy
+
 WORKDIR /app/legacy/
 RUN git clone --branch 1.0 https://gitea.parrrate.ru/PTV/ptvp35.git
 WORKDIR /app/legacy/ptvp35/
+
+
+FROM compile-legacy as compile-1.0
+
 RUN python traced_example.py > traced_example.txt
 RUN python traced_example.py all > traced_example_all.txt
 RUN cp -r docs/source/ ./source/
 RUN cp docs/Makefile ./Makefile
 RUN make html
-RUN cp -r /app/legacy/ptvp35/build/html/ /app/docs/build/html/1.0/
-RUN git reset --hard
+
+
+FROM compile-legacy as compile-1.1rc0
 
 RUN git fetch && git checkout 1.1rc0
 WORKDIR /app/legacy/ptvp35/docs/
 RUN make html
-RUN cp -r /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1rc0/
-RUN rm -r /app/legacy/ptvp35/docs/build/
-WORKDIR /app/legacy/ptvp35/
 
-WORKDIR /app/
+
+FROM compile-legacy as compile-1.1rc2
+
+RUN git fetch && git checkout 1.1rc2
+WORKDIR /app/legacy/ptvp35/docs/
+RUN make html
+
+
+FROM compile-legacy as compile-1.1.0
+
+RUN git fetch && git checkout 1.1.0
+WORKDIR /app/legacy/ptvp35/docs/
+RUN make html
+
+
+FROM compile-sphinx5.3.0-base as compile-latest
 
 COPY docs/Makefile docs/Makefile
 COPY setup.py setup.py
-COPY traced_example.py traced_example.py
+COPY docs/scripts docs/scripts
 COPY ptvp35 ptvp35
 COPY docs/source docs/source
 WORKDIR /app/docs/
 RUN make html
+
+
+FROM node:19
+
+RUN npm install -g http-server
 WORKDIR /app/docs/build/html/
+COPY --from=compile-1.0 /app/legacy/ptvp35/build/html/ /app/docs/build/html/1.0/
+COPY --from=compile-1.1rc0 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1rc0/
+COPY --from=compile-1.1rc2 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1rc2/
+COPY --from=compile-1.1.0 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1.0/
+COPY --from=compile-latest /app/docs/build/html/ /app/docs/build/html/
 CMD [ "http-server", "-p", "80" ]

docs/Makefile

@@ -17,6 +17,6 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	python ../traced_example.py > ../traced_example.txt
-	python ../traced_example.py all > ../traced_example_all.txt
+	python scripts/traced_example.py > scripts/traced_example.txt
+	python scripts/traced_example.py all > scripts/traced_example_all.txt
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/scripts/traced_example.py

@@ -0,0 +1,320 @@
+import asyncio
+import pathlib
+import sys
+import threading
+from contextlib import ExitStack
+
+from rainbowadn.instrument import Instrumentation
+
+try:
+    sys.path.append(str((pathlib.Path(__file__).parent / "../..").absolute()))
+    from ptvp35 import DbConnection, DbFactory, KVJson
+    from ptvp35.instrumentation import InstrumentDiskWrites
+except:
+    raise
+
+
+async def aprint(*args, **kwargs):
+    print(*args, **kwargs)
+
+
+class LogWrites(InstrumentDiskWrites):
+    def __init__(self, /):
+        super().__init__()
+        self.loop = asyncio.get_running_loop()
+
+    def on_write(self, line: str, /) -> None:
+        asyncio.run_coroutine_threadsafe(aprint(f"{self.methodname}[{line}]"), self.loop).result()
+
+
+class LogEE(Instrumentation):
+    def __init__(self, target, methodname: str):
+        super().__init__(target, methodname)
+        self.loop = asyncio.get_running_loop()
+
+    def _target_id(self) -> str:
+        name = self.target.__name__ if hasattr(self.target, "__name__") else self.target.__class__.__name__
+        return f"{name}.{self.methodname}"
+
+    def _print(self, thread, *args) -> None:
+        print(thread, self._target_id(), *args, sep="\t")
+
+    async def aprint(self, thread, *args) -> None:
+        self._print(thread, *args)
+
+    def print(self, *args) -> None:
+        if (ct := threading.current_thread()) is threading.main_thread():
+            self._print("main", *args)
+        else:
+            asyncio.run_coroutine_threadsafe(self.aprint("aux", *args), self.loop).result()
+
+    def instrument(self, method, *args, **kwargs):
+        self.print("enter")
+        try:
+            result = method(*args, **kwargs)
+        except:
+            self.print("error")
+            raise
+        else:
+            self.print("exit")
+        return result
+
+
+class ALogEE(LogEE):
+    async def instrument(self, method, *args, **kwargs):
+        self._print("aio", "enter")
+        try:
+            result = await method(*args, **kwargs)
+        except:
+            self._print("aio", "error")
+            raise
+        else:
+            self._print("aio", "exit")
+        return result
+
+
+async def transaction_test(db: DbConnection):
+    def logdb(*args):
+        if args:
+            args = (
+                " ",
+                " ",
+                "@",
+            ) + args
+        print(db.get("test", "0"), *args, sep="\t")
+
+    def logstate(*args):
+        if args:
+            args = ("@",) + args
+        print(db.get("test", "0"), "|", state.get("test", "0"), *args, sep="\t")
+
+    logdb("empty db")
+    db.set_nowait("test", "1")
+    logdb("after set_nowait")
+    await db.set("test", "2")
+    logdb("after set")
+    try:
+        async with db.transaction() as state:
+            logstate("empty transaction")
+            state.set_nowait("test", "3")
+            logstate("after transaction.set_nowait")
+            state.submit()
+            logstate("after transaction.submit")
+            await state.commit()
+            logstate("after transaction.commit")
+            state.set_nowait("test", print)  # will throw TypeError later
+            logstate()
+    except TypeError:
+        print("type error")
+    logdb("after transaction")
+    async with db.transaction() as state:
+        logstate()
+        state.set_nowait("test", "4")
+        logstate("before implicit transaction.commit")
+    logdb("after transaction with implicit commit")
+    with db.transaction() as state:
+        logstate()
+        state.set_nowait("test", "5")
+        logstate("before implicit transaction.submit")
+    logdb("after transaction with implicit submit")
+
+
+def print_private_db_attrs(db: DbConnection):
+    if run_all:
+        for attr in dir(db):
+            if attr.startswith("_DbConnection") and hasattr(db, attr):
+                print(attr)
+
+
+run_all = "all" in sys.argv
+
+
+async def main():
+    (path := pathlib.Path(__file__).parent / "trace_example.db").unlink(missing_ok=True)
+
+    with ExitStack() as es:
+        LogWrites().enter(es)
+        if run_all:
+            LogEE(__import__("ptvp35").Request, "__init__").enter(es)
+            LogEE(__import__("ptvp35").Request, "waiting").enter(es)
+            LogEE(__import__("ptvp35").Request, "set_result").enter(es)
+            LogEE(__import__("ptvp35").Request, "set_exception").enter(es)
+            ALogEE(__import__("ptvp35").Request, "wait").enter(es)
+
+            LogEE(__import__("ptvp35").LineRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").KVFactory, "run").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "_dbset").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "dbset").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "dbget").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "filter_value").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "request").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "free").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "io2db").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "db2io").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "path2db_sync").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "db2path_sync").enter(es)
+
+            LogEE(__import__("ptvp35").KVRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").KVJson, "line").enter(es)
+            LogEE(__import__("ptvp35").KVJson, "_load_key").enter(es)
+            LogEE(__import__("ptvp35").KVJson, "fromline").enter(es)
+
+            LogEE(__import__("ptvp35").TransactionRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").DbParameters, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").VirtualConnection, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35")._Loop, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "create_future").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "loop").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "run_in_thread").enter(es)
+
+            LogEE(__import__("ptvp35")._Errors, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Errors, "_save_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Errors, "_save").enter(es)
+            LogEE(__import__("ptvp35")._Errors, "save_from_thread").enter(es)
+
+            LogEE(__import__("ptvp35")._File, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._File, "path").enter(es)
+            LogEE(__import__("ptvp35")._File, "tell").enter(es)
+            LogEE(__import__("ptvp35")._File, "write_to_disk_sync").enter(es)
+            LogEE(__import__("ptvp35")._File, "open_sync").enter(es)
+            LogEE(__import__("ptvp35")._File, "close_sync").enter(es)
+
+            LogEE(__import__("ptvp35")._Backup, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "file").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "kvfactory").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_copy_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_recovery_unset_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_finish_recovery_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_recovery_set_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "build_file_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_rebuild_file_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_reload_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "run_in_thread").enter(es)
+            ALogEE(__import__("ptvp35")._Backup, "_reload").enter(es)
+            ALogEE(__import__("ptvp35")._Backup, "reload_if_oversized").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "load_mmdb_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "uninitialize").enter(es)
+
+            LogEE(__import__("ptvp35")._Guard, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "backup").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_write_bytes_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_write_value_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_set_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_unset_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_read_bytes_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_read_value_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_truncate_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "assure_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_file_truncate_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "file_write_sync").enter(es)
+
+            LogEE(__import__("ptvp35")._ReceivingQueue, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._ReceivingQueue, "submit").enter(es)
+
+            LogEE(__import__("ptvp35")._WriteableBuffer, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "writeable").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "loop").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_compressed").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_commit_compressed_sync").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit_compressed").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_clear").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_satisfy_future").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_fail_future").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_do_commit_buffer").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_request_buffer").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit_or_request_so").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_write").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_handle_request").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_close").enter(es)
+
+            LogEE(__import__("ptvp35")._Memory, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_initialize_sync").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_load_from_file_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Memory, "_load_from_file").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_close_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Memory, "_close").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_transaction_buffer").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "get").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "set").enter(es)
+
+            LogEE(__import__("ptvp35")._QueueTask, "__init__").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "_background_cycle").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "_background_task").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "close").enter(es)
+            LogEE(__import__("ptvp35")._QueueTask, "start").enter(es)
+
+            LogEE(__import__("ptvp35")._DbConnection, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "kvprotocol").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "get").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "set").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "set_nowait").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_initialize_running").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_initialize").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "create").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_close_running").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "aclose").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "commit_transaction").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "submit_transaction").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "submit_transaction_request").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "commit").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "loop").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35").DbManager, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").DbManager, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").DbManager, "__aexit__").enter(es)
+
+            LogEE(__import__("ptvp35").Db, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").Db, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").Db, "__aexit__").enter(es)
+
+            LogEE(__import__("ptvp35").FutureContext, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "__aexit__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "wait").enter(es)
+
+            LogEE(__import__("ptvp35").TransactionView, "__init__").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "future_context").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "rollback").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "illuminate").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "ailluminate").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "fork").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "afork").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "clear").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "aclear").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "reset").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "areset").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "get").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "set_nowait").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_delta").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "commit").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_do_gather").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_reduce_future").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_gather").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "commit_transaction").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit_transaction").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit_transaction_request").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "loop").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35").Transaction, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").Transaction, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").Transaction, "__aexit__").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "_clean").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "__enter__").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "__exit__").enter(es)
+        async with DbFactory(path, kvfactory=KVJson()) as db:
+            await transaction_test(db)
+            print_private_db_attrs(db)
+        print_private_db_attrs(db)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/source/conf.py

@@ -9,34 +9,34 @@
 import os.path
 import sys
 
-project = 'ptvp35'
-copyright = '2022, PARRRATE TNV'
-author = 'PARRRATE TNV'
-with open('../../setup.py') as f:
+project = "ptvp35"
+copyright = "2022, PARRRATE TNV"
+author = "PARRRATE TNV"
+with open("../../setup.py") as f:
     _src = f.read()
-    _src = _src[_src.index('version=\'') + 9:]
-    _src = _src[:_src.index('\'')]
+    _src = _src[_src.index('version="') + 9 :]
+    _src = _src[: _src.index('"')]
     release = _src
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
-    'sphinx.ext.autodoc',
+    "sphinx.ext.autodoc",
 ]
 
-templates_path = ['_templates']
+templates_path = ["_templates"]
 exclude_patterns = []
 
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'pydata_sphinx_theme'
+html_theme = "pydata_sphinx_theme"
 html_theme_options = {
     "navbar_center": [],
 }
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 
-sys.path.insert(0, os.path.abspath('../..'))
+sys.path.insert(0, os.path.abspath("../.."))

docs/source/development.rst

@@ -18,3 +18,5 @@ These apply both to the internal PARRRATE TNV team behind Persistence 5 and to e
     * Forking of the repository is encouraged.
     * Usage of the repository as a reference for custom data storage solutions is encouraged.
 * Instrumentation code base is more open to direct code contributions.
+    * Main way of instrumentation is code injection.
+    * Instrumentation on its own shouldn't require changes to the core code, even though instrumentation-allowing core code is preferred.

docs/source/history.rst

@@ -1,12 +1,7 @@
 Historical notes
 ================
 
-Persistence 1.
---------------
-
-* Used by CMB.
-
-Persistence 2. CmbPrst.
+Persistence 1/2. CmbPrst.
 --------------
 
 * Internal storage of CMB.
@@ -45,6 +40,8 @@ Persistence 5 1.1 (5.1.1).
 
 * Non-nightly support for transactions.
 * Instrumentation support.
+* Structural preparations for generic DBs.
+* :code:`VDELETE`
 
 Proposed future versions
 ========================

docs/source/index.rst

@@ -9,6 +9,7 @@ Memory-Resident DataBase for simple single-process asynchronous Python applicati
 
    motivation
    usage
+   structure
    guarantees
    ordering
    projects

docs/source/modules.rst

@@ -1,5 +1,5 @@
-ptvp35
-======
+Modules
+=======
 
 .. toctree::
    :maxdepth: 4

docs/source/motivation.rst

@@ -6,7 +6,7 @@ This page describes reasons for certain design decisions.
 General structure
 -----------------
 
-* We're making a key-value database.
+* A key-value database.
 * Keys and values are python objects.
 * While the DB is offline (after a correct shutdown), all its KVPs are stored as lines in one editable file.
 * DB should be able to survive powercut at any point (D in ACID).

docs/source/ordering.rst

@@ -4,17 +4,17 @@ Traced example of how ordering works in persistence5
 Source
 ------
 
-.. literalinclude :: ../../traced_example.py
+.. literalinclude :: ../scripts/traced_example.py
     :language: python3
 
 Writes/reads log
 ----------------
 
-.. literalinclude :: ../../traced_example.txt
+.. literalinclude :: ../scripts/traced_example.txt
     :language: plain
 
 Everything log
 --------------
 
-.. literalinclude :: ../../traced_example_all.txt
+.. literalinclude :: ../scripts/traced_example_all.txt
     :language: plain

docs/source/ptvp35.rst

@@ -1,5 +1,5 @@
-ptvp35 package
-==============
+ptvp35 (API Reference)
+======================
 
 Module contents
 ---------------

docs/source/structure.rst

@@ -0,0 +1,75 @@
+Inner structure
+===============
+
+Main-Memory DataBase
+--------------------
+
+* Represented via Python dictionary (:code:`dict`).
+    Future version may include more abstract options, i.e. :class:`~ptvp35.DbConnection` would be generic over :attr:`~ptvp35._DbConnection.__mmdb` field
+* Keys and values are almost guaranteed to be serializable.
+    All KVPs loaded on :meth:`~ptvp35.DbManager.__aenter__` are re-serialized during file reload.
+    All KVPs added via submit-like methods are serialized before being added to the MMDB.
+
+DataBase Stream File
+--------------------
+
+* In current implementation, all database storage storage files are Newline-Delimited JSON streams (https://en.wikipedia.org/wiki/JSON_streaming#Newline-Delimited_JSON).
+    .. code-block:: json
+
+        {"key": ["tuple", "example"], "value": {"dict": "example"}}
+        {"key": 123, "value": null}
+* During the runtime, the database uses 6 different files:
+    * `.db` Main file.
+        Should be the only non-error file after correct shutdown.
+    * `.db.backup` Backup file.
+        Generated when the main file is being rebuilt.
+    * `.db.recover` Flag file.
+        Indicates that backup file is valid and that main file's validity is undefined.
+    * `.db.truncate` Auxiliary file created on each write to main file.
+        Contains 16 bytes little-endian representation of up to how many characters the main file is valid.
+    * `.db.truncate_flag` Flag file.
+        Indicates that truncate file is valid and that main file's validity after the specified character count is undefined.
+    * `.db.error` Error log file.
+        In current implementation, it contains only the main file contents that got truncated on recovery.
+* All storage file writes are :code:`fsync`'ed.
+    * :code:`pathlib.Path.write_bytes` usecase relies on synchronisation/file-creation ordering. That may get replaced in future versions.
+
+Request Queue
+-------------
+
+Transaction View (:class:`~ptvp35.TransactionView`)
+--------------------------------------------------
+
+Connection-like interface on top of another connection-like interface.
+
+* Provides most of the same methods as :class:`~ptvp35.DbConnection`.
+    * From the common :class:`~ptvp35.VirtualConnection` interface/base class:
+        * :meth:`~ptvp35.TransactionView.get`
+        * :meth:`~ptvp35.TransactionView.commit_transaction`
+        * :meth:`~ptvp35.TransactionView.submit_transaction_request`
+        * :meth:`~ptvp35.TransactionView.loop`
+        * :meth:`~ptvp35.VirtualConnection.transaction` (default implementation)
+    * Extra common methods (:class:`~ptvp35.ExtendedVirtualConnection`):
+        * :meth:`~ptvp35.TransactionView.set_nowait`
+        * :meth:`~ptvp35.TransactionView.submit_transaction`
+        * :meth:`~ptvp35.TransactionView.commit`
+* Does not have the the analogue for the :meth:`~ptvp35.DbInterface.set` method.
+    * The reason for that is :code:`set` method having semantics contradictory to transactions.
+        * The :code:`set` provides a way to set a *single* value and wait until it's committed.
+        * Transactions are meant for a more fine control.
+    * The equivalent would consist of using a subtransaction or of the following method calls:
+        * :meth:`~ptvp35.TransactionView.set_nowait` to set the value in :code:`__delta`.
+        * :meth:`~ptvp35.TransactionView.submit` to pass all the :code:`__delta` values.
+        * :meth:`~ptvp35.TransactionView.future_context` to get the context for that specific key.
+        * :meth:`~ptvp35.FutureContext.wait` to wait until that key is committed.
+        * :meth:`~ptvp35.TransactionView.illuminate` to clear :code:`__shadow` thus resetting the view to DB state.
+
+Transaction (:class:`~ptvp35.Transaction`)
+-----------------------------------------
+
+Manages a Transaction View.
+
+* Creates and returns a Transaction View on :code:`__aenter__`/:code:`__enter__`.
+* Submits changes on successful :code:`__exit__`.
+* Commits changes on successful :code:`__aexit__`.
+* Rolls back changes on unsuccessful :code:`__exit__`/:code:`__aexit__`.

docs/source/usage.rst

@@ -13,12 +13,10 @@ Default installation option is to use pip+git
 Basic functionality
 -------------------
 
-.. autoclass:: ptvp35.DbFactory
-
-    :code:`DbFactory` class provides context management for database connections (:code:`DbConnection`) via :code:`async with` statement.
-    The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**.
-    Also, that means that each connection start/shutdown is quite time expensive.
-    These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block.
+:class:`~ptvp35.DbFactory` class provides context management for database connections (:class:`~ptvp35.DbConnection`) via :code:`async with` statement.
+The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**.
+Also, that means that each connection start/shutdown is quite time expensive.
+These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block.
 
 .. code-block:: python3
 
@@ -62,26 +60,4 @@ Different ways to get/set a value:
             transaction.set_nowait('increment-5', value5 + 1)
         await connection.commit()
 
-.. autoclass:: ptvp35.DbConnection
-
-    .. automethod:: get
-
-        this method is instant.
-
-    .. automethod:: set
-
-        this method may take time to run.
-        ordering may not be guaranteed (depends on event loop implementation).
-
-    .. automethod:: set_nowait
-
-        this method is instant.
-        ordering is guaranteed.
-
-    .. automethod:: commit
-
-        this method may take time to run.
-        respects the ordering of previously called :code:`set_nowait` methods.
-        will, under most circumstances, also execute later changes.
-
-    .. automethod:: transaction
+For common methods see: :class:`~ptvp35.AbstractDbConnection`

ptvp35/instrumentation.py

@@ -1,12 +1,12 @@
 import ptvp35
 from rainbowadn.instrument import Instrumentation
 
-__all__ = ('InstrumentDiskWrites', 'NightlyInstrumentation')
+__all__ = ("InstrumentDiskWrites",)
 
 
 class InstrumentDiskWrites(Instrumentation):
     def __init__(self, /):
-        super().__init__(ptvp35.DbConnection, '_write_to_disk_sync')
+        super().__init__(ptvp35._File, "write_to_disk_sync")
 
     def on_write(self, line: str, /) -> None:
         pass
@@ -14,12 +14,3 @@ class InstrumentDiskWrites(Instrumentation):
     def instrument(self, method, db, line, /):
         self.on_write(line)
         return method(db, line)
-
-
-class NightlyInstrumentation(Instrumentation):
-    def __init__(self, target, methodname: str):
-        method = getattr(target, methodname)
-        if hasattr(method, '__non_nightly__'):
-            target = method
-            methodname = '__non_nightly__'
-        super().__init__(target, methodname)

setup.py

@@ -1,12 +1,12 @@
 from setuptools import setup
 
 setup(
-    name='ptvp35',
-    version='1.1rc2',
-    packages=['ptvp35'],
-    url='https://gitea.ongoteam.net/PTV/ptvp35',
-    license='MIT',
-    author='PARRRATE TNV',
-    author_email='',
-    description='',
+    name="ptvp35",
+    version="1.1.0",
+    packages=["ptvp35"],
+    url="https://gitea.ongoteam.net/PTV/ptvp35",
+    license="MIT",
+    author="PARRRATE TNV",
+    author_email="",
+    description="",
 )

test_delete.py

@@ -0,0 +1,23 @@
+import asyncio
+import pathlib
+
+from ptvp35 import VDELETE, DbFactory, KVJson
+
+
+async def main():
+    path = pathlib.Path("test_delete.db")
+    path.unlink(missing_ok=True)
+    async with DbFactory(path, kvfactory=KVJson()) as connection:
+        connection.set_nowait(0, 0)
+        print(connection.get(0, 1))
+        await connection.commit()
+        async with connection.transaction() as transaction:
+            print(transaction.get(0, 1))
+            transaction.set_nowait(0, VDELETE)
+            print(transaction.get(0, 1))
+        input()
+        print(connection.get(0, 1))
+    # path.unlink(missing_ok=True)
+
+
+asyncio.run(main())

traced_example.py

@@ -1,286 +0,0 @@
-import asyncio
-import pathlib
-import sys
-import threading
-from contextlib import ExitStack
-
-from ptvp35 import *
-from ptvp35.instrumentation import *
-
-
-async def aprint(*args, **kwargs):
-    print(*args, **kwargs)
-
-
-class LogWrites(InstrumentDiskWrites):
-    def __init__(self, /):
-        super().__init__()
-        self.loop = asyncio.get_running_loop()
-
-    def on_write(self, line: str, /) -> None:
-        asyncio.run_coroutine_threadsafe(
-            aprint(f'{self.methodname}[{line}]'), self.loop
-        ).result()
-
-
-class LogEE(NightlyInstrumentation):
-    def __init__(self, target, methodname: str):
-        super().__init__(target, methodname)
-        self.loop = asyncio.get_running_loop()
-
-    def _target_id(self) -> str:
-        name = (
-            self.target.__name__
-            if
-            hasattr(self.target, '__name__')
-            else
-            self.target.__class__.__name__
-        )
-        return f'{name}.{self.methodname}'
-
-    def _print(self, thread, *args) -> None:
-        print(
-            thread,
-            self._target_id(),
-            *args,
-            sep='\t'
-        )
-
-    async def aprint(self, thread, *args) -> None:
-        self._print(thread, *args)
-
-    def print(self, *args) -> None:
-        if (ct := threading.current_thread()) is threading.main_thread():
-            self._print('main', *args)
-        else:
-            asyncio.run_coroutine_threadsafe(
-                self.aprint('aux', *args), self.loop
-            ).result()
-
-    def instrument(self, method, *args, **kwargs):
-        self.print('enter')
-        try:
-            result = method(*args, **kwargs)
-        except:
-            self.print('error')
-            raise
-        else:
-            self.print('exit')
-        return result
-
-
-class ALogEE(LogEE):
-    async def instrument(self, method, *args, **kwargs):
-        self._print('aio', 'enter')
-        try:
-            result = await method(*args, **kwargs)
-        except:
-            self._print('aio', 'error')
-            raise
-        else:
-            self._print('aio', 'exit')
-        return result
-
-
-async def transaction_test(db: DbConnection):
-    def logdb(*args):
-        if args:
-            args = (' ', ' ', '@',) + args
-        print(db.get('test', '0'), *args, sep='\t')
-
-    def logstate(*args):
-        if args:
-            args = ('@',) + args
-        print(db.get('test', '0'), '|', state.get('test', '0'), *args, sep='\t')
-
-    logdb('empty db')
-    db.set_nowait('test', '1')
-    logdb('after set_nowait')
-    await db.set('test', '2')
-    logdb('after set')
-    try:
-        async with db.transaction() as state:
-            logstate('empty transaction')
-            state.set_nowait('test', '3')
-            logstate('after transaction.set_nowait')
-            state.submit()
-            logstate('after transaction.submit')
-            await state.commit()
-            logstate('after transaction.commit')
-            state.set_nowait('test', print)  # will throw TypeError later
-            logstate()
-    except TypeError:
-        print('type error')
-    logdb('after transaction')
-    async with db.transaction() as state:
-        logstate()
-        state.set_nowait('test', '4')
-        logstate('before implicit transaction.commit')
-    logdb('after transaction with implicit commit')
-    with db.transaction() as state:
-        logstate()
-        state.set_nowait('test', '5')
-        logstate('before implicit transaction.submit')
-    logdb('after transaction with implicit submit')
-
-
-def print_private_db_attrs(db: DbConnection):
-    if run_all:
-        for attr in dir(db):
-            if attr.startswith('_DbConnection') and hasattr(db, attr):
-                print(attr)
-
-
-run_all = 'all' in sys.argv
-
-
-async def main():
-    (path := pathlib.Path('dev.db')).unlink(missing_ok=True)
-
-    with ExitStack() as es:
-        LogWrites().enter(es)
-        if run_all:
-            LogEE(__import__('ptvp35').Request, '__init__').enter(es)
-            LogEE(__import__('ptvp35').Request, 'waiting').enter(es)
-            LogEE(__import__('ptvp35').Request, 'set_result').enter(es)
-            LogEE(__import__('ptvp35').Request, 'set_exception').enter(es)
-            ALogEE(__import__('ptvp35').Request, 'wait').enter(es)
-
-            LogEE(__import__('ptvp35').LineRequest, '__init__').enter(es)
-
-            LogEE(__import__('ptvp35').KVFactory, 'run').enter(es)
-            LogEE(__import__('ptvp35').KVFactory, 'request').enter(es)
-            LogEE(__import__('ptvp35').KVFactory, 'free').enter(es)
-            LogEE(__import__('ptvp35').KVFactory, 'io2db').enter(es)
-            LogEE(__import__('ptvp35').KVFactory, 'db2io').enter(es)
-
-            LogEE(__import__('ptvp35').KVRequest, '__init__').enter(es)
-
-            LogEE(__import__('ptvp35').KVJson, 'line').enter(es)
-            LogEE(__import__('ptvp35').KVJson, '_load_key').enter(es)
-            LogEE(__import__('ptvp35').KVJson, 'fromline').enter(es)
-
-            LogEE(__import__('ptvp35').TransactionRequest, '__init__').enter(es)
-
-            LogEE(__import__('ptvp35').DbParametres, '__init__').enter(es)
-
-            LogEE(__import__('ptvp35').VirtualConnection, 'transaction').enter(es)
-
-            LogEE(__import__('ptvp35').DbConnection, '__init__').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_create_future').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_save_error_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_save_error').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_queue_error').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_save_error_from_thread').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_path2db_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_db2path_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'get').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, 'set').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'set_nowait').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_clear_buffer').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_compress_buffer').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_commit_compressed_buffer_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_commit_compressed_buffer').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_satisfy_buffer_future').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_fail_buffer_future').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_do_commit_buffer').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_commit_buffer').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_request_buffer').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_commit_buffer_or_request_so').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_write').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_truncation_set_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_truncation_unset_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_file_truncate_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_truncation_target_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_truncate_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_assure_truncation_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_write_to_disk_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_file_write_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_reload_if_oversized').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_handle_request').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_background_cycle').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_background_task').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_start_task').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_recovery_set_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_recovery_unset_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_copy_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_finish_recovery_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_build_file_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_run_in_thread').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_rebuild_file_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_file_open_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_file_close_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_reload_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_reload').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_load_mmdb_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_initialize_mmdb_sync').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_load_from_file_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_load_from_file').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_initialize_queue').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_initialize_running').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_initialize').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_close_buffer').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_close_queue').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_close_mmdb_sync').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_close_mmdb').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, '_close_running').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, 'aclose').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, '_transaction_buffer').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, 'commit_transaction').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'submit_transaction').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'submit_transaction_request').enter(es)
-            ALogEE(__import__('ptvp35').DbConnection, 'commit').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'loop').enter(es)
-            LogEE(__import__('ptvp35').DbConnection, 'transaction').enter(es)
-
-            LogEE(__import__('ptvp35').DbFactory, '__init__').enter(es)
-            ALogEE(__import__('ptvp35').DbFactory, '__aenter__').enter(es)
-            ALogEE(__import__('ptvp35').DbFactory, '__aexit__').enter(es)
-
-            LogEE(__import__('ptvp35').Db, '__init__').enter(es)
-            ALogEE(__import__('ptvp35').Db, '__aenter__').enter(es)
-            ALogEE(__import__('ptvp35').Db, '__aexit__').enter(es)
-
-            LogEE(__import__('ptvp35').FutureContext, '__init__').enter(es)
-            ALogEE(__import__('ptvp35').FutureContext, '__aenter__').enter(es)
-            ALogEE(__import__('ptvp35').FutureContext, '__aexit__').enter(es)
-
-            LogEE(__import__('ptvp35').TransactionView, '__init__').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'future_context').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'rollback').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'illuminate').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'ailluminate').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'fork').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'afork').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'clear').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'aclear').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'reset').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'areset').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'get').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'set_nowait').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, '_delta').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'commit').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'submit').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, '_do_gather').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, '_reduce_future').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, '_gather').enter(es)
-            ALogEE(__import__('ptvp35').TransactionView, 'commit_transaction').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'submit_transaction').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'submit_transaction_request').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'loop').enter(es)
-            LogEE(__import__('ptvp35').TransactionView, 'transaction').enter(es)
-
-            LogEE(__import__('ptvp35').Transaction, '__init__').enter(es)
-            ALogEE(__import__('ptvp35').Transaction, '__aenter__').enter(es)
-            ALogEE(__import__('ptvp35').Transaction, '__aexit__').enter(es)
-            LogEE(__import__('ptvp35').Transaction, '_clean').enter(es)
-            LogEE(__import__('ptvp35').Transaction, '__enter__').enter(es)
-            LogEE(__import__('ptvp35').Transaction, '__exit__').enter(es)
-        async with DbFactory(path, kvfactory=KVJson()) as db:
-            await transaction_test(db)
-            print_private_db_attrs(db)
-        print_private_db_attrs(db)
-
-
-if __name__ == '__main__':
-    asyncio.run(main())