formatting

+ move io2db/db2io to kvfactory

.dockerignore

@@ -0,0 +1,4 @@
+.git*
+__pycache__
+*.egg-info
+build

.gitignore

@@ -222,4 +222,5 @@ cython_debug/
 
 # Other
 /dev.py
-/*.db
+*.db
+*.db.*

.idea/.gitignore

@@ -1,8 +0,0 @@
-# Default ignored files
-/shelf/
-/workspace.xml
-# Datasource local storage ignored files
-/dataSources/
-/dataSources.local.xml
-# Editor-based HTTP Client requests
-/httpRequests/

.idea/inspectionProfiles/Project_Default.xml

@@ -1,51 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="DuplicatedCode" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
-    <inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="myValues">
-        <value>
-          <list size="11">
-            <item index="0" class="java.lang.String" itemvalue="nobr" />
-            <item index="1" class="java.lang.String" itemvalue="noembed" />
-            <item index="2" class="java.lang.String" itemvalue="comment" />
-            <item index="3" class="java.lang.String" itemvalue="noscript" />
-            <item index="4" class="java.lang.String" itemvalue="embed" />
-            <item index="5" class="java.lang.String" itemvalue="script" />
-            <item index="6" class="java.lang.String" itemvalue="markdown" />
-            <item index="7" class="java.lang.String" itemvalue="sv3i" />
-            <item index="8" class="java.lang.String" itemvalue="sv3o" />
-            <item index="9" class="java.lang.String" itemvalue="sv3a" />
-            <item index="10" class="java.lang.String" itemvalue="sv3c" />
-          </list>
-        </value>
-      </option>
-      <option name="myCustomValuesEnabled" value="true" />
-    </inspection_tool>
-    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ignoredPackages">
-        <value>
-          <list size="1">
-            <item index="0" class="java.lang.String" itemvalue="nacl" />
-          </list>
-        </value>
-      </option>
-    </inspection_tool>
-    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ignoredIdentifiers">
-        <list>
-          <option value="PySide2.QtWidgets.clicked.connect" />
-          <option value="PySide2.QtWidgets.valueChanged.connect" />
-          <option value="PySide2.QtWidgets.textChanged.connect" />
-          <option value="PySide2.QtCore.Signal.emit" />
-          <option value="PySide2.QtCore.Signal.connect" />
-        </list>
-      </option>
-    </inspection_tool>
-    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
-      <option name="processCode" value="true" />
-      <option name="processLiterals" value="true" />
-      <option name="processComments" value="true" />
-    </inspection_tool>
-  </profile>
-</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -1,6 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>

.idea/misc.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (ptvp35)" project-jdk-type="Python SDK" />
-</project>

.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/ptvp35.iml" filepath="$PROJECT_DIR$/.idea/ptvp35.iml" />
-    </modules>
-  </component>
-</project>

.idea/ptvp35.iml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$">
-      <excludeFolder url="file://$MODULE_DIR$/venv" />
-    </content>
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

Dockerfile

@@ -0,0 +1,71 @@
+# syntax=docker/dockerfile:1
+
+FROM python:3.10 as compile-sphinx5.3.0-base
+
+RUN apt-get update
+RUN apt-get install -y python3-sphinx
+RUN pip install Sphinx==5.3.0 pydata-sphinx-theme==0.12.0
+WORKDIR /app/
+RUN pip install git+https://gitea.parrrate.ru/PTV/rainbowadn.git@e9fba7b064902ceedee0dd5578cb47030665a6aa
+
+RUN mkdir /app/docs/
+
+
+FROM compile-sphinx5.3.0-base as compile-legacy
+
+WORKDIR /app/legacy/
+RUN git clone --branch 1.0 https://gitea.parrrate.ru/PTV/ptvp35.git
+WORKDIR /app/legacy/ptvp35/
+
+
+FROM compile-legacy as compile-1.0
+
+RUN python traced_example.py > traced_example.txt
+RUN python traced_example.py all > traced_example_all.txt
+RUN cp -r docs/source/ ./source/
+RUN cp docs/Makefile ./Makefile
+RUN make html
+
+
+FROM compile-legacy as compile-1.1rc0
+
+RUN git fetch && git checkout 1.1rc0
+WORKDIR /app/legacy/ptvp35/docs/
+RUN make html
+
+
+FROM compile-legacy as compile-1.1rc2
+
+RUN git fetch && git checkout 1.1rc2
+WORKDIR /app/legacy/ptvp35/docs/
+RUN make html
+
+
+FROM compile-legacy as compile-1.1.0
+
+RUN git fetch && git checkout 1.1.0
+WORKDIR /app/legacy/ptvp35/docs/
+RUN make html
+
+
+FROM compile-sphinx5.3.0-base as compile-latest
+
+COPY docs/Makefile docs/Makefile
+COPY setup.py setup.py
+COPY docs/scripts docs/scripts
+COPY ptvp35 ptvp35
+COPY docs/source docs/source
+WORKDIR /app/docs/
+RUN make html
+
+
+FROM node:19
+
+RUN npm install -g http-server
+WORKDIR /app/docs/build/html/
+COPY --from=compile-1.0 /app/legacy/ptvp35/build/html/ /app/docs/build/html/1.0/
+COPY --from=compile-1.1rc0 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1rc0/
+COPY --from=compile-1.1rc2 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1rc2/
+COPY --from=compile-1.1.0 /app/legacy/ptvp35/docs/build/html/ /app/docs/build/html/1.1.0/
+COPY --from=compile-latest /app/docs/build/html/ /app/docs/build/html/
+CMD [ "http-server", "-p", "80" ]

docs/Makefile

@@ -0,0 +1,22 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	python scripts/traced_example.py > scripts/traced_example.txt
+	python scripts/traced_example.py all > scripts/traced_example_all.txt
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/scripts/traced_example.py

@@ -0,0 +1,320 @@
+import asyncio
+import pathlib
+import sys
+import threading
+from contextlib import ExitStack
+
+from rainbowadn.instrument import Instrumentation
+
+try:
+    sys.path.append(str((pathlib.Path(__file__).parent / "../..").absolute()))
+    from ptvp35 import DbConnection, DbFactory, KVJson
+    from ptvp35.instrumentation import InstrumentDiskWrites
+except:
+    raise
+
+
+async def aprint(*args, **kwargs):
+    print(*args, **kwargs)
+
+
+class LogWrites(InstrumentDiskWrites):
+    def __init__(self, /):
+        super().__init__()
+        self.loop = asyncio.get_running_loop()
+
+    def on_write(self, line: str, /) -> None:
+        asyncio.run_coroutine_threadsafe(aprint(f"{self.methodname}[{line}]"), self.loop).result()
+
+
+class LogEE(Instrumentation):
+    def __init__(self, target, methodname: str):
+        super().__init__(target, methodname)
+        self.loop = asyncio.get_running_loop()
+
+    def _target_id(self) -> str:
+        name = self.target.__name__ if hasattr(self.target, "__name__") else self.target.__class__.__name__
+        return f"{name}.{self.methodname}"
+
+    def _print(self, thread, *args) -> None:
+        print(thread, self._target_id(), *args, sep="\t")
+
+    async def aprint(self, thread, *args) -> None:
+        self._print(thread, *args)
+
+    def print(self, *args) -> None:
+        if (ct := threading.current_thread()) is threading.main_thread():
+            self._print("main", *args)
+        else:
+            asyncio.run_coroutine_threadsafe(self.aprint("aux", *args), self.loop).result()
+
+    def instrument(self, method, *args, **kwargs):
+        self.print("enter")
+        try:
+            result = method(*args, **kwargs)
+        except:
+            self.print("error")
+            raise
+        else:
+            self.print("exit")
+        return result
+
+
+class ALogEE(LogEE):
+    async def instrument(self, method, *args, **kwargs):
+        self._print("aio", "enter")
+        try:
+            result = await method(*args, **kwargs)
+        except:
+            self._print("aio", "error")
+            raise
+        else:
+            self._print("aio", "exit")
+        return result
+
+
+async def transaction_test(db: DbConnection):
+    def logdb(*args):
+        if args:
+            args = (
+                " ",
+                " ",
+                "@",
+            ) + args
+        print(db.get("test", "0"), *args, sep="\t")
+
+    def logstate(*args):
+        if args:
+            args = ("@",) + args
+        print(db.get("test", "0"), "|", state.get("test", "0"), *args, sep="\t")
+
+    logdb("empty db")
+    db.set_nowait("test", "1")
+    logdb("after set_nowait")
+    await db.set("test", "2")
+    logdb("after set")
+    try:
+        async with db.transaction() as state:
+            logstate("empty transaction")
+            state.set_nowait("test", "3")
+            logstate("after transaction.set_nowait")
+            state.submit()
+            logstate("after transaction.submit")
+            await state.commit()
+            logstate("after transaction.commit")
+            state.set_nowait("test", print)  # will throw TypeError later
+            logstate()
+    except TypeError:
+        print("type error")
+    logdb("after transaction")
+    async with db.transaction() as state:
+        logstate()
+        state.set_nowait("test", "4")
+        logstate("before implicit transaction.commit")
+    logdb("after transaction with implicit commit")
+    with db.transaction() as state:
+        logstate()
+        state.set_nowait("test", "5")
+        logstate("before implicit transaction.submit")
+    logdb("after transaction with implicit submit")
+
+
+def print_private_db_attrs(db: DbConnection):
+    if run_all:
+        for attr in dir(db):
+            if attr.startswith("_DbConnection") and hasattr(db, attr):
+                print(attr)
+
+
+run_all = "all" in sys.argv
+
+
+async def main():
+    (path := pathlib.Path(__file__).parent / "trace_example.db").unlink(missing_ok=True)
+
+    with ExitStack() as es:
+        LogWrites().enter(es)
+        if run_all:
+            LogEE(__import__("ptvp35").Request, "__init__").enter(es)
+            LogEE(__import__("ptvp35").Request, "waiting").enter(es)
+            LogEE(__import__("ptvp35").Request, "set_result").enter(es)
+            LogEE(__import__("ptvp35").Request, "set_exception").enter(es)
+            ALogEE(__import__("ptvp35").Request, "wait").enter(es)
+
+            LogEE(__import__("ptvp35").LineRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").KVFactory, "run").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "_dbset").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "dbset").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "dbget").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "filter_value").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "request").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "free").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "io2db").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "db2io").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "path2db_sync").enter(es)
+            LogEE(__import__("ptvp35").KVFactory, "db2path_sync").enter(es)
+
+            LogEE(__import__("ptvp35").KVRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").KVJson, "line").enter(es)
+            LogEE(__import__("ptvp35").KVJson, "_load_key").enter(es)
+            LogEE(__import__("ptvp35").KVJson, "fromline").enter(es)
+
+            LogEE(__import__("ptvp35").TransactionRequest, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").DbParameters, "__init__").enter(es)
+
+            LogEE(__import__("ptvp35").VirtualConnection, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35")._Loop, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "create_future").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "loop").enter(es)
+            LogEE(__import__("ptvp35")._Loop, "run_in_thread").enter(es)
+
+            LogEE(__import__("ptvp35")._Errors, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Errors, "_save_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Errors, "_save").enter(es)
+            LogEE(__import__("ptvp35")._Errors, "save_from_thread").enter(es)
+
+            LogEE(__import__("ptvp35")._File, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._File, "path").enter(es)
+            LogEE(__import__("ptvp35")._File, "tell").enter(es)
+            LogEE(__import__("ptvp35")._File, "write_to_disk_sync").enter(es)
+            LogEE(__import__("ptvp35")._File, "open_sync").enter(es)
+            LogEE(__import__("ptvp35")._File, "close_sync").enter(es)
+
+            LogEE(__import__("ptvp35")._Backup, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "file").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "kvfactory").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_copy_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_recovery_unset_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_finish_recovery_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_recovery_set_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "build_file_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_rebuild_file_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "_reload_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "run_in_thread").enter(es)
+            ALogEE(__import__("ptvp35")._Backup, "_reload").enter(es)
+            ALogEE(__import__("ptvp35")._Backup, "reload_if_oversized").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "load_mmdb_sync").enter(es)
+            LogEE(__import__("ptvp35")._Backup, "uninitialize").enter(es)
+
+            LogEE(__import__("ptvp35")._Guard, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "backup").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_write_bytes_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_write_value_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_set_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_unset_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_read_bytes_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_read_value_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_truncate_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "assure_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "_file_truncate_sync").enter(es)
+            LogEE(__import__("ptvp35")._Guard, "file_write_sync").enter(es)
+
+            LogEE(__import__("ptvp35")._ReceivingQueue, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._ReceivingQueue, "submit").enter(es)
+
+            LogEE(__import__("ptvp35")._WriteableBuffer, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "writeable").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "loop").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_compressed").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_commit_compressed_sync").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit_compressed").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_clear").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_satisfy_future").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_fail_future").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_do_commit_buffer").enter(es)
+            LogEE(__import__("ptvp35")._WriteableBuffer, "_request_buffer").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_commit_or_request_so").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_write").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_handle_request").enter(es)
+            ALogEE(__import__("ptvp35")._WriteableBuffer, "_close").enter(es)
+
+            LogEE(__import__("ptvp35")._Memory, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_initialize_sync").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_load_from_file_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Memory, "_load_from_file").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_close_sync").enter(es)
+            ALogEE(__import__("ptvp35")._Memory, "_close").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "_transaction_buffer").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "get").enter(es)
+            LogEE(__import__("ptvp35")._Memory, "set").enter(es)
+
+            LogEE(__import__("ptvp35")._QueueTask, "__init__").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "_background_cycle").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "_background_task").enter(es)
+            ALogEE(__import__("ptvp35")._QueueTask, "close").enter(es)
+            LogEE(__import__("ptvp35")._QueueTask, "start").enter(es)
+
+            LogEE(__import__("ptvp35")._DbConnection, "__init__").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "kvprotocol").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "get").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "set").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "set_nowait").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_initialize_running").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_initialize").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "create").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "_close_running").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "aclose").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "commit_transaction").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "submit_transaction").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "submit_transaction_request").enter(es)
+            ALogEE(__import__("ptvp35")._DbConnection, "commit").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "loop").enter(es)
+            LogEE(__import__("ptvp35")._DbConnection, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35").DbManager, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").DbManager, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").DbManager, "__aexit__").enter(es)
+
+            LogEE(__import__("ptvp35").Db, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").Db, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").Db, "__aexit__").enter(es)
+
+            LogEE(__import__("ptvp35").FutureContext, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "__aexit__").enter(es)
+            ALogEE(__import__("ptvp35").FutureContext, "wait").enter(es)
+
+            LogEE(__import__("ptvp35").TransactionView, "__init__").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "future_context").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "rollback").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "illuminate").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "ailluminate").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "fork").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "afork").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "clear").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "aclear").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "reset").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "areset").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "get").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "set_nowait").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_delta").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "commit").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_do_gather").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_reduce_future").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "_gather").enter(es)
+            ALogEE(__import__("ptvp35").TransactionView, "commit_transaction").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit_transaction").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "submit_transaction_request").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "loop").enter(es)
+            LogEE(__import__("ptvp35").TransactionView, "transaction").enter(es)
+
+            LogEE(__import__("ptvp35").Transaction, "__init__").enter(es)
+            ALogEE(__import__("ptvp35").Transaction, "__aenter__").enter(es)
+            ALogEE(__import__("ptvp35").Transaction, "__aexit__").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "_clean").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "__enter__").enter(es)
+            LogEE(__import__("ptvp35").Transaction, "__exit__").enter(es)
+        async with DbFactory(path, kvfactory=KVJson()) as db:
+            await transaction_test(db)
+            print_private_db_attrs(db)
+        print_private_db_attrs(db)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

docs/source/conf.py

@@ -0,0 +1,42 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+import os.path
+import sys
+
+project = "ptvp35"
+copyright = "2022, PARRRATE TNV"
+author = "PARRRATE TNV"
+with open("../../setup.py") as f:
+    _src = f.read()
+    _src = _src[_src.index('version="') + 9 :]
+    _src = _src[: _src.index('"')]
+    release = _src
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "sphinx.ext.autodoc",
+]
+
+templates_path = ["_templates"]
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "pydata_sphinx_theme"
+html_theme_options = {
+    "navbar_center": [],
+}
+html_static_path = ["_static"]
+
+
+sys.path.insert(0, os.path.abspath("../.."))

docs/source/development.rst

@@ -0,0 +1,22 @@
+Current core development values
+===============================
+
+* Target :code:`async` only.
+* Store the database file as a single human-editable file.
+* Keep the core module a single minimalistic file.
+* Provide for dynamic instrumentation.
+
+Guidelines for developers of Persistence 5
+==========================================
+
+These apply both to the internal PARRRATE TNV team behind Persistence 5 and to external contributors.
+
+* Before contributing a feature to Persistence 5, the following should be considered:
+    * Addition of a simple extensible/flexible support for the further integration of external features is preferred over adding features themselves.
+    * Addition of features as extensions is preferred over adding features to core codebase.
+    * Maintaining extensions as separate projects is preferred over adding them to the core repository.
+    * Forking of the repository is encouraged.
+    * Usage of the repository as a reference for custom data storage solutions is encouraged.
+* Instrumentation code base is more open to direct code contributions.
+    * Main way of instrumentation is code injection.
+    * Instrumentation on its own shouldn't require changes to the core code, even though instrumentation-allowing core code is preferred.

docs/source/guarantees.rst

@@ -0,0 +1,36 @@
+Guarantees
+==========
+
+MMDB-level guarantees
+---------------
+
+* All 0L writes change MMDB instantly.
+* Transaction write works as one atomic write.
+
+Queue-level guarantees
+----------------------
+
+This level is the mediator between MMDB and filesystem levels. This level includes the pre-write compression buffer.
+
+* Queue requests occur in the same orders as they were acted upon at MMDB level and in the same order as they will be acted upon at filesystem level.
+* DB can't close until all requests are done.
+* No request can crash the task.
+* Every request eventually succeedes or fails (except for the case of runtime termination).
+
+Buffer-specific guarantees:
+
+* If buffer dump was ever requested, then this request (queued or indirect) will eventually be satisfied.
+
+Filesystem-level guarantees
+---------------------------
+
+* If main file exists without .recover or .truncate_flag, then it's valid.
+* If .recover file is present, then .backup is valid.
+* If .truncate_flag is present, then .truncate is valid and first :code:`.truncate_target()` (contents of .truncate) characters of main file are valid.
+* Every write is final and irreversible (can't be reversed unintentionally due to termination), otherwise it's not considered done. That's achieved using :code:`os.fsync`.
+
+Performance guarantees
+----------------------
+
+* No normal sync methods (except for :code:`io2db`/:code:`db2io` if they're supplied with blocking :code:`IO`) block on IO. Other methods are explicitly marked with postfix :code:`_sync`.
+* All requests are resolved as soon as their conditions are met.

docs/source/history.rst

@@ -0,0 +1,55 @@
+Historical notes
+================
+
+Persistence 1/2. CmbPrst.
+--------------
+
+* Internal storage of CMB.
+
+Persistence 3. ptvp3.
+---------------------
+
+* Cluster of backed-up-on-disk dictionaries.
+* Used by CBMB.
+
+Persistence 4. ptvp3.4. ShelveSQLite.
+-------------------------------------
+
+* SQLite-based analogue of :code:`shelve` module.
+* Used by v6x12 implementation of CBMB.
+
+Persistence 5. ptvp3.5.
+-----------------------
+
+* First :code:`async` database in the line.
+* File stream storage.
+
+Persistence 5 early release candidates.
+--------------------------------------
+
+* Prevents database corruption.
+
+Persistence 5 1.0 (5.1.0).
+-------------------------
+
+* ACID compliant.
+* Reduced or none blocking code.
+
+Persistence 5 1.1 (5.1.1).
+-------------------------
+
+* Non-nightly support for transactions.
+* Instrumentation support.
+* Structural preparations for generic DBs.
+* :code:`VDELETE`
+
+Proposed future versions
+========================
+
+Persistence 5 1.2.
+------------------
+* More abstract concepts (expansion of :code:`VirtualConnection` and :code:`LineRequest`).
+
+Persistence 5 2.0.
+------------------
+* Any future breaking changes are going to be listed here.

docs/source/index.rst

@@ -0,0 +1,27 @@
+Welcome to Persistence 5
+========================
+
+Memory-Resident DataBase for simple single-process asynchronous Python applications.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   motivation
+   usage
+   structure
+   guarantees
+   ordering
+   projects
+   history
+   development
+   modules
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/modules.rst

@@ -0,0 +1,7 @@
+Modules
+=======
+
+.. toctree::
+   :maxdepth: 4
+
+   ptvp35

docs/source/motivation.rst

@@ -0,0 +1,30 @@
+What is Persistence 5 for?
+==========================
+
+This page describes reasons for certain design decisions.
+
+General structure
+-----------------
+
+* A key-value database.
+* Keys and values are python objects.
+* While the DB is offline (after a correct shutdown), all its KVPs are stored as lines in one editable file.
+* DB should be able to survive powercut at any point (D in ACID).
+
+AP/LP get
+------
+
+The central idea for this database is the zero-latency (as far as python goes) reads. Therefore, the DB has all its data stored as a :code:`dict`.
+
+CAP/CLP set
+-------
+
+The database allows both availability-(zero)latency and consistency (note: it's not ACID consistency, it's CAP consistency and ACID durability) variants for its writes.
+All zero-latency writes are also consistent with respect to (zero-latency) reads.
+
+Consistent get
+------
+
+Later versions of the database will include remote connections, and thus also consistent reads.
+These will probably be implemented as requests (same as writes).
+Some connection variants may not even include availability reads/writes.

docs/source/ordering.rst

@@ -0,0 +1,20 @@
+Traced example of how ordering works in persistence5
+====================================================
+
+Source
+------
+
+.. literalinclude :: ../scripts/traced_example.py
+    :language: python3
+
+Writes/reads log
+----------------
+
+.. literalinclude :: ../scripts/traced_example.txt
+    :language: plain
+
+Everything log
+--------------
+
+.. literalinclude :: ../scripts/traced_example_all.txt
+    :language: plain

docs/source/projects.rst

@@ -0,0 +1,27 @@
+Projects that use persistence5
+==============================
+
+parrrate-music
+--------------
+
+Music bot by PARRRATE TNV.
+
+Uses `cache.db`, `effects.db`, `queue.db`, `session.db`, `volume.db`.
+
+https://gitea.parrrate.ru/PTV/v6d3music
+
+ADaaS -- Audio Download as a Service
+------------------------------------
+
+Standalone variant of parrrate-music's url resolution.
+
+Uses `cache.db`.
+
+https://gitea.parrrate.ru/PTV/adaas.git
+
+bad apple on a database file
+----------------------------
+
+Demonstrates how stable and predictable the inner structure of persistence5 DB file is.
+
+https://youtu.be/csLmTD8UrDg

docs/source/ptvp35.rst

@@ -0,0 +1,10 @@
+ptvp35 (API Reference)
+======================
+
+Module contents
+---------------
+
+.. automodule:: ptvp35
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/structure.rst

@@ -0,0 +1,75 @@
+Inner structure
+===============
+
+Main-Memory DataBase
+--------------------
+
+* Represented via Python dictionary (:code:`dict`).
+    Future version may include more abstract options, i.e. :class:`~ptvp35.DbConnection` would be generic over :attr:`~ptvp35._DbConnection.__mmdb` field
+* Keys and values are almost guaranteed to be serializable.
+    All KVPs loaded on :meth:`~ptvp35.DbManager.__aenter__` are re-serialized during file reload.
+    All KVPs added via submit-like methods are serialized before being added to the MMDB.
+
+DataBase Stream File
+--------------------
+
+* In current implementation, all database storage storage files are Newline-Delimited JSON streams (https://en.wikipedia.org/wiki/JSON_streaming#Newline-Delimited_JSON).
+    .. code-block:: json
+
+        {"key": ["tuple", "example"], "value": {"dict": "example"}}
+        {"key": 123, "value": null}
+* During the runtime, the database uses 6 different files:
+    * `.db` Main file.
+        Should be the only non-error file after correct shutdown.
+    * `.db.backup` Backup file.
+        Generated when the main file is being rebuilt.
+    * `.db.recover` Flag file.
+        Indicates that backup file is valid and that main file's validity is undefined.
+    * `.db.truncate` Auxiliary file created on each write to main file.
+        Contains 16 bytes little-endian representation of up to how many characters the main file is valid.
+    * `.db.truncate_flag` Flag file.
+        Indicates that truncate file is valid and that main file's validity after the specified character count is undefined.
+    * `.db.error` Error log file.
+        In current implementation, it contains only the main file contents that got truncated on recovery.
+* All storage file writes are :code:`fsync`'ed.
+    * :code:`pathlib.Path.write_bytes` usecase relies on synchronisation/file-creation ordering. That may get replaced in future versions.
+
+Request Queue
+-------------
+
+Transaction View (:class:`~ptvp35.TransactionView`)
+--------------------------------------------------
+
+Connection-like interface on top of another connection-like interface.
+
+* Provides most of the same methods as :class:`~ptvp35.DbConnection`.
+    * From the common :class:`~ptvp35.VirtualConnection` interface/base class:
+        * :meth:`~ptvp35.TransactionView.get`
+        * :meth:`~ptvp35.TransactionView.commit_transaction`
+        * :meth:`~ptvp35.TransactionView.submit_transaction_request`
+        * :meth:`~ptvp35.TransactionView.loop`
+        * :meth:`~ptvp35.VirtualConnection.transaction` (default implementation)
+    * Extra common methods (:class:`~ptvp35.ExtendedVirtualConnection`):
+        * :meth:`~ptvp35.TransactionView.set_nowait`
+        * :meth:`~ptvp35.TransactionView.submit_transaction`
+        * :meth:`~ptvp35.TransactionView.commit`
+* Does not have the the analogue for the :meth:`~ptvp35.DbInterface.set` method.
+    * The reason for that is :code:`set` method having semantics contradictory to transactions.
+        * The :code:`set` provides a way to set a *single* value and wait until it's committed.
+        * Transactions are meant for a more fine control.
+    * The equivalent would consist of using a subtransaction or of the following method calls:
+        * :meth:`~ptvp35.TransactionView.set_nowait` to set the value in :code:`__delta`.
+        * :meth:`~ptvp35.TransactionView.submit` to pass all the :code:`__delta` values.
+        * :meth:`~ptvp35.TransactionView.future_context` to get the context for that specific key.
+        * :meth:`~ptvp35.FutureContext.wait` to wait until that key is committed.
+        * :meth:`~ptvp35.TransactionView.illuminate` to clear :code:`__shadow` thus resetting the view to DB state.
+
+Transaction (:class:`~ptvp35.Transaction`)
+-----------------------------------------
+
+Manages a Transaction View.
+
+* Creates and returns a Transaction View on :code:`__aenter__`/:code:`__enter__`.
+* Submits changes on successful :code:`__exit__`.
+* Commits changes on successful :code:`__aexit__`.
+* Rolls back changes on unsuccessful :code:`__exit__`/:code:`__aexit__`.

docs/source/usage.rst

@@ -0,0 +1,63 @@
+Usage
+=====
+
+Installation
+------------
+
+Default installation option is to use pip+git
+
+.. code-block:: console
+
+    (venv) $ pip install git+https://gitea.parrrate.ru/PTV/ptvp35.git
+
+Basic functionality
+-------------------
+
+:class:`~ptvp35.DbFactory` class provides context management for database connections (:class:`~ptvp35.DbConnection`) via :code:`async with` statement.
+The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**.
+Also, that means that each connection start/shutdown is quite time expensive.
+These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block.
+
+.. code-block:: python3
+
+    import pathlib
+    from ptvp35 import DbFactory, KVJson
+
+    async def main():
+        async with DbFactory(pathlib.Path('example.db'), kvfactory=KVJson()) as connection:
+            await _main(connection)
+
+Different ways to get/set a value:
+
+.. code-block:: python3
+
+    from ptvp35 import DbConnection
+
+    async def _main(connection: DbConnection):
+        value0 = connection.get('increment-0', 0)
+        await connection.set('increment-0', value0 + 1)
+
+        value1 = connection.get('increment-1', 0)
+        connection.set_nowait('increment-1', value1 + 1)
+        await connection.commit()
+
+        async with connection.transaction() as transaction:
+            value2 = transaction.get('increment-2', 0)
+            transaction.set_nowait('increment-2', value2 + 1)
+
+        async with connection.transaction() as transaction:
+            value3 = transaction.get('increment-3', 0)
+            transaction.set_nowait('increment-3', value3 + 1)
+            await transaction.commit()
+
+        with connection.transaction() as transaction:
+            value4 = transaction.get('increment-4', 0)
+            transaction.set_nowait('increment-4', value4 + 1)
+            await transaction.commit()
+
+        with connection.transaction() as transaction:
+            value5 = transaction.get('increment-5', 0)
+            transaction.set_nowait('increment-5', value5 + 1)
+        await connection.commit()
+
+For common methods see: :class:`~ptvp35.AbstractDbConnection`

ptvp35/instrumentation.py

@@ -0,0 +1,16 @@
+import ptvp35
+from rainbowadn.instrument import Instrumentation
+
+__all__ = ("InstrumentDiskWrites",)
+
+
+class InstrumentDiskWrites(Instrumentation):
+    def __init__(self, /):
+        super().__init__(ptvp35._File, "write_to_disk_sync")
+
+    def on_write(self, line: str, /) -> None:
+        pass
+
+    def instrument(self, method, db, line, /):
+        self.on_write(line)
+        return method(db, line)

setup.py

@@ -1,12 +1,12 @@
 from setuptools import setup
 
 setup(
-    name='ptvp35',
+    name="ptvp35",
-    version='',
+    version="1.1.0",
-    packages=['ptvp35'],
+    packages=["ptvp35"],
-    url='https://gitea.ongoteam.net/PTV/ptvp35',
+    url="https://gitea.ongoteam.net/PTV/ptvp35",
-    license='',
+    license="MIT",
-    author='PARRRATE T&V',
+    author="PARRRATE TNV",
-    author_email='',
+    author_email="",
-    description=''
+    description="",
 )

test_delete.py

@@ -0,0 +1,23 @@
+import asyncio
+import pathlib
+
+from ptvp35 import VDELETE, DbFactory, KVJson
+
+
+async def main():
+    path = pathlib.Path("test_delete.db")
+    path.unlink(missing_ok=True)
+    async with DbFactory(path, kvfactory=KVJson()) as connection:
+        connection.set_nowait(0, 0)
+        print(connection.get(0, 1))
+        await connection.commit()
+        async with connection.transaction() as transaction:
+            print(transaction.get(0, 1))
+            transaction.set_nowait(0, VDELETE)
+            print(transaction.get(0, 1))
+        input()
+        print(connection.get(0, 1))
+    # path.unlink(missing_ok=True)
+
+
+asyncio.run(main())