1.1rc3: more fsync

.gitignore

@@ -222,5 +222,5 @@ cython_debug/
 
 # Other
 /dev.py
-/*.db
+*.db
-/*.db.*
+*.db.*

docs/scripts/traced_example.py

@@ -5,8 +5,7 @@ import threading
 from contextlib import ExitStack
 
 try:
-    sys.path.append('/app/')
+    sys.path.append(str((pathlib.Path(__file__).parent / '../..').absolute()))
-
     from ptvp35 import *
     from ptvp35.instrumentation import *
 except:

docs/source/motivation.rst

@@ -6,7 +6,7 @@ This page describes reasons for certain design decisions.
 General structure
 -----------------
 
-* We're making a key-value database.
+* A key-value database.
 * Keys and values are python objects.
 * While the DB is offline (after a correct shutdown), all its KVPs are stored as lines in one editable file.
 * DB should be able to survive powercut at any point (D in ACID).

docs/source/structure.rst

@@ -4,14 +4,70 @@ Inner structure
 Main-Memory DataBase
 --------------------
 
+* Represented via Python dictionary (:code:`dict`).
+    Future version may include more abstract options, i.e. :class:`ptvp35.DbConnection` would be generic over :attr:`ptvp35.DbConnection.__mmdb` field
+* Keys and values are almost guaranteed to be serializable.
+    All KVPs loaded on :code:`__aenter__` are re-serialized during file reload.
+    All KVPs added via submit-like methods are serialized before being added to the MMDB.
+
 DataBase Stream File
 --------------------
 
+* In current implementation, all database storage storage files are Newline-Delimited JSON streams (https://en.wikipedia.org/wiki/JSON_streaming#Newline-Delimited_JSON).
+    .. code-block:: json
+
+        {"key": ["tuple", "example"], "value": {"dict": "example"}}
+        {"key": 123, "value": null}
+* During the runtime, the database uses 6 different files:
+    * `.db` Main file.
+        Should be the only non-error file after correct shutdown.
+    * `.db.backup` Backup file.
+        Generated when the main file is being rebuilt.
+    * `.db.recover` Flag file.
+        Indicates that backup file is valid and that main file's validity is undefined.
+    * `.db.truncate` Auxiliary file created on each write to main file.
+        Contains 16 bytes little-endian representation of up to how many characters the main file is valid.
+    * `.db.truncate_flag` Flag file.
+        Indicates that truncate file is valid and that main file's validity after the specified character count is undefined.
+    * `.db.error` Error log file.
+        In current implementation, it contains only the main file contents that got truncated on recovery.
+* All storage file writes are :code:`fsync`'ed.
+    * :code:`pathlib.Path.write_bytes` usecase relies on synchronisation/file-creation ordering. That may get replaced in future versions.
+
 Request Queue
 -------------
 
-Transaction View
+Transaction View (:class:`ptvp35.TransactionView`)
-----------------
+--------------------------------------------------
 
-Transaction
+Connection-like interface on top of another connection-like interface.
------------
+
+* Provides most of the same methods as :class:`ptvp35.DbConnection`.
+    * From the common :class:`ptvp35.VirtualConnection` interface/base class:
+        * :meth:`ptvp35.TransactionView.get`
+        * :meth:`ptvp35.TransactionView.commit_transaction`
+        * :meth:`ptvp35.TransactionView.submit_transaction_request`
+        * :meth:`ptvp35.TransactionView.loop`
+        * :meth:`ptvp35.TransactionView.transaction` (default implementation)
+    * Non-standard common methods:
+        * :meth:`ptvp35.TransactionView.set_nowait`
+        * :meth:`ptvp35.TransactionView.submit_transaction`
+        * :meth:`ptvp35.TransactionView.commit`
+* Does not have the the analogue for the :meth:`ptvp35.DbConnection.set` method.
+    * The reason for that is :code:`set` method having semantics contradictory to transactions.
+        * The :code:`set` provides a way to set a *single* value and wait until it's committed.
+        * Transaction are meant for a more fine control.
+    * The equivalent would consist of the three method calls:
+        * :meth:`ptvp35.TransactionView.set_nowait` to set the value in :code:`__delta`.
+        * :meth:`ptvp35.TransactionView.submit` to pass all the :code:`__delta` values.
+        * :meth:`ptvp35.TransactionView.commit` wait until all changes are commited.
+
+Transaction (:class:`ptvp35.Transaction`)
+-----------------------------------------
+
+Manages a Transaction View.
+
+* Creates and returns a Transaction View on :code:`__aenter__`/:code:`__enter__`.
+* Submits changes on successful :code:`__exit__`.
+* Commits changes on successful :code:`__aexit__`.
+* Rolls back changes on unsuccessful :code:`__exit__`/:code:`__aexit__`.

docs/source/usage.rst

@@ -13,12 +13,10 @@ Default installation option is to use pip+git
 Basic functionality
 -------------------
 
-.. autoclass:: ptvp35.DbFactory
+:class:`ptvp35.DbFactory` class provides context management for database connections (:class:`ptvp35.DbConnection`) via :code:`async with` statement.
-
+The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**.
-    :code:`DbFactory` class provides context management for database connections (:code:`DbConnection`) via :code:`async with` statement.
+Also, that means that each connection start/shutdown is quite time expensive.
-    The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**.
+These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block.
-    Also, that means that each connection start/shutdown is quite time expensive.
-    These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block.
 
 .. code-block:: python3
 
@@ -62,26 +60,16 @@ Different ways to get/set a value:
             transaction.set_nowait('increment-5', value5 + 1)
         await connection.commit()
 
-.. autoclass:: ptvp35.DbConnection
+* :meth:`ptvp35.DbConnection.get`
-
+    this method is instant.
-    .. automethod:: get
+* :meth:`ptvp35.DbConnection.set`
-
+    this method may take time to run.
-        this method is instant.
+    ordering may not be guaranteed (depends on event loop implementation).
-
+* :meth:`ptvp35.DbConnection.set_nowait`
-    .. automethod:: set
+    this method is instant.
-
+    ordering is guaranteed.
-        this method may take time to run.
+* :meth:`ptvp35.DbConnection.commit`
-        ordering may not be guaranteed (depends on event loop implementation).
+    this method may take time to run.
-
+    respects the ordering of previously called :code:`set_nowait` methods.
-    .. automethod:: set_nowait
+    will, depending on event loop implementation, also execute later changes.
-
+* :meth:`ptvp35.DbConnection.transaction`
-        this method is instant.
-        ordering is guaranteed.
-
-    .. automethod:: commit
-
-        this method may take time to run.
-        respects the ordering of previously called :code:`set_nowait` methods.
-        will, under most circumstances, also execute later changes.
-
-    .. automethod:: transaction

ptvp35/__init__.py

@@ -262,7 +262,6 @@ class VirtualConnection(abc.ABC):
     def loop(self, /) -> asyncio.AbstractEventLoop:
         raise NotImplementedError
 
-    @abc.abstractmethod
     def transaction(self, /) -> 'Transaction':
         return Transaction(self)
 
@@ -342,7 +341,9 @@ class DbConnection(VirtualConnection):
 
     def _db2path_sync(self, db: dict, path: pathlib.Path, /) -> int:
         with path.open('w') as file:
-            return self.__kvfactory.db2io(db, file)
+            initial_size = self.__kvfactory.db2io(db, file)
+            os.fsync(file.fileno())
+            return initial_size
 
     def get(self, key: Any, default: Any, /):
         """dict-like get with mandatory default parametre."""
@@ -430,10 +431,15 @@ class DbConnection(VirtualConnection):
         self.__buffer.write(line)
         await self._commit_buffer_or_request_so(request)
 
+    def _write_truncation_bytes(self, s: bytes, /) -> None:
+        # consider subclassing/rewriting to use `os.fsync`
+        self.__path_truncate.write_bytes(s)
+
+    def _write_truncation_value(self, value: int, /) -> None:
+        self._write_truncation_bytes(value.to_bytes(16, 'little'))
+
     def _truncation_set_sync(self, /) -> None:
-        self.__path_truncate.write_bytes(
+        self._write_truncation_value(self.__file.tell())
-            self.__file.tell().to_bytes(16, 'little')
-        )
         self.__path_truncate_flag.touch()
 
     def _truncation_unset_sync(self, /) -> None:
@@ -695,7 +701,7 @@ note: unstable signature."""
         return self.__loop
 
     def transaction(self, /) -> 'Transaction':
-        return Transaction(self)
+        return super().transaction()
 
 
 class DbFactory:

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='ptvp35',
-    version='1.1rc2',
+    version='1.1rc3',
     packages=['ptvp35'],
     url='https://gitea.ongoteam.net/PTV/ptvp35',
     license='MIT',