From dcc9d642aa713074b327d032f8e609a83b2904c8 Mon Sep 17 00:00:00 2001 From: timofey Date: Fri, 13 Jan 2023 15:33:04 +0000 Subject: [PATCH] 1.1rc3: more fsync --- .gitignore | 4 +-- docs/scripts/traced_example.py | 3 +- docs/source/motivation.rst | 2 +- docs/source/structure.rst | 64 +++++++++++++++++++++++++++++++--- docs/source/usage.rst | 46 +++++++++--------------- ptvp35/__init__.py | 18 ++++++---- setup.py | 2 +- 7 files changed, 94 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index c00be02..0e765a9 100644 --- a/.gitignore +++ b/.gitignore @@ -222,5 +222,5 @@ cython_debug/ # Other /dev.py -/*.db -/*.db.* +*.db +*.db.* diff --git a/docs/scripts/traced_example.py b/docs/scripts/traced_example.py index 95c40a2..e2e1c5a 100644 --- a/docs/scripts/traced_example.py +++ b/docs/scripts/traced_example.py @@ -5,8 +5,7 @@ import threading from contextlib import ExitStack try: - sys.path.append('/app/') - + sys.path.append(str((pathlib.Path(__file__).parent / '../..').absolute())) from ptvp35 import * from ptvp35.instrumentation import * except: diff --git a/docs/source/motivation.rst b/docs/source/motivation.rst index 942db8d..fcb341c 100644 --- a/docs/source/motivation.rst +++ b/docs/source/motivation.rst @@ -6,7 +6,7 @@ This page describes reasons for certain design decisions. General structure ----------------- -* We're making a key-value database. +* A key-value database. * Keys and values are python objects. * While the DB is offline (after a correct shutdown), all its KVPs are stored as lines in one editable file. * DB should be able to survive powercut at any point (D in ACID). diff --git a/docs/source/structure.rst b/docs/source/structure.rst index daf6f97..2ce753d 100644 --- a/docs/source/structure.rst +++ b/docs/source/structure.rst @@ -4,14 +4,70 @@ Inner structure Main-Memory DataBase -------------------- +* Represented via Python dictionary (:code:`dict`). + Future version may include more abstract options, i.e. :class:`ptvp35.DbConnection` would be generic over :attr:`ptvp35.DbConnection.__mmdb` field +* Keys and values are almost guaranteed to be serializable. + All KVPs loaded on :code:`__aenter__` are re-serialized during file reload. + All KVPs added via submit-like methods are serialized before being added to the MMDB. + DataBase Stream File -------------------- +* In current implementation, all database storage storage files are Newline-Delimited JSON streams (https://en.wikipedia.org/wiki/JSON_streaming#Newline-Delimited_JSON). + .. code-block:: json + + {"key": ["tuple", "example"], "value": {"dict": "example"}} + {"key": 123, "value": null} +* During the runtime, the database uses 6 different files: + * `.db` Main file. + Should be the only non-error file after correct shutdown. + * `.db.backup` Backup file. + Generated when the main file is being rebuilt. + * `.db.recover` Flag file. + Indicates that backup file is valid and that main file's validity is undefined. + * `.db.truncate` Auxiliary file created on each write to main file. + Contains 16 bytes little-endian representation of up to how many characters the main file is valid. + * `.db.truncate_flag` Flag file. + Indicates that truncate file is valid and that main file's validity after the specified character count is undefined. + * `.db.error` Error log file. + In current implementation, it contains only the main file contents that got truncated on recovery. +* All storage file writes are :code:`fsync`'ed. + * :code:`pathlib.Path.write_bytes` usecase relies on synchronisation/file-creation ordering. That may get replaced in future versions. + Request Queue ------------- -Transaction View ----------------- +Transaction View (:class:`ptvp35.TransactionView`) +-------------------------------------------------- -Transaction ------------ +Connection-like interface on top of another connection-like interface. + +* Provides most of the same methods as :class:`ptvp35.DbConnection`. + * From the common :class:`ptvp35.VirtualConnection` interface/base class: + * :meth:`ptvp35.TransactionView.get` + * :meth:`ptvp35.TransactionView.commit_transaction` + * :meth:`ptvp35.TransactionView.submit_transaction_request` + * :meth:`ptvp35.TransactionView.loop` + * :meth:`ptvp35.TransactionView.transaction` (default implementation) + * Non-standard common methods: + * :meth:`ptvp35.TransactionView.set_nowait` + * :meth:`ptvp35.TransactionView.submit_transaction` + * :meth:`ptvp35.TransactionView.commit` +* Does not have the the analogue for the :meth:`ptvp35.DbConnection.set` method. + * The reason for that is :code:`set` method having semantics contradictory to transactions. + * The :code:`set` provides a way to set a *single* value and wait until it's committed. + * Transaction are meant for a more fine control. + * The equivalent would consist of the three method calls: + * :meth:`ptvp35.TransactionView.set_nowait` to set the value in :code:`__delta`. + * :meth:`ptvp35.TransactionView.submit` to pass all the :code:`__delta` values. + * :meth:`ptvp35.TransactionView.commit` wait until all changes are commited. + +Transaction (:class:`ptvp35.Transaction`) +----------------------------------------- + +Manages a Transaction View. + +* Creates and returns a Transaction View on :code:`__aenter__`/:code:`__enter__`. +* Submits changes on successful :code:`__exit__`. +* Commits changes on successful :code:`__aexit__`. +* Rolls back changes on unsuccessful :code:`__exit__`/:code:`__aexit__`. diff --git a/docs/source/usage.rst b/docs/source/usage.rst index 29e1c7a..a41f5a9 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -13,12 +13,10 @@ Default installation option is to use pip+git Basic functionality ------------------- -.. autoclass:: ptvp35.DbFactory - - :code:`DbFactory` class provides context management for database connections (:code:`DbConnection`) via :code:`async with` statement. - The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**. - Also, that means that each connection start/shutdown is quite time expensive. - These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block. +:class:`ptvp35.DbFactory` class provides context management for database connections (:class:`ptvp35.DbConnection`) via :code:`async with` statement. +The connection isn't just a "connection", it's also the MMDB itself, so **using two connections to one database is an undefined behaviour**. +Also, that means that each connection start/shutdown is quite time expensive. +These two facts together tell that, if you intend on using the connection, you should probably wrap the main program in an :code:`async with` block. .. code-block:: python3 @@ -62,26 +60,16 @@ Different ways to get/set a value: transaction.set_nowait('increment-5', value5 + 1) await connection.commit() -.. autoclass:: ptvp35.DbConnection - - .. automethod:: get - - this method is instant. - - .. automethod:: set - - this method may take time to run. - ordering may not be guaranteed (depends on event loop implementation). - - .. automethod:: set_nowait - - this method is instant. - ordering is guaranteed. - - .. automethod:: commit - - this method may take time to run. - respects the ordering of previously called :code:`set_nowait` methods. - will, under most circumstances, also execute later changes. - - .. automethod:: transaction +* :meth:`ptvp35.DbConnection.get` + this method is instant. +* :meth:`ptvp35.DbConnection.set` + this method may take time to run. + ordering may not be guaranteed (depends on event loop implementation). +* :meth:`ptvp35.DbConnection.set_nowait` + this method is instant. + ordering is guaranteed. +* :meth:`ptvp35.DbConnection.commit` + this method may take time to run. + respects the ordering of previously called :code:`set_nowait` methods. + will, depending on event loop implementation, also execute later changes. +* :meth:`ptvp35.DbConnection.transaction` diff --git a/ptvp35/__init__.py b/ptvp35/__init__.py index e5649da..959ed01 100644 --- a/ptvp35/__init__.py +++ b/ptvp35/__init__.py @@ -262,7 +262,6 @@ class VirtualConnection(abc.ABC): def loop(self, /) -> asyncio.AbstractEventLoop: raise NotImplementedError - @abc.abstractmethod def transaction(self, /) -> 'Transaction': return Transaction(self) @@ -342,7 +341,9 @@ class DbConnection(VirtualConnection): def _db2path_sync(self, db: dict, path: pathlib.Path, /) -> int: with path.open('w') as file: - return self.__kvfactory.db2io(db, file) + initial_size = self.__kvfactory.db2io(db, file) + os.fsync(file.fileno()) + return initial_size def get(self, key: Any, default: Any, /): """dict-like get with mandatory default parametre.""" @@ -430,10 +431,15 @@ class DbConnection(VirtualConnection): self.__buffer.write(line) await self._commit_buffer_or_request_so(request) + def _write_truncation_bytes(self, s: bytes, /) -> None: + # consider subclassing/rewriting to use `os.fsync` + self.__path_truncate.write_bytes(s) + + def _write_truncation_value(self, value: int, /) -> None: + self._write_truncation_bytes(value.to_bytes(16, 'little')) + def _truncation_set_sync(self, /) -> None: - self.__path_truncate.write_bytes( - self.__file.tell().to_bytes(16, 'little') - ) + self._write_truncation_value(self.__file.tell()) self.__path_truncate_flag.touch() def _truncation_unset_sync(self, /) -> None: @@ -695,7 +701,7 @@ note: unstable signature.""" return self.__loop def transaction(self, /) -> 'Transaction': - return Transaction(self) + return super().transaction() class DbFactory: diff --git a/setup.py b/setup.py index e108777..1cffba4 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup setup( name='ptvp35', - version='1.1rc2', + version='1.1rc3', packages=['ptvp35'], url='https://gitea.ongoteam.net/PTV/ptvp35', license='MIT',