PTV/ptvp35
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

initial commit

Browse Source
This commit is contained in:
AF 2021-07-19 12:40:59 +03:00
commit a73ffc1993
9 changed files with 555 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

225
.gitignore vendored Normal file
View File

@ -0,0 +1,225 @@
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/jarRepositories.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# Other
/dev.py
/*.db

8
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/

51
.idea/inspectionProfiles/Project_Default.xml Normal file
View File

@ -0,0 +1,51 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="DuplicatedCode" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="HtmlUnknownTag" enabled="true" level="WARNING" enabled_by_default="true">
<option name="myValues">
<value>
<list size="11">
<item index="0" class="java.lang.String" itemvalue="nobr" />
<item index="1" class="java.lang.String" itemvalue="noembed" />
<item index="2" class="java.lang.String" itemvalue="comment" />
<item index="3" class="java.lang.String" itemvalue="noscript" />
<item index="4" class="java.lang.String" itemvalue="embed" />
<item index="5" class="java.lang.String" itemvalue="script" />
<item index="6" class="java.lang.String" itemvalue="markdown" />
<item index="7" class="java.lang.String" itemvalue="sv3i" />
<item index="8" class="java.lang.String" itemvalue="sv3o" />
<item index="9" class="java.lang.String" itemvalue="sv3a" />
<item index="10" class="java.lang.String" itemvalue="sv3c" />
</list>
</value>
</option>
<option name="myCustomValuesEnabled" value="true" />
</inspection_tool>
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="1">
<item index="0" class="java.lang.String" itemvalue="nacl" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="PySide2.QtWidgets.clicked.connect" />
<option value="PySide2.QtWidgets.valueChanged.connect" />
<option value="PySide2.QtWidgets.textChanged.connect" />
<option value="PySide2.QtCore.Signal.emit" />
<option value="PySide2.QtCore.Signal.connect" />
</list>
</option>
</inspection_tool>
<inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
<option name="processCode" value="true" />
<option name="processLiterals" value="true" />
<option name="processComments" value="true" />
</inspection_tool>
</profile>
</component>

6
.idea/inspectionProfiles/profiles_settings.xml Normal file
View File

@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
.idea/misc.xml Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (ptvp35)" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml Normal file
View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ptvp35.iml" filepath="$PROJECT_DIR$/.idea/ptvp35.iml" />
</modules>
</component>
</project>

10
.idea/ptvp35.iml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

237
ptvp35/__init__.py Normal file
View File

@ -0,0 +1,237 @@
import asyncio
import json
import pathlib
import pickle
from io import StringIO
from typing import Any, Optional, IO, Type
class Request:
def set_result(self, result):
pass
def set_exception(self, exception):
pass
class KVRequest(Request):
def __init__(self, key: Any, value: Any, future: Optional[asyncio.Future]):
self.key = key
self.value = value
self.future = future
def free(self):
return type(self)(self.key, self.value, None)
def line(self) -> str:
raise NotImplementedError
@classmethod
def fromline(cls, line: str) -> 'KVRequest':
raise NotImplementedError
def set_result(self, result):
self.future and self.future.set_result(result)
def set_exception(self, exception):
self.future and self.future.set_exception(exception)
class DumpRequest(Request):
pass
class UnkownRequestType(TypeError):
pass
class KVPickle(KVRequest):
def line(self) -> str:
return pickle.dumps(self.free()).hex() + "\n"
@classmethod
def fromline(cls, line: str) -> 'KVPickle':
return pickle.loads(bytes.fromhex(line.strip()))
class KVJson(KVRequest):
def line(self) -> str:
return json.dumps({'key': self.key, 'value': self.value}) + "\n"
@classmethod
def fromline(cls, line: str) -> 'KVJson':
d = json.loads(line)
return KVJson(d['key'], d['value'], None)
class Db:
__mmdb: Optional[dict]
__loop: Optional[asyncio.AbstractEventLoop]
__queue: Optional[asyncio.Queue]
__file: Optional[IO[str]]
__buffer: Optional[StringIO]
__task: Optional[asyncio.Future]
def __init__(self, path: str, *, kvrequest_type: Type[KVRequest], buffersize=1048576):
self.kvrequest_type = kvrequest_type
self.buffersize = buffersize
self.__path = pathlib.Path(path)
self.__task = None
def io2db(self, io: IO[str], db: dict) -> int:
size = 0
for line in io:
request = self.kvrequest_type.fromline(line)
db[request.key] = request.value
size += len(line)
return size
def db2io(self, db: dict, io: IO[str]) -> int:
size = 0
for key, value in db.items():
size += io.write(self.kvrequest_type(key, value, None).line())
return size
def get(self, key: Any, default: Any):
return self.__mmdb.get(key, default)
async def set(self, key: Any, value: Any):
self.__mmdb[key] = value
future = self.__loop.create_future()
self.__queue.put_nowait(self.kvrequest_type(key, value, future))
await future
def set_nowait(self, key: Any, value: Any):
self.__mmdb[key] = value
self.__queue.put_nowait(self.kvrequest_type(key, value, None))
async def _dump_buffer_or_request_so(self):
if self.__buffer.tell() >= self.buffersize:
await self._dump_buffer()
else:
await self.__queue.put(DumpRequest())
async def _write(self, line: str):
self.__buffer.write(line)
await self._dump_buffer_or_request_so()
def _clear_buffer(self):
self.__buffer = StringIO()
def _compress_buffer(self) -> StringIO:
self.__buffer.seek(0)
bufferdb = {}
self.io2db(self.__buffer, bufferdb)
buffer = StringIO()
self.db2io(bufferdb, buffer)
return buffer
async def _dump_compressed_buffer(self):
buffer = self._compress_buffer()
await self.__loop.run_in_executor(None, self.__file.write, buffer.getvalue())
async def _do_dump_buffer(self):
await self._dump_compressed_buffer()
self._clear_buffer()
async def _reload_if_oversized(self):
if self.__file.tell() > 2 * self.__initial_size:
await self.__loop.run_in_executor(None, self._reload)
async def _dump_buffer(self):
if self.__buffer.tell():
await self._do_dump_buffer()
await self._reload_if_oversized()
async def _handle_request(self, request: Request):
if isinstance(request, self.kvrequest_type):
await self._write(request.line())
elif isinstance(request, DumpRequest):
await self._dump_buffer()
else:
raise UnkownRequestType
request.set_result(None)
async def _background_cycle(self):
request: Request = await self.__queue.get()
try:
await self._handle_request(request)
except Exception as e:
request.set_exception(e)
finally:
self.__queue.task_done()
async def _background_task(self):
while True:
await self._background_cycle()
def _rebuild_file(self, db: {}):
self.__path.touch()
with open(self.__path) as file:
self.io2db(file, db)
with open(self.__path, "w") as file:
self.__initial_size = self.db2io(db, file)
def _reload(self):
self.__file.close()
self._rebuild_file({})
self.__file = open(self.__path, "a")
def _load_from_file(self):
self._rebuild_file(self.__mmdb)
async def _initialize_mmdb(self):
self.__mmdb = {}
await self.__loop.run_in_executor(None, self._load_from_file)
async def _initialize_queue(self):
self.__queue = asyncio.Queue()
self.__file = open(self.__path, "a")
self.__buffer = StringIO()
self.__task = self.__loop.create_task(self._background_task())
async def _initialize(self):
assert self.__task is None
self.__loop = asyncio.get_event_loop()
await self._initialize_mmdb()
await self._initialize_queue()
async def __aenter__(self):
await self._initialize()
return self
async def _aclose(self):
if not self.__task.done():
await self.__queue.join()
self.__task.cancel()
await self._dump_buffer()
self.__file.close()
with open(self.__path, "w") as file:
self.db2io(self.__mmdb, file)
def _uninitialize(self):
self.__mmdb = None
self.__loop = None
self.__queue = None
self.__file = None
self.__buffer = None
self.__task = None
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self._aclose()
self._uninitialize()
def cursor(self, **kwargs):
return Cursor(self, **kwargs)
class Cursor:
def __init__(self, db: Db, default=None):
self.default = default
self.db = db
def __getitem__(self, item):
return self.db.get(item, self.default)
def __setitem__(self, key, value):
self.db.set_nowait(key, value)