diff --git a/plot.py b/plot.py index aabba7c..425a617 100644 --- a/plot.py +++ b/plot.py @@ -13,7 +13,7 @@ def plottable(log: list[tuple[float, int]]): def plot(fn: str): - plt.rcParams['figure.figsize'] = [16, 9] + plt.rcParams['figure.figsize'] = [64, 18] plt.style.use("dark_background") plt.subplots_adjust(left=0.05, right=0.99, top=0.95, bottom=0.05) plt.title(fn) @@ -21,19 +21,19 @@ def plot(fn: str): plt.ylabel('concurrency (1)') with open(fn) as file: - jsonified = json.load(file) - if (log := jsonified.get('DelayedResolver:sleep:concurrency')) is not None: - plt.plot(*plottable(log)) - if (log := jsonified.get('ActiveBinaryTree:add:concurrency')) is not None: - plt.plot(*plottable(log)) - if (log := jsonified.get('ActiveBinaryTree:contains:concurrency')) is not None: - plt.plot(*plottable(log)) - if (log := jsonified.get('Stack:list:concurrency')) is not None: - plt.plot(*plottable(log)) - if (log := jsonified.get('ActiveBinaryTree:add:entry')) is not None: - plt.scatter(*plottable(log), c='tomato', zorder=100, s=.5) - if (log := jsonified.get('ActiveBinaryTree:add:exit')) is not None: - plt.scatter(*plottable(log), c='gold', zorder=99, s=.5) + jsonified: dict = json.load(file) + + def logplot(plot_function, metric: str, **kwargs): + if (log := jsonified.pop(metric, None)) is not None: + plot_function(*plottable(log), label=f'{metric} ({len(log)})', **kwargs) + + logplot(plt.plot, 'DelayedResolver:sleep:concurrency') + logplot(plt.plot, 'ActiveBinaryTree:add:concurrency') + logplot(plt.plot, 'ActiveBinaryTree:contains:concurrency') + logplot(plt.plot, 'Stack:list:concurrency') + logplot(plt.scatter, 'ActiveBinaryTree:add:entry', c='tomato', zorder=100, s=.5) + logplot(plt.scatter, 'ActiveBinaryTree:add:exit', c='gold', zorder=99, s=.5) + plt.legend() plt.show() diff --git a/rainbowadn/collection/linear/__init__.py b/rainbowadn/collection/linear/__init__.py index 28ec67c..c1c67a2 100644 --- a/rainbowadn/collection/linear/__init__.py +++ b/rainbowadn/collection/linear/__init__.py @@ -1,2 +1,3 @@ from .array import * from .stack import * +from .treelist import * diff --git a/rainbowadn/collection/linear/treelist/__init__.py b/rainbowadn/collection/linear/treelist/__init__.py new file mode 100644 index 0000000..9c9abaf --- /dev/null +++ b/rainbowadn/collection/linear/treelist/__init__.py @@ -0,0 +1,2 @@ +from .tlroot import * +from .tlrparametres import TLRParametres diff --git a/rainbowadn/collection/linear/treelist/tlnode.py b/rainbowadn/collection/linear/treelist/tlnode.py new file mode 100644 index 0000000..7adc4b4 --- /dev/null +++ b/rainbowadn/collection/linear/treelist/tlnode.py @@ -0,0 +1,171 @@ +from typing import Generic, Iterable, TypeVar + +from rainbowadn.core import * +from .tlparametres import TLParametres + +__all__ = ('TLNode', 'TLNodeFactory',) + +ElementType = TypeVar('ElementType') + + +class TLNode(RecursiveMentionable, Generic[ElementType]): + def __init__( + self, + source: bytes, + parametres: TLParametres[ElementType], + node_cache: tuple[MetaOrigin['TLNode[ElementType]'], ...], + element_cache: tuple[MetaOrigin[ElementType], ...], + ): + assert isinstance(source, bytes) + assert isinstance(parametres, TLParametres) + assert isinstance(node_cache, tuple) + assert isinstance(element_cache, tuple) + self.parametres = parametres + self.source = source + if parametres.leaf: + assert len(node_cache) == 0 + self.elements = parametres.branching + assert len(element_cache) == self.elements + self.element_cache = element_cache + else: + assert len(element_cache) == 0 + self.nodes = parametres.branching + assert len(node_cache) == self.nodes + self.node_cache = node_cache + + def bytes_no(self, index: int) -> bytes: + assert isinstance(index, int) + assert index < self.parametres.branching + return self.source[HashPoint.HASH_LENGTH * index:HashPoint.HASH_LENGTH * (index + 1)] + + def element_no(self, index: int) -> HashPoint[ElementType]: + assert isinstance(index, int) + assert index < self.elements + meta_origin: MetaOrigin[ElementType] = self.element_cache[index] + return meta_origin.hash_point(self.parametres.tlr.factory, self.bytes_no(index)) + + def node_no(self, index: int) -> HashPoint['TLNode[ElementType]']: + assert isinstance(index, int) + assert index < self.nodes + meta_origin: MetaOrigin[TLNode[ElementType]] = self.node_cache[index] + return meta_origin.hash_point( + TLNodeFactory( + self.parametres.params_no(index) + ), + self.bytes_no(index) + ) + + async def node_no_resolved(self, index: int) -> 'TLNode[ElementType]': + assert isinstance(index, int) + assert index < self.nodes + return await self.node_no(index).resolve() + + def points(self) -> Iterable[HashPoint]: + if self.parametres.leaf: + return [self.element_no(element) for element in range(self.elements)] + else: + return [self.node_no(node) for node in range(self.nodes)] + + def __topology_hash__(self) -> bytes: + return HashPoint.hash(self.source) + + def __bytes__(self): + return self.source + + def __factory__(self) -> RainbowFactory['TLNode[ElementType]']: + return TLNodeFactory( + self.parametres + ) + + async def str(self, tab: int) -> str: + assert isinstance(tab, int) + if self.parametres.size == 0: + return f'-' + else: + return f'{tabulate(tab)}'.join(await gather(*(hash_point_format(point, tab) for point in self.points()))) + + def unit(self, element: HashPoint[ElementType]) -> 'TLNode[ElementType]': + assert isinstance(element, HashPoint) + return TLNode( + bytes(element), + self.parametres.unit(), + (), + (LocalMetaOrigin(element.origin),), + ) + + def source_without_last(self) -> bytes: + return self.source[:-HashPoint.HASH_LENGTH] + + async def last(self) -> 'TLNode[ElementType]': + assert not self.parametres.leaf + return await self.node_no_resolved(self.nodes - 1) + + async def add(self, element: HashPoint[ElementType]) -> 'TLNode[ElementType]': + assert isinstance(element, HashPoint) + if self.parametres.full: + self_hp = HashPoint.of(self) + assert isinstance(self_hp, HashPoint) + unit = self.unit(element) + assert isinstance(unit, TLNode) + unit_hp = HashPoint.of(unit) + assert isinstance(unit_hp, HashPoint) + return TLNode( + bytes(self_hp) + bytes(unit_hp), + self.parametres.superparams(), + (LocalMetaOrigin(self_hp.origin), LocalMetaOrigin(unit_hp.origin)), + (), + ) + elif self.parametres.leaf: + return TLNode( + self.source + bytes(element), + self.parametres.superparams(), + (), + self.element_cache + (LocalMetaOrigin(element.origin),), + ) + elif self.parametres.lastfull: + unit = self.unit(element) + assert isinstance(unit, TLNode) + unit_hp = HashPoint.of(unit) + assert isinstance(unit_hp, HashPoint) + return TLNode( + self.source + bytes(unit_hp), + self.parametres.superparams(), + self.node_cache + (LocalMetaOrigin(unit_hp.origin),), + (), + ) + else: + last = await (await self.last()).add(element) + assert isinstance(last, TLNode) + last_hp = HashPoint.of(last) + assert isinstance(last_hp, HashPoint) + return TLNode( + self.source_without_last() + bytes(last_hp), + self.parametres.superparams(), + self.node_cache[:-1] + (LocalMetaOrigin(last_hp.origin),), + (), + ) + + +class TLNodeFactory(RainbowFactory[TLNode[ElementType]], Generic[ElementType]): + def __init__( + self, + parametres: TLParametres[ElementType], + ): + assert isinstance(parametres, TLParametres) + self.parametres = parametres + if self.parametres.leaf: + self.nodes, self.elements = 0, parametres.branching + else: + self.nodes, self.elements = parametres.branching, 0 + assert isinstance(self.nodes, int) + assert isinstance(self.elements, int) + + def from_bytes(self, source: bytes, resolver: HashResolver) -> TLNode[ElementType]: + assert isinstance(source, bytes) + assert isinstance(resolver, HashResolver) + return TLNode( + source, + self.parametres, + (ResolverMetaOrigin(resolver),) * self.nodes, + (ResolverMetaOrigin(resolver),) * self.elements, + ) diff --git a/rainbowadn/collection/linear/treelist/tlparametres.py b/rainbowadn/collection/linear/treelist/tlparametres.py new file mode 100644 index 0000000..070e706 --- /dev/null +++ b/rainbowadn/collection/linear/treelist/tlparametres.py @@ -0,0 +1,73 @@ +from typing import Generic, TypeVar + +from .tlrparametres import TLRParametres + +__all__ = ('TLParametres',) + +ElementType = TypeVar('ElementType') + + +class TLParametres( + Generic[ElementType] +): + def __init__( + self, + tlr: TLRParametres[ElementType], + size: int, + ): + assert isinstance(tlr, TLRParametres) + assert isinstance(size, int) + self.tlr = tlr + assert size >= 0 + self.order = tlr.order + assert isinstance(self.order, int) + self.size = size + self.height = 0 + while self._capacity() < size: + self.height += 1 + self.capacity = self._capacity() + assert isinstance(self.capacity, int) + self.full = self.size == self.capacity + self.leaf = self.height == 0 + self.subsize = self._subsize() + assert isinstance(self.subsize, int) + self.branching, self.lastsize, self.lastfull = self._branching_lastsize_lastfull() + assert isinstance(self.branching, int) + assert isinstance(self.lastsize, int) + assert isinstance(self.lastfull, bool) + + def _branching_lastsize_lastfull(self) -> tuple[int, int, bool]: + branching, lastsize = divmod(self.size, self.subsize) + if lastsize == 0: + return branching, self.subsize, True + else: + return branching + 1, lastsize, False + + def _capacity(self) -> int: + return self.order ** (self.height + 1) + + def _subsize(self) -> int: + return self.order ** self.height + + def superparams(self) -> 'TLParametres[ElementType]': + return TLParametres(self.tlr, self.size + 1) + + def unit(self) -> 'TLParametres[ElementType]': + return TLParametres(self.tlr, 1) + + def _subsize_no(self, index: int) -> int: + assert isinstance(index, int) + assert index < self.branching + if index == self.branching - 1: + return self.lastsize + else: + return self.subsize + + def params_no(self, index: int) -> 'TLParametres[ElementType]': + assert isinstance(index, int) + assert not self.leaf + assert index < self.branching + return TLParametres( + self.tlr, + self._subsize_no(index), + ) diff --git a/rainbowadn/collection/linear/treelist/tlroot.py b/rainbowadn/collection/linear/treelist/tlroot.py new file mode 100644 index 0000000..f4c872c --- /dev/null +++ b/rainbowadn/collection/linear/treelist/tlroot.py @@ -0,0 +1,76 @@ +from typing import Generic, Iterable, TypeVar + +from rainbowadn.atomic import * +from rainbowadn.core import * +from .tlnode import * +from .tlparametres import TLParametres +from .tlrparametres import TLRParametres + +__all__ = ('TLRoot', 'TLRootFactory',) + +ElementType = TypeVar('ElementType') + + +class TLRoot(RecursiveMentionable, Generic[ElementType]): + def __init__( + self, + node: HashPoint[TLNode[ElementType]], + parametres: TLParametres + ): + assert isinstance(node, HashPoint) + assert isinstance(parametres, TLParametres) + self.node = node + self.parametres = parametres + + def points(self) -> Iterable[HashPoint]: + return [self.node] + + def __bytes__(self): + return bytes(self.node) + bytes(Integer(self.parametres.size)) + + def __factory__(self) -> RainbowFactory['TLRoot[ElementType]']: + return TLRootFactory(self.parametres.tlr) + + async def node_resolved(self) -> TLNode[ElementType]: + return await self.node.resolve() + + async def add(self, element: HashPoint[ElementType]) -> 'TLRoot[ElementType]': + assert isinstance(element, HashPoint) + node = await (await self.node_resolved()).add(element) + return TLRoot( + HashPoint.of(node), + node.parametres + ) + + async def str(self, tab: int) -> str: + assert isinstance(tab, int) + return await hash_point_format(self.node, tab) + + +class TLRootFactory(RainbowFactory[TLRoot[ElementType]]): + def __init__( + self, + tlr: TLRParametres, + ): + assert isinstance(tlr, TLRParametres) + self.tlr = tlr + + def from_bytes(self, source: bytes, resolver: HashResolver) -> TLRoot[ElementType]: + assert isinstance(source, bytes) + assert isinstance(resolver, HashResolver) + size: int = Integer.from_bytes(source[HashPoint.HASH_LENGTH:], resolver).integer + assert isinstance(size, int) + parametres = TLParametres(self.tlr, size) + return TLRoot( + ResolverOrigin( + TLNodeFactory(parametres), source[:HashPoint.HASH_LENGTH], resolver + ).hash_point(), + parametres + ) + + def empty(self) -> TLRoot[ElementType]: + parametres = TLParametres(self.tlr, 0) + return TLRoot( + HashPoint.of(TLNode(b'', parametres, (), ())), + parametres + ) diff --git a/rainbowadn/collection/linear/treelist/tlrparametres.py b/rainbowadn/collection/linear/treelist/tlrparametres.py new file mode 100644 index 0000000..7a2a713 --- /dev/null +++ b/rainbowadn/collection/linear/treelist/tlrparametres.py @@ -0,0 +1,22 @@ +from typing import Generic, TypeVar + +from rainbowadn.core import * + +__all__ = ('TLRParametres',) + +ElementType = TypeVar('ElementType') + + +class TLRParametres( + Generic[ElementType] +): + def __init__( + self, + order: int, + factory: RainbowFactory[ElementType], + ): + assert isinstance(order, int) + assert isinstance(factory, RainbowFactory) + assert order > 1 + self.order = order + self.factory = factory diff --git a/rainbowadn/testing/instrument/instrumentation.py b/rainbowadn/testing/instrument/instrumentation.py index 0135159..4e5d301 100644 --- a/rainbowadn/testing/instrument/instrumentation.py +++ b/rainbowadn/testing/instrument/instrumentation.py @@ -9,8 +9,8 @@ IType = TypeVar('IType') class Instrumentation(Generic[IType]): deinstrumentation = {} - method: Callable - wrap: Callable + _method: Callable + _wrap: Callable def __init__(self, target, methodname: str): assert isinstance(methodname, str) @@ -23,23 +23,24 @@ class Instrumentation(Generic[IType]): def __enter__(self: IType) -> IType: assert not hasattr(self, 'method') - self.method = getattr(self.target, self.methodname) - assert callable(self.method) + method = getattr(self.target, self.methodname) + assert callable(method) + self._method = method - @functools.wraps(self.method) + @functools.wraps(method) def wrap(*args, **kwargs): - return self.instrument(self.method, *args, **kwargs) + return self.instrument(method, *args, **kwargs) - self.wrap = wrap + self._wrap = wrap - setattr(self.target, self.methodname, self.wrap) + setattr(self.target, self.methodname, wrap) return self def schedule_deinstrumentation(self): - self.deinstrumentation[self.wrap] = self.method - del self.wrap - del self.method + self.deinstrumentation[self._wrap] = self._method + del self._wrap + del self._method def deinstrument(self): while (method := getattr(self.target, self.methodname)) in self.deinstrumentation: diff --git a/rainbowadn/testing/test_all.py b/rainbowadn/testing/test_all.py index e2bac9a..45f462c 100644 --- a/rainbowadn/testing/test_all.py +++ b/rainbowadn/testing/test_all.py @@ -9,6 +9,7 @@ import nacl.signing from rainbowadn.atomic import * from rainbowadn.chain import * from rainbowadn.collection.comparison import * +from rainbowadn.collection.linear import * from rainbowadn.collection.pair import * from rainbowadn.collection.trees.binary import * from rainbowadn.core import * @@ -26,7 +27,7 @@ class TestAll(unittest.IsolatedAsyncioTestCase): @classmethod def dr(cls) -> ExtendableResolver: dr = DictResolver() - dr = DelayedResolver(dr, lambda: 0.000) + # dr = DelayedResolver(dr, lambda: 0.000) return dr async def test_bankchain(self): @@ -192,3 +193,10 @@ class TestAll(unittest.IsolatedAsyncioTestCase): with encrypt_ctr: await Encrypted.encrypt(target, new_key) print(encrypt_ctr.counter) + + async def test_tl(self): + root = TLRootFactory(TLRParametres(2, Plain.factory())).empty() + for char in string.ascii_uppercase: + root = await root.add(HashPoint.of(Plain(char.encode()))) + print(await root.str(0)) + print((await root.node_resolved()).parametres.height) diff --git a/trace.py b/trace.py index 1564fbe..bbe9847 100644 --- a/trace.py +++ b/trace.py @@ -82,47 +82,112 @@ def get_instrumentations() -> list[Instrumentation]: ] -async def main(): - set_gather_linear() +async def _generate( + blocks: int, + subjects_min: int, + subjects_max: int, + transactions_min: int, + transactions_max: int, +) -> BankChain: bank: BankChain = BankChain.empty(ReductionChainMetaFactory().loose()) # bank = await mock(bank) - for _ in range(16): + for _ in range(blocks): bank = await bank.adds( [ Transaction.make( [], - [CoinData.of(Subject(SigningKey.generate().verify_key), 0)] * 16, + [CoinData.of(Subject(SigningKey.generate().verify_key), 0)] * random.randint( + subjects_min, + subjects_max + ), [] ) - for _ in range(16) + for + _ + in + range( + random.randint( + transactions_min, + transactions_max + ) + ) ] ) - print('built') + print('generated') + return bank + + +async def _migrate(bank: BankChain) -> BankChain: assert_true(await bank.verify()) bank = BankChain.from_reference( ReductionChainMetaFactory(), await get_dr().migrate_resolved(bank.reference) ) - print('saved') - set_gather_asyncio() + print('migrated') + return bank + + +async def _instrument(bank: BankChain) -> list[Instrumentation]: with ExitStack() as estack: - instrumentations = get_instrumentations() + instrumentations: list[Instrumentation] = get_instrumentations() for stacked in instrumentations: stacked.enter(estack) assert_true(await bank.verify()) - print(Instrumentation.deinstrumentation) + print('deinstrumentation (should be empty):', Instrumentation.deinstrumentation) + print('instrumented') + return instrumentations + + +class DeintrumentationSize(Instrumentation): + def instrument(self, method, *args, **kwargs): + print('deinstrumentation size', len(self.deinstrumentation)) + return method(*args, **kwargs) + + +async def _trace(): + set_gather_linear() + bank = await _generate( + 16, + 8, 15, + 8, 15, + ) + bank = await _migrate(bank) + set_gather_asyncio() + instrumentations = await _instrument(bank) print('traced') - fn = f'trace/{int(time.time())}-{os.urandom(2).hex()}.json' + return instrumentations + + +def _fn() -> str: + return f'trace/{int(time.time())}-{os.urandom(2).hex()}.json' + + +def _jsonify(instrumentations: list[Instrumentation]) -> dict: jsonified = {} for dumped in instrumentations: jsonified |= jsonify(dumped) + return jsonified + + +def _dump(fn: str, jsonified: dict) -> None: with open(fn, 'w') as file: json.dump( jsonified, file ) print('dumped') + + +def _copy(fn: str) -> None: shutil.copy(fn, f'trace/latest.json') print('copied') + + +async def main(): + instrumentations = await _trace() + fn = _fn() + jsonified = _jsonify(instrumentations) + _dump(fn, jsonified) + _copy(fn) plot(fn) print('plotted')