From c502aa424527251d0d1649587d3ee6ff60a3e279 Mon Sep 17 00:00:00 2001 From: miruka Date: Wed, 6 May 2020 13:39:53 -0400 Subject: [PATCH] Add model batch removal optimization for filtering --- src/backend/models/filters.py | 7 +++--- src/backend/models/model.py | 24 ++++++++++++++++++- src/backend/models/proxy.py | 7 +++--- src/backend/pyotherside_events.py | 1 + .../PythonBridge/Privates/EventHandlers.qml | 9 ++++--- src/gui/PythonBridge/PythonBridge.qml | 2 ++ 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/backend/models/filters.py b/src/backend/models/filters.py index d10148cf..c6beb802 100644 --- a/src/backend/models/filters.py +++ b/src/backend/models/filters.py @@ -58,7 +58,7 @@ class ModelFilter(ModelProxy): take_out = [] bring_back = [] - for key, item in self.items(): + for key, item in sorted(self.items(), key=lambda kv: kv[1]): if not self.accept_item(item): take_out.append(key) @@ -66,8 +66,9 @@ class ModelFilter(ModelProxy): if self.accept_item(item): bring_back.append(key) - for key in take_out: - self.filtered_out[key] = self.pop(key) + with self.batch_remove(): + for key in take_out: + self.filtered_out[key] = self.pop(key) for key in bring_back: self[key] = self.filtered_out.pop(key) diff --git a/src/backend/models/model.py b/src/backend/models/model.py index 81bb9fac..3c5927ad 100644 --- a/src/backend/models/model.py +++ b/src/backend/models/model.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: LGPL-3.0-or-later +import itertools from bisect import bisect +from contextlib import contextmanager from threading import RLock from typing import ( TYPE_CHECKING, Any, Dict, Iterator, List, MutableMapping, Optional, @@ -42,6 +44,8 @@ class Model(MutableMapping): self.take_items_ownership: bool = True + self._active_batch_remove_indice: Optional[List[int]] = None + if self.sync_id: self.instances[self.sync_id] = self @@ -144,7 +148,10 @@ class Model(MutableMapping): proxy.source_item_deleted(self, key) if self.sync_id: - ModelItemDeleted(self.sync_id, index) + if self._active_batch_remove_indice is None: + ModelItemDeleted(self.sync_id, index) + else: + self._active_batch_remove_indice.append(index) def __iter__(self) -> Iterator: @@ -170,3 +177,18 @@ class Model(MutableMapping): new = type(self)(sync_id=sync_id) new.update(self) return new + + + @contextmanager + def batch_remove(self): + try: + self._active_batch_remove_indice = [] + yield None + finally: + indice = self._active_batch_remove_indice + groups = [list(group) for item, group in itertools.groupby(indice)] + + for grp in groups: + ModelItemDeleted(self.sync_id, index=grp[0], count=len(grp)) + + self._active_batch_remove_indice = None diff --git a/src/backend/models/proxy.py b/src/backend/models/proxy.py index 5e0c0c8b..e9814afe 100644 --- a/src/backend/models/proxy.py +++ b/src/backend/models/proxy.py @@ -43,6 +43,7 @@ class ModelProxy(Model): def source_cleared(self, source: Model) -> None: if self.accept_source(source): - for source_sync_id, key in self.copy(): - if source_sync_id == source.sync_id: - del self[source_sync_id, key] + with self.batch_remove(): + for source_sync_id, key in self.copy(): + if source_sync_id == source.sync_id: + del self[source_sync_id, key] diff --git a/src/backend/pyotherside_events.py b/src/backend/pyotherside_events.py index a7233662..edd08648 100644 --- a/src/backend/pyotherside_events.py +++ b/src/backend/pyotherside_events.py @@ -82,6 +82,7 @@ class ModelItemDeleted(ModelEvent): """Indicate the removal of a `ModelItem` from a `Backend` `Model`.""" index: int = field() + count: int = 1 @dataclass diff --git a/src/gui/PythonBridge/Privates/EventHandlers.qml b/src/gui/PythonBridge/Privates/EventHandlers.qml index 6c5b3aa8..b81bd6aa 100644 --- a/src/gui/PythonBridge/Privates/EventHandlers.qml +++ b/src/gui/PythonBridge/Privates/EventHandlers.qml @@ -24,6 +24,7 @@ QtObject { const onError = Globals.pendingCoroutines[uuid].onError delete Globals.pendingCoroutines[uuid] + Globals.pendingCoroutinesChanged() if (error) { const type = py.getattr(py.getattr(error, "__class__"), "__name__") @@ -43,6 +44,7 @@ QtObject { function onLoopException(message, error, traceback) { + if (traceback.includes("429, None")) return // No need to log these here, the asyncio exception handler does it const type = py.getattr(py.getattr(error, "__class__"), "__name__") utils.showError(type, traceback, message) @@ -68,9 +70,10 @@ QtObject { } - function onModelItemDeleted(syncId, index) { - // print("delete", syncId, index) - ModelStore.get(syncId).remove(index) + function onModelItemDeleted(syncId, index, count=1) { + // print("delete", syncId, index, count) + print(syncId, index, count) + ModelStore.get(syncId).remove(index, count) } diff --git a/src/gui/PythonBridge/PythonBridge.qml b/src/gui/PythonBridge/PythonBridge.qml index 8c82107a..069ab48c 100644 --- a/src/gui/PythonBridge/PythonBridge.qml +++ b/src/gui/PythonBridge/PythonBridge.qml @@ -26,6 +26,7 @@ Python { const future = privates.makeFuture() Globals.pendingCoroutines[uuid] = {future, onSuccess, onError} + Globals.pendingCoroutinesChanged() call("BRIDGE.call_backend_coro", [name, uuid, args], pyFuture => { future.privates.pythonFuture = pyFuture @@ -43,6 +44,7 @@ Python { const uuid = accountId + "." + name + "." + CppUtils.uuid() Globals.pendingCoroutines[uuid] = {onSuccess, onError} + Globals.pendingCoroutinesChanged() const call_args = [accountId, name, uuid, args]