Add Sorted Containers to replace blist

packaging/flatpak/mirage.flatpak.yaml

@@ -605,6 +605,15 @@ modules:
   - type: file
     url: https://files.pythonhosted.org/packages/6f/8f/457f4a5390eeae1cc3aeab89deb7724c965be841ffca6cfca9197482e470/soupsieve-2.0.1-py3-none-any.whl
     sha256: 1634eea42ab371d3d346309b93df7870a88610f0725d47528be902a0d95ecc55
+- name: python3-sortedcontainers
+  buildsystem: simple
+  build-commands:
+  - pip3 install --exists-action=i --no-index --find-links="file://${PWD}" --prefix=${FLATPAK_DEST}
+    "sortedcontainers==2.2.2"
+  sources:
+  - type: file
+    url: https://files.pythonhosted.org/packages/23/8c/22a47a4bf8c5289e4ed946d2b0e4df62bca385b9599cc1e46878f2e2529c/sortedcontainers-2.2.2-py2.py3-none-any.whl
+    sha256: c633ebde8580f241f274c1f8994a665c0e54a17724fecd0cae2f079e09c36d3f
 - name: python3-tinycss2
   buildsystem: simple
   build-commands:

requirements.txt

@@ -1,16 +1,17 @@
-Pillow          >= 7.0.0,  < 8
+Pillow            >= 7.0.0,  < 8
-aiofiles        >= 0.4.0,  < 0.5
+aiofiles          >= 0.4.0,  < 0.5
-appdirs         >= 1.4.4,  < 2
+appdirs           >= 1.4.4,  < 2
-cairosvg        >= 2.4.2,  < 3
+cairosvg          >= 2.4.2,  < 3
-filetype        >= 1.0.7,  < 2
+filetype          >= 1.0.7,  < 2
-html_sanitizer  >= 1.9.1,  < 2
+html_sanitizer    >= 1.9.1,  < 2
-lxml            >= 4.5.1,  < 5
+lxml              >= 4.5.1,  < 5
-matrix-nio[e2e] >= 0.15.0, < 0.16
+matrix-nio[e2e]   >= 0.15.0, < 0.16
-mistune         >= 0.8.4,  < 0.9
+mistune           >= 0.8.4,  < 0.9
-pymediainfo     >= 4.2.1,  < 5
+pymediainfo       >= 4.2.1,  < 5
-plyer           >= 1.4.3,  < 2
+plyer             >= 1.4.3,  < 2
-dbus-python     >= 1.2.16, < 2; platform_system == "Linux"
+sortedcontainers  >= 2.2.2,  < 3
+dbus-python       >= 1.2.16, < 2; platform_system == "Linux"
 
-async_generator >= 1.10,  < 2;   python_version < "3.7"
+async_generator   >= 1.10,  < 2;   python_version < "3.7"
-dataclasses     >= 0.6,   < 0.7; python_version < "3.7"
+dataclasses       >= 0.6,   < 0.7; python_version < "3.7"
-pyfastcopy      >= 1.0.3, < 2;   python_version < "3.8"
+pyfastcopy        >= 1.0.3, < 2;   python_version < "3.8"

src/backend/models/items.py

@@ -39,7 +39,7 @@ class PingStatus(AutoStrEnum):
     Failed  = auto()
 
 
-@dataclass
+@dataclass(eq=False)
 class Homeserver(ModelItem):
     """A homeserver we can connect to. The `id` field is the server's URL."""
 
@@ -55,7 +55,7 @@ class Homeserver(ModelItem):
         return (self.name.lower(), self.id) < (other.name.lower(), other.id)
 
 
-@dataclass
+@dataclass(eq=False)
 class Account(ModelItem):
     """A logged in matrix account."""
 
@@ -82,10 +82,10 @@ class Account(ModelItem):
 
     def __lt__(self, other: "Account") -> bool:
         """Sort by order, then by user ID."""
-        return (self.order, self.id.lower()) < (other.order, other.id.lower())
+        return (self.order, self.id) < (other.order, other.id)
 
 
-@dataclass
+@dataclass(eq=False)
 class Room(ModelItem):
     """A matrix room we are invited to, are or were member of."""
 
@@ -149,14 +149,16 @@ class Room(ModelItem):
                 self.for_account,
                 other.bookmarked,
                 self.left,
-                other.inviter_id,
+                bool(other.inviter_id),
                 (self.display_name or self.id).lower(),
+                self.id,
             ) < (
                 other.for_account,
                 self.bookmarked,
                 other.left,
-                self.inviter_id,
+                bool(self.inviter_id),
                 (other.display_name or other.id).lower(),
+                other.id,
             )
 
         # Left rooms may still have an inviter_id, so check left first.
@@ -164,29 +166,31 @@ class Room(ModelItem):
             self.for_account,
             other.bookmarked,
             self.left,
-            other.inviter_id,
+            bool(other.inviter_id),
             bool(other.highlights),
             bool(other.local_highlights),
             bool(other.unreads),
             bool(other.local_unreads),
             other.last_event_date,
             (self.display_name or self.id).lower(),
+            self.id,
 
         ) < (
             other.for_account,
             self.bookmarked,
             other.left,
-            self.inviter_id,
+            bool(self.inviter_id),
             bool(self.highlights),
             bool(self.local_highlights),
             bool(self.unreads),
             bool(self.local_unreads),
             self.last_event_date,
             (other.display_name or other.id).lower(),
+            other.id,
         )
 
 
-@dataclass
+@dataclass(eq=False)
 class AccountOrRoom(Account, Room):
     type:          Union[Type[Account], Type[Room]] = Account
     account_order: int                              = -1
@@ -199,16 +203,18 @@ class AccountOrRoom(Account, Room):
                 other.type is Account,
                 other.bookmarked,
                 self.left,
-                other.inviter_id,
+                bool(other.inviter_id),
                 (self.display_name or self.id).lower(),
+                self.id,
             ) < (
                 other.account_order,
                 other.id if other.type is Account else other.for_account,
                 self.type is Account,
                 self.bookmarked,
                 other.left,
-                self.inviter_id,
+                bool(self.inviter_id),
                 (other.display_name or other.id).lower(),
+                other.id,
             )
 
         return (
@@ -217,13 +223,14 @@ class AccountOrRoom(Account, Room):
             other.type is Account,
             other.bookmarked,
             self.left,
-            other.inviter_id,
+            bool(other.inviter_id),
             bool(other.highlights),
             bool(other.local_highlights),
             bool(other.unreads),
             bool(other.local_unreads),
             other.last_event_date,
             (self.display_name or self.id).lower(),
+            self.id,
 
         ) < (
             other.account_order,
@@ -231,17 +238,18 @@ class AccountOrRoom(Account, Room):
             self.type is Account,
             self.bookmarked,
             other.left,
-            self.inviter_id,
+            bool(self.inviter_id),
             bool(self.highlights),
             bool(self.local_highlights),
             bool(self.unreads),
             bool(self.local_unreads),
             self.last_event_date,
             (other.display_name or other.id).lower(),
+            other.id,
         )
 
 
-@dataclass
+@dataclass(eq=False)
 class Member(ModelItem):
     """A member in a matrix room."""
 
@@ -262,19 +270,19 @@ class Member(ModelItem):
     def __lt__(self, other: "Member") -> bool:
         """Sort by presence, power level, then by display name/user ID."""
 
-        name       = self.display_name or self.id[1:]
-        other_name = other.display_name or other.id[1:]
 
         return (
             self.invited,
             other.power_level,
             self.presence,
-            name.lower(),
+            (self.display_name or self.id[1:]).lower(),
+            self.id,
         ) < (
             other.invited,
             self.power_level,
             other.presence,
-            other_name.lower(),
+            (other.display_name or other.id[1:]).lower(),
+            other.id,
         )
 
 
@@ -287,7 +295,7 @@ class UploadStatus(AutoStrEnum):
     Error     = auto()
 
 
-@dataclass
+@dataclass(eq=False)
 class Upload(ModelItem):
     """Represent a running or failed file upload operation."""
 
@@ -310,10 +318,10 @@ class Upload(ModelItem):
     def __lt__(self, other: "Upload") -> bool:
         """Sort by the start date, from newest upload to oldest."""
 
-        return self.start_date > other.start_date
+        return (self.start_date, self.id) > (other.start_date, other.id)
 
 
-@dataclass
+@dataclass(eq=False)
 class Event(ModelItem):
     """A matrix state event or message."""
 
@@ -367,7 +375,7 @@ class Event(ModelItem):
     def __lt__(self, other: "Event") -> bool:
         """Sort by date in descending order, from newest to oldest."""
 
-        return self.date > other.date
+        return (self.date, self.id) > (other.date, other.id)
 
     @staticmethod
     def parse_links(text: str) -> List[str]:
@@ -389,9 +397,9 @@ class Event(ModelItem):
             if lxml.etree.tostring(el) not in ignore
         ]
 
-    def serialize_field(self, field: str) -> Any:
+    def serialized_field(self, field: str) -> Any:
         if field == "source":
             source_dict = asdict(self.source) if self.source else {}
             return json.dumps(source_dict)
 
-        return super().serialize_field(field)
+        return super().serialized_field(field)

src/backend/models/model.py

@@ -2,13 +2,14 @@
 # SPDX-License-Identifier: LGPL-3.0-or-later
 
 import itertools
-from bisect import bisect
 from contextlib import contextmanager
 from threading import RLock
 from typing import (
     TYPE_CHECKING, Any, Dict, Iterator, List, MutableMapping, Optional, Tuple,
 )
 
+from sortedcontainers import SortedList
+
 from ..pyotherside_events import ModelCleared, ModelItemDeleted, ModelItemSet
 from . import SyncId
 
@@ -36,10 +37,10 @@ class Model(MutableMapping):
 
 
     def __init__(self, sync_id: Optional[SyncId]) -> None:
-        self.sync_id:      Optional[SyncId]       = sync_id
+        self.sync_id:      Optional[SyncId]        = sync_id
-        self.write_lock:   RLock                  = RLock()
+        self.write_lock:   RLock                   = RLock()
-        self._data:        Dict[Any, "ModelItem"] = {}
+        self._data:        Dict[Any, "ModelItem"]  = {}
-        self._sorted_data: List["ModelItem"]      = []
+        self._sorted_data: SortedList["ModelItem"] = SortedList()
 
         self.take_items_ownership: bool = True
 
@@ -95,7 +96,7 @@ class Model(MutableMapping):
                             getattr(new, field) != getattr(existing, field)
 
                     if changed:
-                        changed_fields[field] = new.serialize_field(field)
+                        changed_fields[field] = new.serialized_field(field)
 
             # Set parent model on new item
 
@@ -110,8 +111,8 @@ class Model(MutableMapping):
                 index_then = self._sorted_data.index(existing)
                 del self._sorted_data[index_then]
 
-            index_now = bisect(self._sorted_data, new)
+            self._sorted_data.add(new)
-            self._sorted_data.insert(index_now, new)
+            index_now = self._sorted_data.index(new)
 
             # Insert into dict data
 

src/backend/models/model_item.py

@@ -11,16 +11,19 @@ if TYPE_CHECKING:
     from .model import Model
 
 
-@dataclass
+@dataclass(eq=False)
 class ModelItem:
     """Base class for items stored inside a `Model`.
 
     This class must be subclassed and not used directly.
-    All subclasses must be dataclasses.
+    All subclasses must use the `@dataclass(eq=False)` decorator.
 
     Subclasses are also expected to implement `__lt__()`,
     to provide support for comparisons with the `<`, `>`, `<=`, `=>` operators
     and thus allow a `Model` to keep its data sorted.
+
+    Make sure to respect SortedList requirements when implementing `__lt__()`:
+    http://www.grantjenks.com/docs/sortedcontainers/introduction.html#caveats
     """
 
     id: Any = field()
@@ -32,88 +35,92 @@ class ModelItem:
 
 
     def __setattr__(self, name: str, value) -> None:
-        """If this item is in a `Model`, alert it of attribute changes."""
+        self.set_fields(**{name: value})
-
-        if (
-            name == "parent_model" or
-            not self.parent_model or
-            getattr(self, name) == value
-        ):
-            super().__setattr__(name, value)
-            return
-
-        super().__setattr__(name, value)
-        self._notify_parent_model({name: self.serialize_field(name)})
-
-
-    def _notify_parent_model(self, changed_fields: Dict[str, Any]) -> None:
-        parent = self.parent_model
-
-        if not parent or not parent.sync_id or not changed_fields:
-            return
-
-        with parent.write_lock:
-            index_then = parent._sorted_data.index(self)
-            parent._sorted_data.sort()
-            index_now = parent._sorted_data.index(self)
-
-            ModelItemSet(parent.sync_id, index_then, index_now, changed_fields)
-
-        for sync_id, proxy in parent.proxies.items():
-            if sync_id != parent.sync_id:
-                proxy.source_item_set(parent, self.id, self, changed_fields)
 
 
     def __delattr__(self, name: str) -> None:
         raise NotImplementedError()
 
 
-    def serialize_field(self, field: str) -> Any:
-        return serialize_value_for_qml(
-            getattr(self, field),
-            json_list_dicts=True,
-        )
-
-
     @property
     def serialized(self) -> Dict[str, Any]:
         """Return this item as a dict ready to be passed to QML."""
 
         return {
-            name: self.serialize_field(name) for name in dir(self)
+            name: self.serialized_field(name)
-            if not (
+            for name in self.__dataclass_fields__  # type: ignore
-                name.startswith("_") or name in ("parent_model", "serialized")
-            )
         }
 
 
-    def set_fields(self, **fields: Any) -> None:
+    def serialized_field(self, field: str) -> Any:
-        """Set multiple fields's values at once.
+        """Return a field's value in a form suitable for passing to QML."""
 
-        The parent model will be resorted only once, and one `ModelItemSet`
+        value = getattr(self, field)
-        event will be sent informing QML of all the changed fields.
+        return serialize_value_for_qml(value, json_list_dicts=True)
+
+
+    def set_fields(self, _force: bool = False, **fields: Any) -> None:
+        """Set one or more field's value and call `ModelItem.notify_change`.
+
+        For efficiency, to change multiple fields, this method should be
+        used rather than setting them one after another with `=` or `setattr`.
         """
 
-        for name, value in fields.copy().items():
+        parent = self.parent_model
-            if getattr(self, name) == value:
-                del fields[name]
-            else:
-                super().__setattr__(name, value)
-                fields[name] = self.serialize_field(name)
 
-        self._notify_parent_model(fields)
+        # If we're currently being created or haven't been put in a model yet:
+        if not parent:
+            for name, value in fields.items():
+                super().__setattr__(name, value)
+            return
+
+        with parent.write_lock:
+            qml_changes = {}
+            changes     = {
+                name: value for name, value in fields.items()
+                if _force or getattr(self, name) != value
+            }
+
+            if not changes:
+                return
+
+            # To avoid corrupting the SortedList, we have to take out the item,
+            # apply the field changes, *then* add it back in.
+
+            index_then = parent._sorted_data.index(self)
+            del parent._sorted_data[index_then]
+
+            for name, value in changes.items():
+                super().__setattr__(name, value)
+
+                if name in self.__dataclass_fields__:  # type: ignore
+                    qml_changes[name] = self.serialized_field(name)
+
+            parent._sorted_data.add(self)
+            index_now = parent._sorted_data.index(self)
+
+            # Now, inform QML about changed dataclass fields if any.
+
+            if not parent.sync_id or not qml_changes:
+                return
+
+            ModelItemSet(parent.sync_id, index_then, index_now, qml_changes)
+
+        # Inform any proxy connected to the parent model of the field changes
+
+        for sync_id, proxy in parent.proxies.items():
+            if sync_id != parent.sync_id:
+                proxy.source_item_set(parent, self.id, self, qml_changes)
 
 
     def notify_change(self, *fields: str) -> None:
-        """Manually notify the parent model that a field changed.
+        """Notify the parent model that fields of this item have changed.
 
-        The model cannot automatically detect changes inside object
+        The model cannot automatically detect changes inside
-        fields, such as list or dicts having their data modified.
+        object fields, such as list or dicts having their data modified.
-
+        In these cases, this method should be called.
-        Use this method to manually notify it that such fields were changed,
-        and avoid having to reassign the field itself.
         """
 
-        self._notify_parent_model({
+        kwargs           = {name: getattr(self, name) for name in fields}
-            name: self.serialize_field(name) for name in fields
+        kwargs["_force"] = True
-        })
+        self.set_fields(**kwargs)