Past events loading (with two bugs)

TODO.md

@@ -1,6 +1,7 @@
 - license headers
 - replace "property var" by "property <object>" where applicable
 - [debug mode](https://docs.python.org/3/library/asyncio-dev.html)
+- `pyotherside.atexit()`
 
 ideas
 (^/v) messages unread + messages still sending
@@ -17,7 +18,9 @@ OLD
   - Don't bake in size properties for components
 
 - Bug fixes
-  - 100% CPU usage when hitting top edge to trigger messages loading
+  - Past events loading (limit 100) freezes the GUI - need to move upsert func
+    to a WorkerScript
+  - Past events loading: text binding loop on name request
   - `MessageDelegate.qml:63: TypeError: 'reloadPreviousItem' not a function`
 
 - UI

src/python/backend.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import random
 from pathlib import Path
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional, Set, Tuple
 
 from atomicfile import AtomicFile
 
@@ -18,8 +18,12 @@ CONFIG_LOCK   = asyncio.Lock()
 class Backend:
     def __init__(self, app: App) -> None:
         self.app = app
+
         self.clients: Dict[str, MatrixClient] = {}
 
+        self.past_tokens:        Dict[str, str] =  {}    # {room_id: token}
+        self.fully_loaded_rooms: Set[str]       = set()  # {room_id}
+
 
     def __repr__(self) -> str:
         return f"{type(self).__name__}(clients={self.clients!r})"

src/python/matrix_client.py

@@ -114,7 +114,7 @@ class MatrixClient(nio.AsyncClient):
         response = await self.get_profile(user_id)
 
         if isinstance(response, nio.ProfileGetError):
-            log.warning("Error getting profile for %r: %s", user_id, response)
+            log.warning("%s: %s", user_id, response)
 
         users.UserUpdated(
             user_id        = user_id,
@@ -124,6 +124,11 @@ class MatrixClient(nio.AsyncClient):
         )
 
 
+    @property
+    def all_rooms(self) -> Dict[str, MatrixRoom]:
+        return {**self.invited_rooms, **self.rooms}
+
+
     async def send_markdown(self, room_id: str, text: str) -> None:
         content = {
             "body":           text,
@@ -149,6 +154,35 @@ class MatrixClient(nio.AsyncClient):
                 log.error("Failed to send message: %s", response)
 
 
+    async def load_past_events(self, room_id: str, limit: int = 100) -> bool:
+        if room_id in self.backend.fully_loaded_rooms:
+            return False
+
+        response = await self.room_messages(
+            room_id = room_id,
+            start   = self.backend.past_tokens[room_id],
+            limit   = limit,
+        )
+
+        more_to_load = True
+        print(len(response.chunk))
+
+        if self.backend.past_tokens[room_id] == response.end:
+            self.backend.fully_loaded_rooms.add(room_id)
+            more_to_load = False
+
+        self.backend.past_tokens[room_id] = response.end
+
+        for event in response.chunk:
+            for cb in self.event_callbacks:
+                if (cb.filter is None or isinstance(event, cb.filter)):
+                    await cb.func(
+                        self.all_rooms[room_id], event, from_past=True
+                    )
+
+        return more_to_load
+
+
     # Callbacks for nio responses
 
     @staticmethod
@@ -176,9 +210,12 @@ class MatrixClient(nio.AsyncClient):
                 inviter      = room.inviter or "",
             )
 
-        for room_id, _ in resp.rooms.join.items():
+        for room_id, info in resp.rooms.join.items():
             room = self.rooms[room_id]
 
+            if room_id not in self.backend.past_tokens:
+                self.backend.past_tokens[room_id] = info.timeline.prev_batch
+
             rooms.RoomUpdated(
                 user_id      = self.user_id,
                 category     = "Rooms",
@@ -208,7 +245,9 @@ class MatrixClient(nio.AsyncClient):
     # %S = sender's displayname
     # %T = target (ev.state_key)'s displayname
 
-    async def onRoomMessageText(self, room, ev) -> None:
+    # pylint: disable=unused-argument
+
+    async def onRoomMessageText(self, room, ev, from_past=False) -> None:
         co = HTML_FILTER.filter(
             ev.formatted_body
             if ev.format == "org.matrix.custom.html" else html.escape(ev.body)
@@ -217,26 +256,27 @@ class MatrixClient(nio.AsyncClient):
         TimelineMessageReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomCreateEvent(self, room, ev) -> None:
+    async def onRoomCreateEvent(self, room, ev, from_past=False) -> None:
         co = "%S allowed users on other matrix servers to join this room." \
              if ev.federate else \
              "%S blocked users on other matrix servers from joining this room."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomGuestAccessEvent(self, room, ev) -> None:
+    async def onRoomGuestAccessEvent(self, room, ev, from_past=False) -> None:
         allowed = "allowed" if ev.guest_access else "forbad"
         co      = f"%S {allowed} guests to join the room."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomJoinRulesEvent(self, room, ev) -> None:
+    async def onRoomJoinRulesEvent(self, room, ev, from_past=False) -> None:
         access = "public" if ev.join_rule == "public" else "invite-only"
         co     = f"%S made the room {access}."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomHistoryVisibilityEvent(self, room, ev) -> None:
+    async def onRoomHistoryVisibilityEvent(self, room, ev, from_past=False
+                                          ) -> None:
         if ev.history_visibility == "shared":
             to = "all room members"
         elif ev.history_visibility == "world_readable":
@@ -254,7 +294,7 @@ class MatrixClient(nio.AsyncClient):
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onPowerLevelsEvent(self, room, ev) -> None:
+    async def onPowerLevelsEvent(self, room, ev, from_past=False) -> None:
         co = "%S changed the room's permissions."  # TODO: improve
         TimelineEventReceived.from_nio(room, ev, content=co)
 
@@ -322,9 +362,8 @@ class MatrixClient(nio.AsyncClient):
         return None
 
 
-    async def onRoomMemberEvent(self, room, ev) -> None:
+    async def onRoomMemberEvent(self, room, ev, from_past=False) -> None:
-        # TODO: ignore for past events
+        if not from_past and ev.content["membership"] != "leave":
-        if ev.content["membership"] != "leave":
             users.UserUpdated(
                 user_id        = ev.state_key,
                 display_name   = ev.content["displayname"] or "",
@@ -338,40 +377,40 @@ class MatrixClient(nio.AsyncClient):
             TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomAliasEvent(self, room, ev) -> None:
+    async def onRoomAliasEvent(self, room, ev, from_past=False) -> None:
         co = f"%S set the room's main address to {ev.canonical_alias}."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomNameEvent(self, room, ev) -> None:
+    async def onRoomNameEvent(self, room, ev, from_past=False) -> None:
         co = f"%S changed the room's name to \"{ev.name}\"."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomTopicEvent(self, room, ev) -> None:
+    async def onRoomTopicEvent(self, room, ev, from_past=False) -> None:
         co = f"%S changed the room's topic to \"{ev.topic}\"."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onRoomEncryptionEvent(self, room, ev) -> None:
+    async def onRoomEncryptionEvent(self, room, ev, from_past=False) -> None:
         co = f"%S turned on encryption for this room."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onOlmEvent(self, room, ev) -> None:
+    async def onOlmEvent(self, room, ev, from_past=False) -> None:
         co = f"%S hasn't sent your device the keys to decrypt this message."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onMegolmEvent(self, room, ev) -> None:
+    async def onMegolmEvent(self, room, ev, from_past=False) -> None:
-        await self.onOlmEvent(room, ev)
+        await self.onOlmEvent(room, ev, from_past=False)
 
 
-    async def onBadEvent(self, room, ev) -> None:
+    async def onBadEvent(self, room, ev, from_past=False) -> None:
         co = f"%S sent a malformed event."
         TimelineEventReceived.from_nio(room, ev, content=co)
 
 
-    async def onUnknownBadEvent(self, room, ev) -> None:
+    async def onUnknownBadEvent(self, room, ev, from_past=False) -> None:
         co = f"%S sent an event this client doesn't understand."
         TimelineEventReceived.from_nio(room, ev, content=co)

src/qml/Chat/Timeline/EventList.qml

@@ -34,15 +34,23 @@ HRectangle {
         // reloaded from network.
         cacheBuffer: height * 6
 
-        // Declaring this "alias" provides the on... signal
+        // Declaring this as "alias" provides the on... signal
         property real yPos: visibleArea.yPosition
-
+        property bool canLoad: true
         property int zz: 0
+
         onYPosChanged: {
-            if (chatPage.category != "Invites" && yPos <= 0.1) {
+            if (chatPage.category != "Invites" && canLoad && yPos <= 0.1) {
                 zz += 1
-                print(zz)
+                print(canLoad, zz)
-                //Backend.loadPastEvents(chatPage.roomId)
+                canLoad = false
+                py.callClientCoro(
+                    chatPage.userId,
+                    "load_past_events",
+                    [chatPage.roomId],
+                    {},
+                    function(more_to_load) { canLoad = more_to_load }
+                )
             }
         }
     }

src/qml/EventHandlers/rooms.js

@@ -86,13 +86,13 @@ function onTimelineEventReceived(
         "senderId":     sender_id,
         "content":      content,
         "isLocalEcho":  true
-    }, 1, 500)
+    }, 1, 250)
 
     if (found.length > 0) {
         timelines.set(found[0], item)
     } else {
         // Multiple clients will emit duplicate events with the same eventId
-        timelines.upsert({"eventId": event_id}, item, true, 500)
+        timelines.upsert({"eventId": event_id}, item, true, 250)
     }
 }