From ecba7c47ecf6019491e5a46234e8e519ab4178d9 Mon Sep 17 00:00:00 2001 From: miruka Date: Fri, 5 Jul 2019 02:45:30 -0400 Subject: [PATCH] Past events loading (with two bugs) --- TODO.md | 5 +- src/python/backend.py | 6 ++- src/python/matrix_client.py | 79 +++++++++++++++++++++-------- src/qml/Chat/Timeline/EventList.qml | 18 +++++-- src/qml/EventHandlers/rooms.js | 4 +- 5 files changed, 83 insertions(+), 29 deletions(-) diff --git a/TODO.md b/TODO.md index 58bae277..e0465c13 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,7 @@ - license headers - replace "property var" by "property " where applicable - [debug mode](https://docs.python.org/3/library/asyncio-dev.html) +- `pyotherside.atexit()` ideas (^/v) messages unread + messages still sending @@ -17,7 +18,9 @@ OLD - Don't bake in size properties for components - Bug fixes - - 100% CPU usage when hitting top edge to trigger messages loading + - Past events loading (limit 100) freezes the GUI - need to move upsert func + to a WorkerScript + - Past events loading: text binding loop on name request - `MessageDelegate.qml:63: TypeError: 'reloadPreviousItem' not a function` - UI diff --git a/src/python/backend.py b/src/python/backend.py index f7cbf711..a77dfbd3 100644 --- a/src/python/backend.py +++ b/src/python/backend.py @@ -2,7 +2,7 @@ import asyncio import json import random from pathlib import Path -from typing import Dict, Optional, Tuple +from typing import Dict, Optional, Set, Tuple from atomicfile import AtomicFile @@ -18,8 +18,12 @@ CONFIG_LOCK = asyncio.Lock() class Backend: def __init__(self, app: App) -> None: self.app = app + self.clients: Dict[str, MatrixClient] = {} + self.past_tokens: Dict[str, str] = {} # {room_id: token} + self.fully_loaded_rooms: Set[str] = set() # {room_id} + def __repr__(self) -> str: return f"{type(self).__name__}(clients={self.clients!r})" diff --git a/src/python/matrix_client.py b/src/python/matrix_client.py index 2a5924d1..80201495 100644 --- a/src/python/matrix_client.py +++ b/src/python/matrix_client.py @@ -114,7 +114,7 @@ class MatrixClient(nio.AsyncClient): response = await self.get_profile(user_id) if isinstance(response, nio.ProfileGetError): - log.warning("Error getting profile for %r: %s", user_id, response) + log.warning("%s: %s", user_id, response) users.UserUpdated( user_id = user_id, @@ -124,6 +124,11 @@ class MatrixClient(nio.AsyncClient): ) + @property + def all_rooms(self) -> Dict[str, MatrixRoom]: + return {**self.invited_rooms, **self.rooms} + + async def send_markdown(self, room_id: str, text: str) -> None: content = { "body": text, @@ -149,6 +154,35 @@ class MatrixClient(nio.AsyncClient): log.error("Failed to send message: %s", response) + async def load_past_events(self, room_id: str, limit: int = 100) -> bool: + if room_id in self.backend.fully_loaded_rooms: + return False + + response = await self.room_messages( + room_id = room_id, + start = self.backend.past_tokens[room_id], + limit = limit, + ) + + more_to_load = True + print(len(response.chunk)) + + if self.backend.past_tokens[room_id] == response.end: + self.backend.fully_loaded_rooms.add(room_id) + more_to_load = False + + self.backend.past_tokens[room_id] = response.end + + for event in response.chunk: + for cb in self.event_callbacks: + if (cb.filter is None or isinstance(event, cb.filter)): + await cb.func( + self.all_rooms[room_id], event, from_past=True + ) + + return more_to_load + + # Callbacks for nio responses @staticmethod @@ -176,9 +210,12 @@ class MatrixClient(nio.AsyncClient): inviter = room.inviter or "", ) - for room_id, _ in resp.rooms.join.items(): + for room_id, info in resp.rooms.join.items(): room = self.rooms[room_id] + if room_id not in self.backend.past_tokens: + self.backend.past_tokens[room_id] = info.timeline.prev_batch + rooms.RoomUpdated( user_id = self.user_id, category = "Rooms", @@ -208,7 +245,9 @@ class MatrixClient(nio.AsyncClient): # %S = sender's displayname # %T = target (ev.state_key)'s displayname - async def onRoomMessageText(self, room, ev) -> None: + # pylint: disable=unused-argument + + async def onRoomMessageText(self, room, ev, from_past=False) -> None: co = HTML_FILTER.filter( ev.formatted_body if ev.format == "org.matrix.custom.html" else html.escape(ev.body) @@ -217,26 +256,27 @@ class MatrixClient(nio.AsyncClient): TimelineMessageReceived.from_nio(room, ev, content=co) - async def onRoomCreateEvent(self, room, ev) -> None: + async def onRoomCreateEvent(self, room, ev, from_past=False) -> None: co = "%S allowed users on other matrix servers to join this room." \ if ev.federate else \ "%S blocked users on other matrix servers from joining this room." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomGuestAccessEvent(self, room, ev) -> None: + async def onRoomGuestAccessEvent(self, room, ev, from_past=False) -> None: allowed = "allowed" if ev.guest_access else "forbad" co = f"%S {allowed} guests to join the room." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomJoinRulesEvent(self, room, ev) -> None: + async def onRoomJoinRulesEvent(self, room, ev, from_past=False) -> None: access = "public" if ev.join_rule == "public" else "invite-only" co = f"%S made the room {access}." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomHistoryVisibilityEvent(self, room, ev) -> None: + async def onRoomHistoryVisibilityEvent(self, room, ev, from_past=False + ) -> None: if ev.history_visibility == "shared": to = "all room members" elif ev.history_visibility == "world_readable": @@ -254,7 +294,7 @@ class MatrixClient(nio.AsyncClient): TimelineEventReceived.from_nio(room, ev, content=co) - async def onPowerLevelsEvent(self, room, ev) -> None: + async def onPowerLevelsEvent(self, room, ev, from_past=False) -> None: co = "%S changed the room's permissions." # TODO: improve TimelineEventReceived.from_nio(room, ev, content=co) @@ -322,9 +362,8 @@ class MatrixClient(nio.AsyncClient): return None - async def onRoomMemberEvent(self, room, ev) -> None: - # TODO: ignore for past events - if ev.content["membership"] != "leave": + async def onRoomMemberEvent(self, room, ev, from_past=False) -> None: + if not from_past and ev.content["membership"] != "leave": users.UserUpdated( user_id = ev.state_key, display_name = ev.content["displayname"] or "", @@ -338,40 +377,40 @@ class MatrixClient(nio.AsyncClient): TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomAliasEvent(self, room, ev) -> None: + async def onRoomAliasEvent(self, room, ev, from_past=False) -> None: co = f"%S set the room's main address to {ev.canonical_alias}." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomNameEvent(self, room, ev) -> None: + async def onRoomNameEvent(self, room, ev, from_past=False) -> None: co = f"%S changed the room's name to \"{ev.name}\"." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomTopicEvent(self, room, ev) -> None: + async def onRoomTopicEvent(self, room, ev, from_past=False) -> None: co = f"%S changed the room's topic to \"{ev.topic}\"." TimelineEventReceived.from_nio(room, ev, content=co) - async def onRoomEncryptionEvent(self, room, ev) -> None: + async def onRoomEncryptionEvent(self, room, ev, from_past=False) -> None: co = f"%S turned on encryption for this room." TimelineEventReceived.from_nio(room, ev, content=co) - async def onOlmEvent(self, room, ev) -> None: + async def onOlmEvent(self, room, ev, from_past=False) -> None: co = f"%S hasn't sent your device the keys to decrypt this message." TimelineEventReceived.from_nio(room, ev, content=co) - async def onMegolmEvent(self, room, ev) -> None: - await self.onOlmEvent(room, ev) + async def onMegolmEvent(self, room, ev, from_past=False) -> None: + await self.onOlmEvent(room, ev, from_past=False) - async def onBadEvent(self, room, ev) -> None: + async def onBadEvent(self, room, ev, from_past=False) -> None: co = f"%S sent a malformed event." TimelineEventReceived.from_nio(room, ev, content=co) - async def onUnknownBadEvent(self, room, ev) -> None: + async def onUnknownBadEvent(self, room, ev, from_past=False) -> None: co = f"%S sent an event this client doesn't understand." TimelineEventReceived.from_nio(room, ev, content=co) diff --git a/src/qml/Chat/Timeline/EventList.qml b/src/qml/Chat/Timeline/EventList.qml index 8841cefb..497004a2 100644 --- a/src/qml/Chat/Timeline/EventList.qml +++ b/src/qml/Chat/Timeline/EventList.qml @@ -34,15 +34,23 @@ HRectangle { // reloaded from network. cacheBuffer: height * 6 - // Declaring this "alias" provides the on... signal + // Declaring this as "alias" provides the on... signal property real yPos: visibleArea.yPosition - + property bool canLoad: true property int zz: 0 + onYPosChanged: { - if (chatPage.category != "Invites" && yPos <= 0.1) { + if (chatPage.category != "Invites" && canLoad && yPos <= 0.1) { zz += 1 - print(zz) - //Backend.loadPastEvents(chatPage.roomId) + print(canLoad, zz) + canLoad = false + py.callClientCoro( + chatPage.userId, + "load_past_events", + [chatPage.roomId], + {}, + function(more_to_load) { canLoad = more_to_load } + ) } } } diff --git a/src/qml/EventHandlers/rooms.js b/src/qml/EventHandlers/rooms.js index 2d5e8bcf..59a2916d 100644 --- a/src/qml/EventHandlers/rooms.js +++ b/src/qml/EventHandlers/rooms.js @@ -86,13 +86,13 @@ function onTimelineEventReceived( "senderId": sender_id, "content": content, "isLocalEcho": true - }, 1, 500) + }, 1, 250) if (found.length > 0) { timelines.set(found[0], item) } else { // Multiple clients will emit duplicate events with the same eventId - timelines.upsert({"eventId": event_id}, item, true, 500) + timelines.upsert({"eventId": event_id}, item, true, 250) } }