From a0f9acddaa99ea876470daccbcc1c882f14474e3 Mon Sep 17 00:00:00 2001 From: miruka Date: Wed, 17 Apr 2019 21:08:32 -0400 Subject: [PATCH] Load past events when scrolling up Also keep two nio clients internally: one for loop-syncing, one for the rest; since they can't handle more than one talk at a time. --- TODO.md | 2 - harmonyqml/backend/backend.py | 15 +++++ harmonyqml/backend/client.py | 65 ++++++++++++++++++---- harmonyqml/backend/html_filter.py | 6 +- harmonyqml/backend/network_manager.py | 65 +++++++++++++--------- harmonyqml/backend/signal_manager.py | 15 +++++ harmonyqml/components/chat/MessageList.qml | 10 +--- 7 files changed, 129 insertions(+), 49 deletions(-) diff --git a/TODO.md b/TODO.md index 9202c814..6428bfa2 100644 --- a/TODO.md +++ b/TODO.md @@ -17,8 +17,6 @@ - Use Loader? for MessageDelegate to show sub-components based on condition - Better names and organization for the Message components -- Load previous events on scroll up - - Migrate more JS functions to their own files - Accept room\_id arg for getUser diff --git a/harmonyqml/backend/backend.py b/harmonyqml/backend/backend.py index 490b434f..3398f7be 100644 --- a/harmonyqml/backend/backend.py +++ b/harmonyqml/backend/backend.py @@ -15,6 +15,8 @@ from .html_filter import HtmlFilter class Backend(QObject): def __init__(self) -> None: super().__init__() + self.past_tokens: Dict[str, str] = {} + self._client_manager: ClientManager = ClientManager() self._models: QMLModels = QMLModels() self._html_filter: HtmlFilter = HtmlFilter() @@ -56,3 +58,16 @@ class Backend(QObject): # pylint:disable=no-self-use md5 = hashlib.md5(bytes(string, "utf-8")).hexdigest() return float("0.%s" % int(md5[-10:], 16)) + + + @pyqtSlot(str) + def loadPastEvents(self, room_id: str) -> None: + if not room_id in self.past_tokens: + return # Initial sync not done yet + + for client in self.clientManager.clients.values(): + if room_id in client.nio.rooms: + client.loadPastEvents(room_id, self.past_tokens[room_id]) + break + else: + raise ValueError(f"Room not found in any client: {room_id}") diff --git a/harmonyqml/backend/client.py b/harmonyqml/backend/client.py index 67775f7e..6ae81db5 100644 --- a/harmonyqml/backend/client.py +++ b/harmonyqml/backend/client.py @@ -14,6 +14,8 @@ from PyQt5.QtCore import QObject, pyqtProperty, pyqtSignal, pyqtSlot import nio import nio.responses as nr +from .network_manager import NetworkManager + # One pool per hostname/remote server; # multiple Client for different accounts on the same server can exist. _POOLS: DefaultDict[str, ThreadPoolExecutor] = \ @@ -37,11 +39,13 @@ def futurize(func: Callable) -> Callable: class Client(QObject): - roomInvited = pyqtSignal(str) - roomJoined = pyqtSignal(str) - roomLeft = pyqtSignal(str) - roomEventReceived = pyqtSignal(str, str, dict) - roomTypingUsersUpdated = pyqtSignal(str, list) + roomInvited = pyqtSignal(str) + roomJoined = pyqtSignal(str) + roomLeft = pyqtSignal(str) + roomSyncPrevBatchTokenReceived = pyqtSignal(str, str) + roomPastPrevBatchTokenReceived = pyqtSignal(str, str) + roomEventReceived = pyqtSignal(str, str, dict) + roomTypingUsersUpdated = pyqtSignal(str, list) def __init__(self, hostname: str, username: str, device_id: str = "" @@ -52,13 +56,18 @@ class Client(QObject): self.host: str = host self.port: int = int(port[0]) if port else 443 + self.pool: ThreadPoolExecutor = _POOLS[self.host] + self.nio: nio.client.HttpClient = \ nio.client.HttpClient(self.host, username, device_id) - self.pool: ThreadPoolExecutor = _POOLS[self.host] + # Since nio clients can't handle more than one talk operation + # at a time, this one is used exclusively to poll the sync API + self.nio_sync: nio.client.HttpClient = \ + nio.client.HttpClient(self.host, username, device_id) - from .network_manager import NetworkManager - self.net: NetworkManager = NetworkManager(self) + self.net = NetworkManager(self.host, self.port, self.nio) + self.net_sync = NetworkManager(self.host, self.port, self.nio_sync) self._stop_sync: Event = Event() @@ -78,7 +87,10 @@ class Client(QObject): @futurize def login(self, password: str, device_name: str = "") -> None: self.net.write(self.nio.connect()) - self.net.talk(self.nio.login, password, device_name) + response = self.net.talk(self.nio.login, password, device_name) + + self.net_sync.write(self.nio_sync.connect()) + self.nio_sync.receive_response(response) self.startSyncing() @@ -89,6 +101,9 @@ class Client(QObject): self.net.write(self.nio.connect()) response = nr.LoginResponse(user_id, device_id, token) self.nio.receive_response(response) + + self.net_sync.write(self.nio_sync.connect()) + self.nio_sync.receive_response(response) self.startSyncing() @@ -97,13 +112,16 @@ class Client(QObject): def logout(self) -> None: self._stop_sync.set() self.net.write(self.nio.disconnect()) + self.net_sync.write(self.nio_sync.disconnect()) @pyqtSlot() @futurize def startSyncing(self) -> None: while True: - self._on_sync(self.net.talk(self.nio.sync, timeout=10_000)) + self._on_sync(self.net_sync.talk( + self.nio_sync.sync, timeout=10_000 + )) if self._stop_sync.is_set(): self._stop_sync.clear() @@ -111,12 +129,18 @@ class Client(QObject): def _on_sync(self, response: nr.SyncResponse) -> None: + self.nio.receive_response(response) + for room_id in response.rooms.invite: self.roomInvited.emit(room_id) for room_id, room_info in response.rooms.join.items(): self.roomJoined.emit(room_id) + self.roomSyncPrevBatchTokenReceived.emit( + room_id, room_info.timeline.prev_batch + ) + for ev in room_info.timeline.events: self.roomEventReceived.emit( room_id, type(ev).__name__, ev.__dict__ @@ -130,3 +154,24 @@ class Client(QObject): for room_id in response.rooms.leave: self.roomLeft.emit(room_id) + + + @futurize + def loadPastEvents(self, room_id: str, start_token: str) -> None: + # From QML, use Backend.loastPastEvents instead + self._on_past_events( + room_id, + self.net.talk( + self.nio.room_messages, room_id, start=start_token, limit=100 + ) + ) + + + def _on_past_events(self, room_id: str, response: nr.RoomMessagesResponse + ) -> None: + self.roomPastPrevBatchTokenReceived.emit(room_id, response.end) + + for ev in response.chunk: + self.roomEventReceived.emit( + room_id, type(ev).__name__, ev.__dict__ + ) diff --git a/harmonyqml/backend/html_filter.py b/harmonyqml/backend/html_filter.py index 11584d96..64d20a1f 100644 --- a/harmonyqml/backend/html_filter.py +++ b/harmonyqml/backend/html_filter.py @@ -41,11 +41,11 @@ class HtmlFilter(QObject): @pyqtSlot(str, result=str) def filter(self, html: str) -> str: html = self._sanitizer.sanitize(html) - if not html: - return "" - tree = etree.fromstring(html, parser=etree.HTMLParser()) + if tree is None: + return "" + for el in tree.iter("img"): el = self._wrap_img_in_a(el) diff --git a/harmonyqml/backend/network_manager.py b/harmonyqml/backend/network_manager.py index f1319249..26753a44 100644 --- a/harmonyqml/backend/network_manager.py +++ b/harmonyqml/backend/network_manager.py @@ -5,13 +5,13 @@ import logging import socket import ssl import time +from threading import Lock from typing import Callable, Optional, Tuple from uuid import UUID +import nio import nio.responses as nr -from .client import Client - OptSock = Optional[ssl.SSLSocket] NioRequestFunc = Callable[..., Tuple[UUID, bytes]] @@ -26,16 +26,21 @@ class NetworkManager: http_retry_codes = {408, 429, 500, 502, 503, 504, 507} - def __init__(self, client: Client) -> None: - self.client = client + def __init__(self, host: str, port: int, nio_client: nio.client.HttpClient + ) -> None: + self.host = host + self.port = port + self.nio = nio_client + self._ssl_context: ssl.SSLContext = ssl.create_default_context() self._ssl_session: Optional[ssl.SSLSession] = None + self._lock: Lock = Lock() def _get_socket(self) -> ssl.SSLSocket: sock = self._ssl_context.wrap_socket( # type: ignore - socket.create_connection((self.client.host, self.client.port)), - server_hostname = self.client.host, + socket.create_connection((self.host, self.port)), + server_hostname = self.host, session = self._ssl_session, ) self._ssl_session = self._ssl_session or sock.session @@ -53,8 +58,12 @@ class NetworkManager: response = None while not response: - self.client.nio.receive(sock.recv(4096)) - response = self.client.nio.next_response() + left_to_send = self.nio.data_to_send() + if left_to_send: + self.write(left_to_send, sock) + + self.nio.receive(sock.recv(4096)) + response = self.nio.next_response() if isinstance(response, nr.ErrorResponse): raise NioErrorResponse(response) @@ -66,6 +75,9 @@ class NetworkManager: def write(self, data: bytes, with_sock: OptSock = None) -> None: + if not data: + return + sock = with_sock or self._get_socket() sock.sendall(data) @@ -74,29 +86,30 @@ class NetworkManager: def talk(self, nio_func: NioRequestFunc, *args, **kwargs) -> nr.Response: - while True: - to_send = nio_func(*args, **kwargs)[1] - sock = self._get_socket() + with self._lock: + while True: + to_send = nio_func(*args, **kwargs)[1] + sock = self._get_socket() - try: - self.write(to_send, sock) - response = self.read(sock) + try: + self.write(to_send, sock) + response = self.read(sock) - except NioErrorResponse as err: - logging.error("read bad response for %s: %s", nio_func, err) - self._close_socket(sock) + except NioErrorResponse as err: + logging.error("bad read for %s: %s", nio_func, err) + self._close_socket(sock) - if self._should_abort_talk(err): - logging.error("aborting talk") + if self._should_abort_talk(err): + logging.error("aborting talk") + break + + time.sleep(10) + + else: break - time.sleep(10) - - else: - break - - self._close_socket(sock) - return response + self._close_socket(sock) + return response def _should_abort_talk(self, err: NioErrorResponse) -> bool: diff --git a/harmonyqml/backend/signal_manager.py b/harmonyqml/backend/signal_manager.py index a463f20c..c79b7b89 100644 --- a/harmonyqml/backend/signal_manager.py +++ b/harmonyqml/backend/signal_manager.py @@ -81,6 +81,21 @@ class SignalManager(QObject): del rooms[rooms.indexWhere("room_id", room_id)] + def onRoomSyncPrevBatchTokenReceived( + self, _: Client, room_id: str, token: str + ) -> None: + + if room_id not in self.backend.past_tokens: + self.backend.past_tokens[room_id] = token + + + def onRoomPastPrevBatchTokenReceived( + self, _: Client, room_id: str, token: str + ) -> None: + + self.backend.past_tokens[room_id] = token + + def onRoomEventReceived( self, _: Client, room_id: str, etype: str, edict: Dict[str, Any] ) -> None: diff --git a/harmonyqml/components/chat/MessageList.qml b/harmonyqml/components/chat/MessageList.qml index ba7aa5b5..29081f63 100644 --- a/harmonyqml/components/chat/MessageList.qml +++ b/harmonyqml/components/chat/MessageList.qml @@ -26,14 +26,8 @@ Rectangle { // reloaded from network. cacheBuffer: height * 6 - function goToEnd() { - messageListView.positionViewAtEnd() - //messageListView.flick(0, -messageListView.bottomMargin * 100) + onMovementEnded: if (atYBeginning) { + Backend.loadPastEvents(chatPage.room.room_id) } - - //Connections { - //target: messageListView.model - //onChanged: goToEnd() - //} } }