Build system, messages support and more

src/python/__about__.py

@@ -0,0 +1,12 @@
+"""<SHORTDESC>"""
+
+__pkg_name__    = "harmonyqml"
+__pretty_name__ = "Harmony QML"
+__version__     = "0.1.0"
+__status__      = "Development"
+# __status__    = "Production"
+
+__author__   = "miruka"
+__email__    = "miruka@disroot.org"
+
+__license__  = "GPLv3"

src/python/__init__.py

@@ -0,0 +1 @@
+from .app import APP

src/python/app.py

@@ -0,0 +1,89 @@
+import asyncio
+import signal
+from concurrent.futures import Future
+from pathlib import Path
+from threading import Thread
+from typing import Any, Coroutine, Dict, List, Optional, Sequence
+from uuid import uuid4
+
+from appdirs import AppDirs
+
+from . import __about__
+from .events.app import CoroutineDone, ExitRequested
+
+
+class App:
+    def __init__(self) -> None:
+        self.appdirs = AppDirs(appname=__about__.__pkg_name__, roaming=True)
+
+        from .backend import Backend
+        self.backend = Backend(app=self)
+
+        self.loop        = asyncio.get_event_loop()
+        self.loop_thread = Thread(target=self._loop_starter)
+        self.loop_thread.start()
+
+
+    def is_debug_on(self, cli_flags: Sequence[str] = ()) -> bool:
+        return "-d" in cli_flags or "--debug" in cli_flags
+
+
+    def _loop_starter(self) -> None:
+        asyncio.set_event_loop(self.loop)
+        self.loop.run_forever()
+
+
+    def run_in_loop(self, coro: Coroutine) -> Future:
+        return asyncio.run_coroutine_threadsafe(coro, self.loop)
+
+
+    def _call_coro(self, coro: Coroutine) -> str:
+        uuid = str(uuid4())
+
+        self.run_in_loop(coro).add_done_callback(
+            lambda future: CoroutineDone(uuid=uuid, result=future.result())
+        )
+        return uuid
+
+
+    def call_backend_coro(self,
+                          name:   str,
+                          args:   Optional[List[str]]      = None,
+                          kwargs: Optional[Dict[str, Any]] = None) -> str:
+        return self._call_coro(
+            getattr(self.backend, name)(*args or [], **kwargs or {})
+        )
+
+
+    def call_client_coro(self,
+                         account_id: str,
+                         name:       str,
+                         args:       Optional[List[str]]      = None,
+                         kwargs:     Optional[Dict[str, Any]] = None) -> str:
+        client = self.backend.clients[account_id]
+        return self._call_coro(
+            getattr(client, name)(*args or [], **kwargs or {})
+        )
+
+
+    def pdb(self, additional_data: Sequence = ()) -> None:
+        # pylint: disable=all
+        ad = additional_data
+        rl = self.run_in_loop
+        ba = self.backend
+        cl = self.backend.clients
+        tcl = lambda user: cl[f"@test_{user}:matrix.org"]
+
+        import json
+        jd = lambda obj: print(json.dumps(obj, indent=4, ensure_ascii=False))
+
+        print("\n=> Run `socat readline tcp:127.0.0.1:4444` in a terminal "
+              "to connect to pdb.")
+        import remote_pdb
+        remote_pdb.RemotePdb("127.0.0.1", 4444).set_trace()
+
+
+# Make CTRL-C work again
+signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+APP = App()

src/python/backend.py

@@ -0,0 +1,129 @@
+import asyncio
+import json
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+
+from atomicfile import AtomicFile
+
+from .app import App
+from .events import users
+from .matrix_client import MatrixClient
+
+SavedAccounts = Dict[str, Dict[str, str]]
+CONFIG_LOCK   = asyncio.Lock()
+
+
+class Backend:
+    def __init__(self, app: App) -> None:
+        self.app = app
+        self.clients: Dict[str, MatrixClient] = {}
+
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(clients={self.clients!r})"
+
+
+    # Clients management
+
+    async def login_client(self,
+                           user:       str,
+                           password:   str,
+                           device_id:  Optional[str] = None,
+                           homeserver: str = "https://matrix.org") -> str:
+        client = MatrixClient(
+            user=user, homeserver=homeserver, device_id=device_id
+        )
+        await client.login(password)
+        self.clients[client.user_id] = client
+        users.AccountUpdated(client.user_id)
+        return client.user_id
+
+
+    async def resume_client(self,
+                            user_id:    str,
+                            token:      str,
+                            device_id:  str,
+                            homeserver: str = "https://matrix.org") -> None:
+        client = MatrixClient(
+            user=user_id, homeserver=homeserver, device_id=device_id
+        )
+        await client.resume(user_id=user_id, token=token, device_id=device_id)
+        self.clients[client.user_id] = client
+        users.AccountUpdated(client.user_id)
+
+
+    async def logout_client(self, user_id: str) -> None:
+        client = self.clients.pop(user_id, None)
+        if client:
+            await client.logout()
+            users.AccountDeleted(user_id)
+
+
+    async def logout_all_clients(self) -> None:
+        await asyncio.gather(*(
+            self.logout_client(user_id) for user_id in self.clients.copy()
+        ))
+
+
+    # Saved account operations - TODO: Use aiofiles?
+
+    @property
+    def saved_accounts_path(self) -> Path:
+        return Path(self.app.appdirs.user_config_dir) / "accounts.json"
+
+
+    @property
+    def saved_accounts(self) -> SavedAccounts:
+        try:
+            return json.loads(self.saved_accounts_path.read_text())
+        except (json.JSONDecodeError, FileNotFoundError):
+            return {}
+
+
+    async def has_saved_accounts(self) -> bool:
+        return bool(self.saved_accounts)
+
+
+    async def load_saved_accounts(self) -> Tuple[str, ...]:
+        async def resume(user_id: str, info: Dict[str, str]) -> str:
+            await self.resume_client(
+                user_id    = user_id,
+                token      = info["token"],
+                device_id  = info["device_id"],
+                homeserver = info["homeserver"],
+            )
+            return user_id
+
+        return await asyncio.gather(*(
+            resume(uid, info) for uid, info in self.saved_accounts.items()
+        ))
+
+
+    async def save_account(self, user_id: str) -> None:
+        client = self.clients[user_id]
+
+        await self._write_config({
+            **self.saved_accounts,
+            client.user_id: {
+                "homeserver": client.homeserver,
+                "token":      client.access_token,
+                "device_id":  client.device_id,
+            }
+        })
+
+
+    async def forget_account(self, user_id: str) -> None:
+        await self._write_config({
+            uid: info
+            for uid, info in self.saved_accounts.items() if uid != user_id
+        })
+
+
+    async def _write_config(self, accounts: SavedAccounts) -> None:
+        js = json.dumps(accounts, indent=4, ensure_ascii=False, sort_keys=True)
+
+        async with CONFIG_LOCK:
+            self.saved_accounts_path.parent.mkdir(parents=True, exist_ok=True)
+
+            with AtomicFile(self.saved_accounts_path, "w") as new:
+                new.write(js)

src/python/events/app.py

@@ -0,0 +1,16 @@
+from typing import Any
+
+from dataclasses import dataclass, field
+
+from .event import Event
+
+
+@dataclass
+class ExitRequested(Event):
+    exit_code: int = 0
+
+
+@dataclass
+class CoroutineDone(Event):
+    uuid:   str = field()
+    result: Any = None

src/python/events/event.py

@@ -0,0 +1,23 @@
+from enum import Enum
+
+from dataclasses import dataclass
+
+import pyotherside
+
+
+class AutoStrEnum(Enum):
+    @staticmethod
+    def _generate_next_value_(name, *_):
+        return name
+
+
+@dataclass
+class Event:
+    def __post_init__(self) -> None:
+        # CPython >= 3.6 or any Python >= 3.7 needed for correct dict order
+        args = [
+            # pylint: disable=no-member
+            getattr(self, field)
+            for field in self.__dataclass_fields__  # type: ignore
+        ]
+        pyotherside.send(type(self).__name__, *args)

src/python/events/rooms.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+from typing import Dict, Optional
+
+from dataclasses import dataclass, field
+
+from .event import Event
+
+
+@dataclass
+class RoomUpdated(Event):
+    user_id:         str                = field()
+    category:        str                = field()
+    room_id:         str                = field()
+    display_name:    Optional[str]      = None
+    avatar_url:      Optional[str]      = None
+    topic:           Optional[str]      = None
+    last_event_date: Optional[datetime] = None
+
+    inviter:    Optional[str]            = None
+    left_event: Optional[Dict[str, str]] = None
+
+
+@dataclass
+class RoomDeleted(Event):
+    user_id:  str = field()
+    category: str = field()
+    room_id:  str = field()
+
+
+@dataclass
+class RoomMemberUpdated(Event):
+    room_id: str  = field()
+    user_id: str  = field()
+    typing:  bool = field()
+
+
+@dataclass
+class RoomMemberDeleted(Event):
+    room_id: str  = field()
+    user_id: str  = field()

src/python/events/rooms_timeline.py

@@ -0,0 +1,32 @@
+from datetime import datetime
+from enum import auto
+
+from dataclasses import dataclass, field
+
+from .event import AutoStrEnum, Event
+
+
+class EventType(AutoStrEnum):
+    text     = auto()
+    html     = auto()
+    file     = auto()
+    image    = auto()
+    audio    = auto()
+    video    = auto()
+    location = auto()
+    notice   = auto()
+
+
+@dataclass
+class TimelineEvent(Event):
+    type:          EventType = field()
+    room_id:       str       = field()
+    event_id:      str       = field()
+    sender_id:     str       = field()
+    date:          datetime  = field()
+    is_local_echo: bool      = field()
+
+
+@dataclass
+class HtmlMessageReceived(TimelineEvent):
+    content: str = field()

src/python/events/users.py

@@ -0,0 +1,53 @@
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from dataclasses import dataclass, field
+
+from .event import Event
+
+# Logged-in accounts
+
+@dataclass
+class AccountUpdated(Event):
+    user_id: str = field()
+
+
+@dataclass
+class AccountDeleted(Event):
+    user_id: str = field()
+
+
+# Accounts and room members details
+
+@dataclass
+class UserUpdated(Event):
+    user_id:        str           = field()
+    display_name:   Optional[str] = None
+    avatar_url:     Optional[str] = None
+    status_message: Optional[str] = None
+
+
+# Devices
+
+class Trust(Enum):
+    blacklisted = -1
+    undecided = 0
+    trusted = 1
+
+
+@dataclass
+class DeviceUpdated(Event):
+    user_id:        str                = field()
+    device_id:      str                = field()
+    ed25519_key:    str                = field()
+    trust:          Trust              = Trust.undecided
+    display_name:   Optional[str]      = None
+    last_seen_ip:   Optional[str]      = None
+    last_seen_date: Optional[datetime] = None
+
+
+@dataclass
+class DeviceDeleted(Event):
+    user_id:   str = field()
+    device_id: str = field()

src/python/html_filter.py

@@ -0,0 +1,153 @@
+# Copyright 2019 miruka
+# This file is part of harmonyqml, licensed under GPLv3.
+
+import re
+
+import mistune
+from lxml.html import HtmlElement, etree  # nosec
+
+import html_sanitizer.sanitizer as sanitizer
+
+
+class HtmlFilter:
+    link_regexes = [re.compile(r, re.IGNORECASE) for r in [
+        (r"(?P<body>.+://(?P<host>[a-z0-9._-]+)(?:/[/\-_.,a-z0-9%&?;=~]*)?"
+         r"(?:\([/\-_.,a-z0-9%&?;=~]*\))?)"),
+        r"mailto:(?P<body>[a-z0-9._-]+@(?P<host>[a-z0-9_.-]+[a-z]))",
+        r"tel:(?P<body>[0-9+-]+)(?P<host>)",
+        r"(?P<body>magnet:\?xt=urn:[a-z0-9]+:.+)(?P<host>)",
+    ]]
+
+
+    def __init__(self) -> None:
+        self._sanitizer = sanitizer.Sanitizer(self.sanitizer_settings)
+
+        # The whitespace remover doesn't take <pre> into account
+        sanitizer.normalize_overall_whitespace         = lambda html: html
+        sanitizer.normalize_whitespace_in_text_or_tail = lambda el: el
+
+        # hard_wrap: convert all \n to <br> without required two spaces
+        self._markdown_to_html = mistune.Markdown(hard_wrap=True)
+
+
+    def from_markdown(self, text: str) -> str:
+        return self.filter(self._markdown_to_html(text))
+
+
+    def filter(self, html: str) -> str:
+        html = self._sanitizer.sanitize(html)
+        tree = etree.fromstring(html, parser=etree.HTMLParser())
+
+        if tree is None:
+            return ""
+
+        for el in tree.iter("img"):
+            el = self._wrap_img_in_a(el)
+
+        for el in tree.iter("a"):
+            el = self._append_img_to_a(el)
+
+        result = b"".join((etree.tostring(el, encoding="utf-8")
+                           for el in tree[0].iterchildren()))
+
+        return str(result, "utf-8")
+
+
+    @property
+    def sanitizer_settings(self) -> dict:
+        # https://matrix.org/docs/spec/client_server/latest.html#m-room-message-msgtypes
+        return {
+            "tags": {
+                # TODO: mx-reply, audio, video
+                "font", "h1", "h2", "h3", "h4", "h5", "h6",
+                "blockquote", "p", "a", "ul", "ol", "sup", "sub", "li",
+                "b", "i", "s", "u", "code", "hr", "br",
+                "table", "thead", "tbody", "tr", "th", "td",
+                "pre", "img",
+            },
+            "attributes": {
+                # TODO: translate font attrs to qt html subset
+                "font": {"data-mx-bg-color", "data-mx-color"},
+                "a":    {"href"},
+                "img":  {"width", "height", "alt", "title", "src"},
+                "ol":   {"start"},
+                "code": {"class"},
+            },
+            "empty": {"hr", "br", "img"},
+            "separate": {
+                "a", "p", "li", "table", "tr", "th", "td", "br", "hr"
+            },
+            "whitespace": {},
+            "add_nofollow": False,
+            "autolink": {  # FIXME: arg dict not working
+                "link_regexes": self.link_regexes,
+                "avoid_hosts": [],
+            },
+            "sanitize_href": lambda href: href,
+            "element_preprocessors": [
+                sanitizer.bold_span_to_strong,
+                sanitizer.italic_span_to_em,
+                sanitizer.tag_replacer("strong", "b"),
+                sanitizer.tag_replacer("em", "i"),
+                sanitizer.tag_replacer("strike", "s"),
+                sanitizer.tag_replacer("del", "s"),
+                sanitizer.tag_replacer("span", "font"),
+                self._remove_empty_font,
+                sanitizer.tag_replacer("form", "p"),
+                sanitizer.tag_replacer("div", "p"),
+                sanitizer.tag_replacer("caption", "p"),
+                sanitizer.target_blank_noopener,
+            ],
+            "element_postprocessors": [],
+            "is_mergeable": lambda e1, e2: e1.attrib == e2.attrib,
+        }
+
+
+    def _remove_empty_font(self, el: HtmlElement) -> HtmlElement:
+        if el.tag != "font":
+            return el
+
+        if not self.sanitizer_settings["attributes"]["font"] & set(el.keys()):
+            el.clear()
+
+        return el
+
+
+    def _wrap_img_in_a(self, el: HtmlElement) -> HtmlElement:
+        link   = el.attrib.get("src", "")
+        width  = el.attrib.get("width", "256")
+        height = el.attrib.get("height", "256")
+
+        if el.getparent().tag == "a" or el.tag != "img" or \
+           not self._is_image_path(link):
+            return el
+
+        el.tag    = "a"
+        el.attrib.clear()
+        el.attrib["href"] = link
+        el.append(etree.Element("img", src=link, width=width, height=height))
+        return el
+
+
+    def _append_img_to_a(self, el: HtmlElement) -> HtmlElement:
+        link = el.attrib.get("href", "")
+
+        if not (el.tag == "a" and self._is_image_path(link)):
+            return el
+
+        for _ in el.iter("img"):  # if the <a> already has an <img> child
+            return el
+
+        el.append(etree.Element("br"))
+        el.append(etree.Element("img", src=link, width="256", height="256"))
+        return el
+
+
+    @staticmethod
+    def _is_image_path(link: str) -> bool:
+        return bool(re.match(
+            r".+\.(jpg|jpeg|png|gif|bmp|webp|tiff|svg)$", link, re.IGNORECASE
+        ))
+
+
+HTML_FILTER = HtmlFilter()

src/python/matrix_client.py

@@ -0,0 +1,182 @@
+import asyncio
+import inspect
+import logging as log
+import platform
+from contextlib import suppress
+from datetime import datetime
+from types import ModuleType
+from typing import Dict, Optional, Type
+
+import nio
+
+from . import __about__
+from .events import rooms, users
+from .events.rooms_timeline import EventType, HtmlMessageReceived
+from .html_filter import HTML_FILTER
+
+
+class MatrixClient(nio.AsyncClient):
+    def __init__(self,
+                 user:       str,
+                 homeserver: str           = "https://matrix.org",
+                 device_id:  Optional[str] = None) -> None:
+
+        # TODO: ensure homeserver starts with a scheme://
+        self.sync_task: Optional[asyncio.Future] = None
+        super().__init__(homeserver=homeserver, user=user, device_id=device_id)
+
+        self.connect_callbacks()
+
+
+    def __repr__(self) -> str:
+        return "%s(user_id=%r, homeserver=%r, device_id=%r)" % (
+            type(self).__name__, self.user_id, self.homeserver, self.device_id
+        )
+
+
+    @staticmethod
+    def _classes_defined_in(module: ModuleType) -> Dict[str, Type]:
+        return {
+            m[0]: m[1] for m in inspect.getmembers(module, inspect.isclass)
+            if not m[0].startswith("_") and
+            m[1].__module__.startswith(module.__name__)
+        }
+
+
+    def connect_callbacks(self) -> None:
+        for name, class_ in self._classes_defined_in(nio.responses).items():
+            with suppress(AttributeError):
+                self.add_response_callback(getattr(self, f"on{name}"), class_)
+
+        # TODO: get this implemented in AsyncClient
+        # for name, class_ in self._classes_defined_in(nio.events).items():
+            # with suppress(AttributeError):
+                # self.add_event_callback(getattr(self, f"on{name}"), class_)
+
+
+    async def start_syncing(self) -> None:
+        self.sync_task = asyncio.ensure_future(
+            self.sync_forever(timeout=10_000)
+        )
+
+        def callback(task):
+            raise task.exception()
+
+        self.sync_task.add_done_callback(callback)
+
+
+    @property
+    def default_device_name(self) -> str:
+        os_ = f" on {platform.system()}".rstrip()
+        os_ = f"{os_} {platform.release()}".rstrip() if os_ != " on" else ""
+        return f"{__about__.__pretty_name__}{os_}"
+
+
+    async def login(self, password: str) -> None:
+        response = await super().login(password, self.default_device_name)
+
+        if isinstance(response, nio.LoginError):
+            print(response)
+        else:
+            await self.start_syncing()
+
+
+    async def resume(self, user_id: str, token: str, device_id: str) -> None:
+        response = nio.LoginResponse(user_id, device_id, token)
+        await self.receive_response(response)
+        await self.start_syncing()
+
+
+    async def logout(self) -> None:
+        if self.sync_task:
+            self.sync_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self.sync_task
+
+        await self.close()
+
+
+    async def request_user_update_event(self, user_id: str) -> None:
+        response = await self.get_profile(user_id)
+
+        if isinstance(response, nio.ProfileGetError):
+            log.warning("Error getting profile for %r: %s", user_id, response)
+
+        users.UserUpdated(
+            user_id        = user_id,
+            display_name   = getattr(response, "displayname", None),
+            avatar_url     = getattr(response, "avatar_url", None),
+            status_message = None,  # TODO
+        )
+
+
+    # Callbacks for nio responses
+
+    @staticmethod
+    def _get_room_name(room: nio.rooms.MatrixRoom) -> Optional[str]:
+        # FIXME: reimplanted because of nio's non-standard room.display_name
+        name = room.name or room.canonical_alias
+        if name:
+            return name
+
+        name = room.group_name()
+        return None if name == "Empty room?" else name
+
+
+    async def onSyncResponse(self, resp: nio.SyncResponse) -> None:
+        for room_id, info in resp.rooms.invite.items():
+            room: nio.rooms.MatrixRoom = self.invited_rooms[room_id]
+
+            rooms.RoomUpdated(
+                user_id      = self.user_id,
+                category     = "Invites",
+                room_id      = room_id,
+                display_name = self._get_room_name(room),
+                avatar_url   = room.gen_avatar_url,
+                topic        = room.topic,
+                inviter      = room.inviter,
+            )
+
+        for room_id, info in resp.rooms.join.items():
+            room = self.rooms[room_id]
+
+            rooms.RoomUpdated(
+                user_id      = self.user_id,
+                category     = "Rooms",
+                room_id      = room_id,
+                display_name = self._get_room_name(room),
+                avatar_url   = room.gen_avatar_url,
+                topic        = room.topic,
+            )
+
+            asyncio.gather(*(
+                getattr(self, f"on{type(ev).__name__}")(room_id, ev)
+                for ev in info.timeline.events
+                if hasattr(self, f"on{type(ev).__name__}")
+            ))
+
+        for room_id, info in resp.rooms.leave.items():
+            rooms.RoomUpdated(
+                user_id  = self.user_id,
+                category = "Left",
+                room_id  = room_id,
+                # left_event TODO
+            )
+
+
+    # Callbacks for nio events
+
+    async def onRoomMessageText(self, room_id: str, ev: nio.RoomMessageText
+                               ) -> None:
+        is_html = ev.format == "org.matrix.custom.html"
+        filter_ = HTML_FILTER.filter
+
+        HtmlMessageReceived(
+            type          = EventType.html if is_html else EventType.text,
+            room_id       = room_id,
+            event_id      = ev.event_id,
+            sender_id     = ev.sender,
+            date          = datetime.fromtimestamp(ev.server_timestamp / 1000),
+            is_local_echo = False,
+            content       = filter_(ev.formatted_body) if is_html else ev.body,
+        )