Rewrite media caching (old image provider)

- Doesn't use pyotherside's image provider feature, for more flexibility
  and simplicity
- Suitable for supporting matrix media events and more later
- Avoid a lot of duplicate files that the old cache created due to
  server not returning what we expect, mistakes in Python/QML code, etc
- Changed file structure
  (e.g. thumbnails/32x32/<mxc id> instead of
   thumbnails/<mxc id>.32.32.crop)

- Backend.wait_until_account_exist: start issuing warnings if the
  function runs for more than 10s, which means in most case a bad user
  ID was passed

- New HMxcImage QML component, used in H(User/Room)Avatar
This commit is contained in:
miruka 2019-11-03 13:48:12 -04:00
parent 55d4035f60
commit 2f19ff493b
20 changed files with 291 additions and 261 deletions

View File

@ -38,12 +38,7 @@ class App:
self.backend = Backend(app=self)
self.debug = False
from .image_provider import ImageProvider
self.image_provider = ImageProvider(self)
pyotherside.set_image_provider(self.image_provider.get)
self.loop = asyncio.get_event_loop()
self.loop = asyncio.get_event_loop()
self.loop_thread = Thread(target=self._loop_starter)
self.loop_thread.start()

View File

@ -109,6 +109,7 @@ class Backend:
async def wait_until_client_exists(self, user_id: str = "") -> None:
loops = 0
while True:
if user_id and user_id in self.clients:
return
@ -116,7 +117,12 @@ class Backend:
if not user_id and self.clients:
return
if loops and loops % 100 == 0: # every 10s except first time
log.warning("Waiting for account %s to exist, %ds passed",
user_id, loops // 10)
await asyncio.sleep(0.1)
loops += 1
# General functions

View File

@ -1,183 +0,0 @@
import asyncio
import logging as log
import random
import re
from dataclasses import dataclass, field
from io import BytesIO
from pathlib import Path
from typing import Optional, Tuple
from urllib.parse import urlparse
import aiofiles
from PIL import Image as PILImage
import nio
import pyotherside
from nio.api import ResizingMethod
from . import utils
POSFormat = int
Size = Tuple[int, int]
ImageData = Tuple[bytearray, Size, int] # last int: pyotherside format enum
CONCURRENT_DOWNLOADS_LIMIT = asyncio.BoundedSemaphore(8)
with BytesIO() as img_out:
PILImage.new("RGBA", (1, 1), (0, 0, 0, 0)).save(img_out, "PNG")
TRANSPARENT_1X1_PNG = (img_out.getvalue(), pyotherside.format_data)
@dataclass
class Thumbnail:
provider: "ImageProvider" = field()
mxc: str = field()
width: int = field()
height: int = field()
def __post_init__(self) -> None:
self.mxc = re.sub(r"#auto$", "", self.mxc)
if not re.match(r"^mxc://.+/.+", self.mxc):
raise ValueError(f"Invalid mxc URI: {self.mxc}")
@property
def server_size(self) -> Tuple[int, int]:
# https://matrix.org/docs/spec/client_server/latest#thumbnails
if self.width > 640 or self.height > 480:
return (800, 600)
if self.width > 320 or self.height > 240:
return (640, 480)
if self.width > 96 or self.height > 96:
return (320, 240)
if self.width > 32 or self.height > 32:
return (96, 96)
return (32, 32)
@property
def resize_method(self) -> ResizingMethod:
return ResizingMethod.scale \
if self.width > 96 or self.height > 96 else ResizingMethod.crop
@property
def http(self) -> str:
return nio.Api.mxc_to_http(self.mxc)
@property
def local_path(self) -> Path:
parsed = urlparse(self.mxc)
name = "%s.%03d.%03d.%s" % (
parsed.path.lstrip("/"),
self.server_size[0],
self.server_size[1],
self.resize_method.value,
)
return self.provider.cache / parsed.netloc / name
async def read_data(self, data: bytes, mime: Optional[str],
) -> Tuple[bytes, POSFormat]:
if mime == "image/svg+xml":
return (data, pyotherside.format_svg_data)
if mime in ("image/jpeg", "image/png"):
return (data, pyotherside.format_data)
try:
with BytesIO(data) as img_in:
image = PILImage.open(img_in)
if image.mode == "RGB":
return (data, pyotherside.format_rgb888)
if image.mode == "RGBA":
return (data, pyotherside.format_argb32)
with BytesIO() as img_out:
image.save(img_out, "PNG")
return (img_out.getvalue(), pyotherside.format_data)
except OSError as err:
log.warning("Unable to process image: %s - %r", self.http, err)
return TRANSPARENT_1X1_PNG
async def download(self) -> Tuple[bytes, POSFormat]:
client = random.choice(
tuple(self.provider.app.backend.clients.values()),
)
parsed = urlparse(self.mxc)
async with CONCURRENT_DOWNLOADS_LIMIT:
resp = await client.thumbnail(
server_name = parsed.netloc,
media_id = parsed.path.lstrip("/"),
width = self.server_size[0],
height = self.server_size[1],
method = self.resize_method,
)
if isinstance(resp, nio.ThumbnailError):
log.warning("Downloading thumbnail failed - %s", resp)
return TRANSPARENT_1X1_PNG
body, pos_format = await self.read_data(resp.body, resp.content_type)
self.local_path.parent.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(self.local_path, "wb") as file:
# body might have been converted, always save the original image.
await file.write(resp.body)
return (body, pos_format)
async def local_read(self) -> Tuple[bytes, POSFormat]:
data = self.local_path.read_bytes()
with BytesIO(data) as data_io:
return await self.read_data(data, utils.guess_mime(data_io))
async def get_data(self) -> ImageData:
try:
data, pos_format = await self.local_read()
except (OSError, IOError, FileNotFoundError):
data, pos_format = await self.download()
with BytesIO(data) as img_in:
real_size = PILImage.open(img_in).size
return (bytearray(data), real_size, pos_format)
class ImageProvider:
def __init__(self, app) -> None:
self.app = app
self.cache = Path(self.app.appdirs.user_cache_dir) / "thumbnails"
self.cache.mkdir(parents=True, exist_ok=True)
def get(self, image_id: str, requested_size: Size) -> ImageData:
if requested_size[0] < 1 or requested_size[1] < 1:
raise ValueError(f"width or height < 1: {requested_size!r}")
try:
thumb = Thumbnail(self, image_id, *requested_size)
except ValueError as err:
log.warning(err)
data, pos_format = TRANSPARENT_1X1_PNG
return (bytearray(data), (1, 1), pos_format)
return asyncio.run_coroutine_threadsafe(
thumb.get_data(), self.app.loop,
).result()

View File

@ -93,6 +93,10 @@ class MatrixClient(nio.AsyncClient):
self.skipped_events: DefaultDict[str, int] = DefaultDict(lambda: 0)
from .media_cache import MediaCache
cache_dir = Path(self.backend.app.appdirs.user_cache_dir)
self.media_cache = MediaCache(self, cache_dir)
self.connect_callbacks()

182
src/python/media_cache.py Normal file
View File

@ -0,0 +1,182 @@
import asyncio
import io
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import DefaultDict, Optional, Tuple
from urllib.parse import urlparse
import aiofiles
import nio
from PIL import Image as PILImage
from .matrix_client import MatrixClient
Size = Tuple[int, int]
CONCURRENT_DOWNLOADS_LIMIT = asyncio.BoundedSemaphore(8)
ACCESS_LOCKS: DefaultDict[str, asyncio.Lock] = DefaultDict(asyncio.Lock)
@dataclass
class DownloadFailed(Exception):
message: str = field()
http_code: int = field()
@dataclass
class Media:
cache: "MediaCache" = field()
mxc: str = field()
def __post_init__(self) -> None:
self.mxc = re.sub(r"#auto$", "", self.mxc)
if not re.match(r"^mxc://.+/.+", self.mxc):
raise ValueError(f"Invalid mxc URI: {self.mxc}")
@property
def http(self) -> str:
return nio.Api.mxc_to_http(self.mxc)
@property
def local_path(self) -> Path:
parsed = urlparse(self.mxc)
name = parsed.path.lstrip("/")
return self.cache.downloads_dir / parsed.netloc / name
async def get(self) -> Path:
async with ACCESS_LOCKS[self.mxc]:
try:
return await self._get_local_existing_file()
except FileNotFoundError:
return await self._download()
async def _get_local_existing_file(self) -> Path:
if not self.local_path.exists():
raise FileNotFoundError()
return self.local_path
async def _download(self) -> Path:
async with CONCURRENT_DOWNLOADS_LIMIT:
body = await self._get_remote_data()
self.local_path.parent.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(self.local_path, "wb") as file:
await file.write(body)
return self.local_path
async def _get_remote_data(self) -> bytes:
raise NotImplementedError()
@dataclass
class Thumbnail(Media):
cache: "MediaCache" = field()
mxc: str = field()
wanted_size: Size = field()
server_size: Optional[Size] = field(init=False, repr=False, default=None)
@staticmethod
def normalize_size(size: Size) -> Size:
# https://matrix.org/docs/spec/client_server/latest#thumbnails
if size[0] > 640 or size[1] > 480:
return (800, 600)
if size[0] > 320 or size[1] > 240:
return (640, 480)
if size[0] > 96 or size[1] > 96:
return (320, 240)
if size[0] > 32 or size[1] > 32:
return (96, 96)
return (32, 32)
@property
def local_path(self) -> Path:
# example: thumbnails/matrix.org/32x32/<mxc id>
parsed = urlparse(self.mxc)
size = self.normalize_size(self.server_size or self.wanted_size)
name = "%dx%d/%s" % (size[0], size[1], parsed.path.lstrip("/"))
return self.cache.thumbs_dir / parsed.netloc / name
async def _get_local_existing_file(self) -> Path:
if self.local_path.exists():
return self.local_path
# If we have a bigger size thumbnail than the wanted_size for this pic,
# return it instead of asking the server for a smaller thumbnail.
try_sizes = ((32, 32), (96, 96), (320, 240), (640, 480), (800, 600))
parts = list(self.local_path.parts)
size = self.normalize_size(self.server_size or self.wanted_size)
for width, height in try_sizes:
if width < size[0] or height < size[1]:
continue
parts[-2] = f"{width}x{height}"
path = Path("/".join(parts))
if path.exists():
return path
raise FileNotFoundError()
async def _get_remote_data(self) -> bytes:
parsed = urlparse(self.mxc)
resp = await self.cache.client.thumbnail(
server_name = parsed.netloc,
media_id = parsed.path.lstrip("/"),
width = self.wanted_size[0],
height = self.wanted_size[1],
)
with io.BytesIO(resp.body) as img:
# The server may return a thumbnail bigger than what we asked for
self.server_size = PILImage.open(img).size
if isinstance(resp, nio.ErrorResponse):
raise DownloadFailed(resp.message, resp.status_code)
return resp.body
@dataclass
class MediaCache:
client: MatrixClient = field()
base_dir: Path = field()
def __post_init__(self) -> None:
self.thumbs_dir = self.base_dir / "thumbnails"
self.downloads_dir = self.base_dir / "downloads"
self.thumbs_dir.mkdir(parents=True, exist_ok=True)
self.downloads_dir.mkdir(parents=True, exist_ok=True)
async def thumbnail(self, mxc: str, width: int, height: int) -> str:
return str(await Thumbnail(self, mxc, (width, height)).get())

View File

@ -8,16 +8,6 @@ Rectangle {
implicitWidth: theme.controls.avatar.size
implicitHeight: theme.controls.avatar.size
property string name: ""
property var imageUrl: ""
property var toolTipImageUrl: imageUrl
property alias fillMode: avatarImage.fillMode
property alias animate: avatarImage.animate
readonly property alias hovered: hoverHandler.hovered
readonly property var params: Utils.thumbnailParametersFor(width, height)
color: avatarImage.visible ? "transparent" : Utils.hsluv(
name ? Utils.hueFrom(name) : 0,
name ? theme.controls.avatar.background.saturation : 0,
@ -25,6 +15,18 @@ Rectangle {
theme.controls.avatar.background.opacity
)
property string clientUserId
property string name
property alias mxc: avatarImage.mxc
property alias toolTipMxc: avatarToolTipImage.mxc
property alias sourceOverride: avatarImage.sourceOverride
property alias toolTipSourceOverride: avatarToolTipImage.sourceOverride
property alias fillMode: avatarImage.fillMode
property alias animate: avatarImage.animate
readonly property alias hovered: hoverHandler.hovered
HLabel {
z: 1
anchors.centerIn: parent
@ -41,23 +43,24 @@ Rectangle {
)
}
HImage {
HMxcImage {
id: avatarImage
anchors.fill: parent
visible: imageUrl
visible: Boolean(sourceOverride || mxc)
z: 2
sourceSize.width: params.width
sourceSize.height: params.height
sourceSize.width: parent.width
sourceSize.height: parent.height
fillMode: Image.PreserveAspectCrop
source: Qt.resolvedUrl(imageUrl)
animate: false
clientUserId: avatar.clientUserId
loadingLabel.font.pixelSize: theme.fontSize.small
HoverHandler { id: hoverHandler }
HToolTip {
id: avatarToolTip
visible: toolTipImageUrl && hoverHandler.hovered
visible: (toolTipSourceOverride || toolTipMxc) &&
hoverHandler.hovered
delay: 1000
backgroundColor: theme.controls.avatar.hoveredImage.background
@ -68,10 +71,11 @@ Rectangle {
background.border.width * 2,
)
contentItem: HImage {
contentItem: HMxcImage {
id: avatarToolTipImage
fillMode: Image.PreserveAspectCrop
source: Qt.resolvedUrl(toolTipImageUrl)
clientUserId: avatar.clientUserId
mxc: avatarImage.mxc
sourceSize.width: avatarToolTip.dimension
sourceSize.height: avatarToolTip.dimension

View File

@ -0,0 +1,40 @@
import QtQuick 2.12
import "../utils.js" as Utils
HImage {
id: image
source: sourceOverride || (show ? cachedPath : "")
onMxcChanged: Qt.callLater(update)
onWidthChanged: Qt.callLater(update)
onHeightChanged: Qt.callLater(update)
onVisibleChanged: Qt.callLater(update)
property string clientUserId
property string mxc
property string sourceOverride: ""
property bool show: false
property string cachedPath: ""
function update() {
let w = sourceSize.width || width
let h = sourceSize.height || height
if (! image.mxc || w < 1 || h < 1 ) {
show = false
return
}
let arg = [image.mxc, w, h]
if (! image) return // if it was destroyed
py.callClientCoro(clientUserId, "media_cache.thumbnail", arg, path => {
if (! image) return
image.cachedPath = path
show = image.visible
})
}
}

View File

@ -1,13 +1,10 @@
import QtQuick 2.12
HAvatar {
property string displayName: ""
property string avatarUrl: ""
name: displayName[0] == "#" && displayName.length > 1 ?
displayName.substring(1) :
displayName
imageUrl: avatarUrl ? ("image://python/" + avatarUrl) : null
toolTipImageUrl: avatarUrl ? ("image://python/" + avatarUrl) : null
property string displayName
}

View File

@ -1,17 +1,9 @@
import QtQuick 2.12
HAvatar {
property string userId: ""
property string displayName: ""
property string avatarUrl: ""
readonly property var defaultImageUrl:
avatarUrl ? ("image://python/" + avatarUrl) : null
readonly property var defaultToolTipImageUrl:
avatarUrl ? ("image://python/" + avatarUrl) : null
name: displayName || userId.substring(1) // no leading @
imageUrl: defaultImageUrl
toolTipImageUrl:defaultToolTipImageUrl
property string userId
property string displayName
}

View File

@ -36,6 +36,7 @@ Rectangle {
HUserAvatar {
id: bannerAvatar
clientUserId: chatPage.userId
anchors.centerIn: parent
}
}

View File

@ -11,7 +11,7 @@ Banner {
avatar.userId: inviterId
avatar.displayName: inviterName
avatar.avatarUrl: inviterAvatar
avatar.mxc: inviterAvatar
labelText: qsTr("%1 invited you to this room.").arg(
Utils.coloredNameHtml(inviterName, inviterId)

View File

@ -8,7 +8,7 @@ Banner {
// TODO: avatar func auto
avatar.userId: chatPage.userId
avatar.displayName: chatPage.userInfo.display_name
avatar.avatarUrl: chatPage.userInfo.avatar_url
avatar.mxc: chatPage.userInfo.avatar_url
labelText: qsTr("You are not part of this room anymore.")
buttonModel: [

View File

@ -58,9 +58,10 @@ Rectangle {
HUserAvatar {
id: avatar
clientUserId: chatPage.userId
userId: writingUserId
displayName: writingUserInfo.display_name
avatarUrl: writingUserInfo.avatar_url
mxc: writingUserInfo.avatar_url
}
HScrollableTextArea {

View File

@ -23,8 +23,9 @@ Rectangle {
HRoomAvatar {
id: avatar
clientUserId: chatPage.userId
displayName: chatPage.roomInfo.display_name
avatarUrl: chatPage.roomInfo.avatar_url
mxc: chatPage.roomInfo.avatar_url
Layout.alignment: Qt.AlignTop
}

View File

@ -7,9 +7,12 @@ HTileDelegate {
backgroundColor: theme.chat.roomSidePane.member.background
image: HUserAvatar {
clientUserId: chatPage.userId
userId: model.user_id
displayName: model.display_name
avatarUrl: model.avatar_url
mxc: model.avatar_url
width: height
height: memberDelegate.height
}
title.text: model.display_name || model.user_id

View File

@ -52,9 +52,10 @@ HRowLayout {
HUserAvatar {
id: avatar
clientUserId: chatPage.userId
userId: model.sender_id
displayName: model.sender_name
avatarUrl: model.sender_avatar
mxc: model.sender_avatar
width: parent.width
height: collapseAvatar ? 1 : 58
}

View File

@ -26,7 +26,9 @@ HGridLayout {
if (avatar.changed) {
saveButton.avatarChangeRunning = true
let path = Qt.resolvedUrl(avatar.imageUrl).replace(/^file:/, "")
let path =
Qt.resolvedUrl(avatar.sourceOverride).replace(/^file:/, "")
py.callClientCoro(userId, "set_avatar_from_file", [path], () => {
saveButton.avatarChangeRunning = false
@ -53,14 +55,15 @@ HGridLayout {
Component.onCompleted: nameField.field.forceActiveFocus()
HUserAvatar {
property bool changed: avatar.imageUrl != avatar.defaultImageUrl
property bool changed: Boolean(sourceOverride)
id: avatar
clientUserId: editAccount.userId
userId: editAccount.userId
displayName: nameField.field.text
avatarUrl: accountInfo.avatar_url
imageUrl: fileDialog.selectedFile || fileDialog.file || defaultImageUrl
toolTipImageUrl: ""
mxc: accountInfo.avatar_url
toolTipMxc: ""
sourceOverride: fileDialog.selectedFile || fileDialog.file
Layout.alignment: Qt.AlignHCenter
@ -71,11 +74,11 @@ HGridLayout {
z: 10
visible: opacity > 0
opacity: ! fileDialog.dialog.visible &&
(! avatar.imageUrl || avatar.hovered) ? 1 : 0
(! avatar.mxc || avatar.hovered) ? 1 : 0
anchors.fill: parent
color: Utils.hsluv(0, 0, 0,
(! avatar.imageUrl && overlayHover.hovered) ? 0.8 : 0.7
(! avatar.mxc && overlayHover.hovered) ? 0.8 : 0.7
)
Behavior on opacity { HNumberAnimation {} }
@ -90,7 +93,7 @@ HGridLayout {
HIcon {
svgName: "upload-avatar"
colorize: (! avatar.imageUrl && overlayHover.hovered) ?
colorize: (! avatar.mxc && overlayHover.hovered) ?
theme.colors.accentText : theme.icons.colorize
dimension: 64
@ -100,11 +103,11 @@ HGridLayout {
Item { Layout.preferredHeight: theme.spacing }
HLabel {
text: avatar.imageUrl ?
text: avatar.mxc ?
qsTr("Change profile picture") :
qsTr("Upload profile picture")
color: (! avatar.imageUrl && overlayHover.hovered) ?
color: (! avatar.mxc && overlayHover.hovered) ?
theme.colors.accentText : theme.colors.brightText
Behavior on color { HColorAnimation {} }

View File

@ -46,9 +46,10 @@ HTileDelegate {
image: HUserAvatar {
clientUserId: model.data.user_id
userId: model.data.user_id
displayName: model.data.display_name
avatarUrl: model.data.avatar_url
mxc: model.data.avatar_url
}
title.color: theme.sidePane.account.name

View File

@ -32,8 +32,9 @@ HTileDelegate {
image: HRoomAvatar {
clientUserId: model.user_id
displayName: model.data.display_name
avatarUrl: model.data.avatar_url
mxc: model.data.avatar_url
}
title.color: theme.sidePane.room.name

View File

@ -154,25 +154,6 @@ function filterModelSource(source, filter_text, property="filter_string") {
}
function thumbnailParametersFor(width, height) {
// https://matrix.org/docs/spec/client_server/latest#thumbnails
if (width > 640 || height > 480)
return {width: 800, height: 600, fillMode: Image.PreserveAspectFit}
if (width > 320 || height > 240)
return {width: 640, height: 480, fillMode: Image.PreserveAspectFit}
if (width > 96 || height > 96)
return {width: 320, height: 240, fillMode: Image.PreserveAspectFit}
if (width > 32 || height > 32)
return {width: 96, height: 96, fillMode: Image.PreserveAspectCrop}
return {width: 32, height: 32, fillMode: Image.PreserveAspectCrop}
}
function fitSize(width, height, max) {
if (width >= height) {
let new_width = Math.min(width, max)