Zergling_man/moment
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add visible indicator when downloading files

Browse Source 
Downloading file messages will now show a transfer control above the
composer, similar to uploads. Measuring the progress or pausing the
operation is not possible yet.
This commit is contained in:
miruka
2021-01-20 15:50:04 -04:00
parent 86f0a8a6a0
commit 7af1456c1d
16 changed files with 167 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/backend/backend.py
View File

@@ -68,8 +68,8 @@ class Backend:
- `("<user_id>", "rooms")`: rooms our account `user_id` is part of;
- `("<user_id>", "uploads")`: ongoing or failed file uploads for
our account `user_id`;
- `("<user_id>", "transfers")`: ongoing or failed file
uploads/downloads for our account `user_id`;
- `("<user_id>", "<room_id>", "members")`: members in the room
`room_id` that our account `user_id` is part of;
@@ -322,7 +322,7 @@ class Backend:
self.models["accounts"].pop(user_id, None)
self.models["matching_accounts"].pop(user_id, None)
self.models[user_id, "uploads"].clear()
self.models[user_id, "transfers"].clear()
for room_id in self.models[user_id, "rooms"]:
self.models["all_rooms"].pop(room_id, None)

5
src/backend/errors.py
View File

@@ -90,6 +90,11 @@ class MatrixTooLarge(MatrixError):
m_code: str = "M_TOO_LARGE"
@dataclass
class MatrixBadGateway(MatrixError):
http_code: int = 502
m_code: Optional[str] = None
@dataclass
class MatrixBadGateway(MatrixError):
http_code: int = 502

84
src/backend/matrix_client.py
View File

@@ -9,7 +9,6 @@ import io
import logging as log
import platform
import re
import sys
import textwrap
import traceback
from contextlib import suppress
@@ -42,8 +41,8 @@ from .errors import (
from .html_markdown import HTML_PROCESSOR as HTML
from .media_cache import Media, Thumbnail
from .models.items import (
ZERO_DATE, Account, Event, Member, Room, TypeSpecifier, Upload,
UploadStatus,
ZERO_DATE, Account, Event, Member, Room, Transfer, TransferStatus,
TypeSpecifier,
)
from .models.model_store import ModelStore
from .nio_callbacks import NioCallbacks
@@ -55,11 +54,6 @@ from .pyotherside_events import (
if TYPE_CHECKING:
from .backend import Backend
if sys.version_info >= (3, 7):
current_task = asyncio.current_task
else:
current_task = asyncio.Task.current_task
CryptDict = Dict[str, Any]
PathCallable = Union[
str, Path, Callable[[], Coroutine[None, None, Union[str, Path]]],
@@ -190,8 +184,8 @@ class MatrixClient(nio.AsyncClient):
self.sync_task: Optional[asyncio.Future] = None
self.start_task: Optional[asyncio.Future] = None
self.upload_monitors: Dict[UUID, nio.TransferMonitor] = {}
self.upload_tasks: Dict[UUID, asyncio.Task] = {}
self.transfer_monitors: Dict[UUID, nio.TransferMonitor] = {}
self.transfer_tasks: Dict[UUID, asyncio.Task] = {}
self.send_message_tasks: Dict[UUID, asyncio.Task] = {}
self._presence: str = ""
@@ -628,23 +622,23 @@ class MatrixClient(nio.AsyncClient):
await self._send_message(room_id, content, tx_id)
async def toggle_pause_upload(
async def toggle_pause_transfer(
self, room_id: str, uuid: Union[str, UUID],
) -> None:
if isinstance(uuid, str):
uuid = UUID(uuid)
pause = not self.upload_monitors[uuid].pause
pause = not self.transfer_monitors[uuid].pause
self.upload_monitors[uuid].pause = pause
self.models[room_id, "uploads"][str(uuid)].paused = pause
self.transfer_monitors[uuid].pause = pause
self.models[room_id, "transfers"][str(uuid)].paused = pause
async def cancel_upload(self, uuid: Union[str, UUID]) -> None:
async def cancel_transfer(self, uuid: Union[str, UUID]) -> None:
if isinstance(uuid, str):
uuid = UUID(uuid)
self.upload_tasks[uuid].cancel()
self.transfer_tasks[uuid].cancel()
async def send_clipboard_image(
@@ -691,9 +685,9 @@ class MatrixClient(nio.AsyncClient):
try:
await self._send_file(item_uuid, room_id, path, reply_to_event_id)
except (nio.TransferCancelledError, asyncio.CancelledError):
self.upload_monitors.pop(item_uuid, None)
self.upload_tasks.pop(item_uuid, None)
self.models[room_id, "uploads"].pop(str(item_uuid), None)
self.transfer_monitors.pop(item_uuid, None)
self.transfer_tasks.pop(item_uuid, None)
self.models[room_id, "transfers"].pop(str(item_uuid), None)
async def _send_file(
@@ -708,10 +702,10 @@ class MatrixClient(nio.AsyncClient):
# TODO: this function is way too complex, and most of it should be
# refactored into nio.
self.upload_tasks[item_uuid] = current_task() # type: ignore
self.transfer_tasks[item_uuid] = utils.current_task() # type: ignore
upload_item = Upload(item_uuid)
self.models[room_id, "uploads"][str(item_uuid)] = upload_item
transfer = Transfer(item_uuid, is_upload=True)
self.models[room_id, "transfers"][str(item_uuid)] = transfer
transaction_id = uuid4()
path = Path(await path() if callable(path) else path)
@@ -726,18 +720,18 @@ class MatrixClient(nio.AsyncClient):
# This error will be caught again by the try block later below
size = 0
upload_item.set_fields(
status=UploadStatus.Uploading, filepath=path, total_size=size,
transfer.set_fields(
status=TransferStatus.Transfering, filepath=path, total_size=size,
)
monitor = nio.TransferMonitor(size)
self.upload_monitors[item_uuid] = monitor
self.transfer_monitors[item_uuid] = monitor
def on_transferred(transferred: int) -> None:
upload_item.uploaded = transferred
transfer.transferred = transferred
def on_speed_changed(speed: float) -> None:
upload_item.set_fields(
transfer.set_fields(
speed = speed,
time_left = monitor.remaining_time or timedelta(0),
)
@@ -761,8 +755,8 @@ class MatrixClient(nio.AsyncClient):
raise nio.TransferCancelledError()
except (MatrixError, OSError) as err:
upload_item.set_fields(
status = UploadStatus.Error,
transfer.set_fields(
status = TransferStatus.Error,
error = type(err),
error_args = err.args,
)
@@ -771,8 +765,8 @@ class MatrixClient(nio.AsyncClient):
while True:
await asyncio.sleep(0.1)
upload_item.status = UploadStatus.Caching
local_media = await Media.from_existing_file(
transfer.status = TransferStatus.Caching
local_media = await Media.from_existing_file(
self.backend.media_cache, self.user_id, url, path,
)
@@ -787,7 +781,7 @@ class MatrixClient(nio.AsyncClient):
"body": path.name,
"info": {
"mimetype": mime,
"size": upload_item.total_size,
"size": transfer.total_size,
},
}
@@ -822,20 +816,20 @@ class MatrixClient(nio.AsyncClient):
thumb_ext = "png" if thumb_info.mime == "image/png" else "jpg"
thumb_name = f"{path.stem}_thumbnail.{thumb_ext}"
upload_item.set_fields(
status = UploadStatus.Uploading,
transfer.set_fields(
status = TransferStatus.Transfering,
filepath = Path(thumb_name),
total_size = len(thumb_data),
)
try:
upload_item.total_size = thumb_info.size
transfer.total_size = thumb_info.size
monitor = nio.TransferMonitor(thumb_info.size)
monitor.on_transferred = on_transferred
monitor.on_speed_changed = on_speed_changed
self.upload_monitors[item_uuid] = monitor
self.transfer_monitors[item_uuid] = monitor
thumb_url, _, thumb_crypt_dict = await self.upload(
lambda *_: thumb_data,
@@ -851,7 +845,7 @@ class MatrixClient(nio.AsyncClient):
except MatrixError as err:
log.warning(f"Failed uploading thumbnail {path}: {err}")
else:
upload_item.status = UploadStatus.Caching
transfer.status = TransferStatus.Caching
await Thumbnail.from_bytes(
self.backend.media_cache,
@@ -907,9 +901,9 @@ class MatrixClient(nio.AsyncClient):
content["msgtype"] = "m.file"
content["filename"] = path.name
del self.upload_monitors[item_uuid]
del self.upload_tasks[item_uuid]
del self.models[room_id, "uploads"][str(upload_item.id)]
del self.transfer_monitors[item_uuid]
del self.transfer_tasks[item_uuid]
del self.models[room_id, "transfers"][str(transfer.id)]
if reply_to_event_id:
await self.send_text(
@@ -1004,7 +998,7 @@ class MatrixClient(nio.AsyncClient):
"""Send a message event with `content` dict to a room."""
self.send_message_tasks[transaction_id] = \
current_task() # type: ignore
utils.current_task() # type: ignore
async with self.backend.send_locks[room_id]:
await self.room_send(
@@ -2068,13 +2062,13 @@ class MatrixClient(nio.AsyncClient):
avatar_size = (48, 48)
avatar_path = await self.backend.media_cache.get_thumbnail(
avatar_path = await Thumbnail(
cache = self.backend.media_cache,
client_user_id = self.user_id,
mxc = mxc,
title = f"user_{user_id}.notification",
width = avatar_size[0],
height = avatar_size[1],
)
wanted_size = avatar_size,
).get()
image_data = None
create = False

116
src/backend/media_cache.py
View File

@@ -13,11 +13,14 @@ from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Optional
from urllib.parse import urlparse
from uuid import uuid4
import nio
from PIL import Image as PILImage
from .utils import Size, atomic_write
from .models.items import Transfer, TransferStatus
from .models.model import Model
from .utils import Size, atomic_write, current_task
if TYPE_CHECKING:
from .backend import Backend
@@ -25,8 +28,6 @@ if TYPE_CHECKING:
if sys.version_info < (3, 8):
import pyfastcopy # noqa
CryptDict = Optional[Dict[str, Any]]
CONCURRENT_DOWNLOADS_LIMIT = asyncio.BoundedSemaphore(8)
ACCESS_LOCKS: DefaultDict[str, asyncio.Lock] = DefaultDict(asyncio.Lock)
@@ -47,45 +48,35 @@ class MediaCache:
self.downloads_dir.mkdir(parents=True, exist_ok=True)
async def get_media(
self,
client_user_id: str,
mxc: str,
title: str,
crypt_dict: CryptDict = None,
) -> Path:
"""Return `Media.get()`'s result. Intended for QML."""
return await Media(self, client_user_id, mxc, title, crypt_dict).get()
async def get_media(self, *args) -> Path:
"""Return `Media(self, ...).get()`'s result. Intended for QML."""
return await Media(self, *args).get()
async def get_thumbnail(
self,
client_user_id: str,
mxc: str,
title: str,
width: int,
height: int,
crypt_dict: CryptDict = None,
) -> Path:
"""Return `Thumbnail.get()`'s result. Intended for QML."""
async def get_thumbnail(self, width: float, height: float, *args) -> Path:
"""Return `Thumbnail(self, ...).get()`'s result. Intended for QML."""
# QML sometimes pass float sizes, which matrix API doesn't like.
size = (round(width), round(height))
thumb = Thumbnail(self, client_user_id, mxc, title, crypt_dict, size)
return await thumb.get()
return await Thumbnail(
self, *args, wanted_size=size, # type: ignore
).get()
@dataclass
class Media:
"""A matrix media file."""
"""A matrix media file that is downloaded or has yet to be.
cache: "MediaCache" = field()
client_user_id: str = field()
mxc: str = field()
title: str = field()
crypt_dict: CryptDict = field(repr=False)
If the `room_id` is not set, no `Transfer` model item will be registered
while this media is being downloaded.
"""
cache: "MediaCache" = field()
client_user_id: str = field()
mxc: str = field()
title: str = field()
room_id: Optional[str] = None
filesize: Optional[int] = None
crypt_dict: Optional[Dict[str, Any]] = field(default=None, repr=False)
def __post_init__(self) -> None:
@@ -155,12 +146,39 @@ class Media:
async def _get_remote_data(self) -> bytes:
"""Return the file's data from the matrix server, decrypt if needed."""
parsed = urlparse(self.mxc)
client = self.cache.backend.clients[self.client_user_id]
resp = await self.cache.backend.clients[self.client_user_id].download(
server_name = parsed.netloc,
media_id = parsed.path.lstrip("/"),
)
transfer: Optional[Transfer] = None
model: Optional[Model] = None
if self.room_id:
model = self.cache.backend.models[self.room_id, "transfers"]
transfer = Transfer(
id = uuid4(),
is_upload = False,
filepath = self.local_path,
total_size = self.filesize or 0,
status = TransferStatus.Transfering,
)
assert model is not None
client.transfer_tasks[transfer.id] = current_task() # type: ignore
model[str(transfer.id)] = transfer
try:
parsed = urlparse(self.mxc)
resp = await client.download(
server_name = parsed.netloc,
media_id = parsed.path.lstrip("/"),
)
except (nio.TransferCancelledError, asyncio.CancelledError):
if transfer and model:
del model[str(transfer.id)]
del client.transfer_tasks[transfer.id]
raise
if transfer and model:
del model[str(transfer.id)]
del client.transfer_tasks[transfer.id]
return await self._decrypt(resp.body)
@@ -196,8 +214,13 @@ class Media:
"""Copy an existing file to cache and return a `Media` for it."""
media = cls(
cache, client_user_id, mxc, existing.name, {}, **kwargs,
) # type: ignore
cache = cache,
client_user_id = client_user_id,
mxc = mxc,
title = existing.name,
filesize = existing.stat().st_size,
**kwargs,
)
media.local_path.parent.mkdir(parents=True, exist_ok=True)
if not media.local_path.exists() or overwrite:
@@ -221,8 +244,8 @@ class Media:
"""Create a cached file from bytes data and return a `Media` for it."""
media = cls(
cache, client_user_id, mxc, filename, {}, **kwargs,
) # type: ignore
cache, client_user_id, mxc, filename, filesize=len(data), **kwargs,
)
media.local_path.parent.mkdir(parents=True, exist_ok=True)
if not media.local_path.exists() or overwrite:
@@ -237,14 +260,9 @@ class Media:
@dataclass
class Thumbnail(Media):
"""The thumbnail of a matrix media, which is a media itself."""
"""A matrix media's thumbnail, which is downloaded or has yet to be."""
cache: "MediaCache" = field()
client_user_id: str = field()
mxc: str = field()
title: str = field()
crypt_dict: CryptDict = field(repr=False)
wanted_size: Size = field()
wanted_size: Size = (800, 600)
server_size: Optional[Size] = field(init=False, repr=False, default=None)

35
src/backend/models/items.py
View File

@@ -286,37 +286,38 @@ class Member(ModelItem):
)
class UploadStatus(AutoStrEnum):
class TransferStatus(AutoStrEnum):
"""Enum describing the status of an upload operation."""
Preparing = auto()
Uploading = auto()
Caching = auto()
Error = auto()
Preparing = auto()
Transfering = auto()
Caching = auto()
Error = auto()
@dataclass(eq=False)
class Upload(ModelItem):
"""Represent a running or failed file upload operation."""
class Transfer(ModelItem):
"""Represent a running or failed file upload/download operation."""
id: UUID = field()
filepath: Path = Path("-")
id: UUID = field()
is_upload: bool = field()
filepath: Path = Path("-")
total_size: int = 0
uploaded: int = 0
speed: float = 0
time_left: timedelta = timedelta(0)
paused: bool = False
total_size: int = 0
transferred: int = 0
speed: float = 0
time_left: timedelta = timedelta(0)
paused: bool = False
status: UploadStatus = UploadStatus.Preparing
status: TransferStatus = TransferStatus.Preparing
error: OptionalExceptionType = type(None)
error_args: Tuple[Any, ...] = ()
start_date: datetime = field(init=False, default_factory=datetime.now)
def __lt__(self, other: "Upload") -> bool:
"""Sort by the start date, from newest upload to oldest."""
def __lt__(self, other: "Transfer") -> bool:
"""Sort by the start date, from newest transfer to oldest."""
return (self.start_date, self.id) > (other.start_date, other.id)

2
src/backend/nio_callbacks.py
View File

@@ -197,6 +197,8 @@ class NioCallbacks:
client_user_id = self.user_id,
mxc = ev.url,
title = ev.body,
room_id = room.room_id,
filesize = info.get("size") or 0,
crypt_dict = media_crypt_dict,
).get_local()
except FileNotFoundError:

2
src/backend/utils.py
View File

@@ -38,8 +38,10 @@ from .pcn.section import Section
if sys.version_info >= (3, 7):
from contextlib import asynccontextmanager
current_task = asyncio.current_task
else:
from async_generator import asynccontextmanager
current_task = asyncio.Task.current_task
Size = Tuple[int, int]
BytesOrPIL = Union[bytes, PILImage.Image]