Adjust flatpak packaging script according to the feedback

.gitignore

@@ -24,5 +24,6 @@ packaging/flatpak/flatpak-env
 packaging/flatpak/requirements.txt
 packaging/flatpak/flatpak-requirements.txt
 packaging/flatpak/flatpak-pip.json
+packaging/flatpak/flatpak-pip-generator
 .flatpak-builder/
 flatpak-build/

mirage.pro

@@ -44,7 +44,7 @@ dev {
     shortcuts.files = extra/linux/mirage.desktop
 
     icons256.path = $$PREFIX/share/icons/hicolor/256x256/apps
-    icons256.extra = mkdir -p $(INSTALL_ROOT)/$$PREFIX/share/icons/hicolor/256x256/apps && cp $$PWD/extra/linux/mirage.png $(INSTALL_ROOT)/$$PREFIX/share/icons/hicolor/256x256/apps/$${TARGET}.png
+    icons256.files = extra/linux/mirage.png
     INSTALLS += icons256
 
     INSTALLS += executables shortcuts icons256
@@ -107,3 +107,5 @@ for(file, $$list($$glob_filenames(*.py))) {
 QMAKE_CLEAN *= $$MOC_DIR $$OBJECTS_DIR $$RCC_DIR $$PYCACHE_DIRS $$QRC_FILE
 QMAKE_CLEAN *= $$BUILD_DIR $$TARGET Makefile mirage.pro.user .qmake.stash
 QMAKE_CLEAN *= $$glob_filenames(*.pyc, *.qmlc, *.jsc, *.egg-info)
+QMAKE_CLEAN *= packaging/flatpak/flatpak-env packaging/flatpak/requirements.txt packaging/flatpak/flatpak-requirements.txt
+QMAKE_CLEAN *= packaging/flatpak/flatpak-pip.json .flatpak-builder

packaging/flatpak/README.md

@@ -3,38 +3,48 @@
 ## Manifest
 
 Flatpak packaging manifest is generated by running
-
+[generate-flatpak-script.sh](generate-flatpak-script.sh). This script
-```
+requires `libolm` to be installed on the development PC as it will
-packaging/flatpak/generate-flatpak-script.sh
+create Python virtual environment and install all the requirements in
-```
+it.
-
-from the root of the project. This script requires `libolm` to be
-installed on the development PC as it will create Python virtual
-environment and install all the requirements in it.
 
 Note that the requirements are taken from
-`packaging/flatpak/requirements.flatpak.txt` and the ones specified in
+[requirements.flatpak.txt](requirements.flatpak.txt) and the ones
-the project' root `requirements.txt`. At the moment of writing,
+specified in the project' root
+[requirements.txt](../../requirements.txt). At the moment of writing,
 included packages are `multidict` with the version that installs in
 Flatpak (newer versions seem to have some issues), `uvloop`, and few
 packages required for building other packages.
 
 In addition, the list of ignored packages is in
-`packaging/flatpak/generate-flatpak-script.sh`.
+[generate-flatpak-script.sh](generate-flatpak-script.sh).
 
 Flatpak manifest is created automatically by
-`generate-flatpak-script.sh` using `mirage.flatpak.base.yaml` and
+[generate-flatpak-script.sh](generate-flatpak-script.sh) using
-replacing the marked placeholder with Python module dependencies.
+[mirage.flatpak.base.yaml](mirage.flatpak.base.yaml) and replacing the
+marked placeholder with Python module dependencies.
 
 
 ## Building Flatpak
 
-To build flatpak package, you will need flatpak, runtime and SDK (KDE
+To build flatpak package, you will need flatpak, flatpak-builder,
-5.12), and flatpak-builder.
+runtime and SDK (KDE 5.12), and flatpak-builder. flatpak-builder is
+usually available from the same repository as flatpak. See
+https://flatpak.org/setup/ for setting it up. To install runtimes
+(adjust as needed if you prefer system-wide installation):
 
-To build, run
+```
+flatpak install --user flathub org.kde.Platform//5.12 org.kde.Sdk//5.12
+```
+
+To build, run from the root of the project:
 
 ```
 flatpak-builder --repo=../flatpak-repo --force-clean flatpak-build packaging/flatpak/mirage.flatpak.yaml
 ```
 
+To create bundle, run
+
+```
+flatpak build-bundle ../flatpak-repo ../mirage.flatpak io.github.mirukana.mirage
+```

packaging/flatpak/collector.py

@@ -1,33 +1,32 @@
 import json
 import yaml
 
-with open('mirage.flatpak.base.yaml') as f:
+with open("mirage.flatpak.base.yaml") as f:
     base = yaml.load(f, Loader=yaml.FullLoader)
 
-with open('flatpak-pip.json') as f:
+with open("flatpak-pip.json") as f:
-    modules = json.load(f)['modules']
+    modules = json.load(f)["modules"]
 
 # set some modules in front as dependencies and dropping matrix-nio
 # which is declared separately
 front = []
 back = []
 for m in modules:
-    n = m['name']
+    n = m["name"]
-    if n.startswith('python3-') and \
+    if n.startswith("python3-") and \
-       n[len('python3-'):] in ['multidict', 'cffi', 'pytest-runner', 'setuptools-scm']:
+       n[len("python3-"):] in ["cffi", "importlib-metadata", "multidict", "pytest-runner", "setuptools-scm"]:
         front.append(m)
     else:
         back.append(m)
 
 # replace placeholder with modules
 phold = None
-for i in range(len(base['modules'])):
+for i in range(len(base["modules"])):
-    if base['modules'][i]['name'] == 'PLACEHOLDER PYTHON DEPENDENCIES':
+    if base["modules"][i]["name"] == "PLACEHOLDER PYTHON DEPENDENCIES":
         phold = i
         break
 
-base['modules'] = base['modules'][:i] + front + back + base['modules'][i+1:]
+base["modules"] = base["modules"][:i] + front + back + base["modules"][i+1:]
 
-with open('mirage.flatpak.yaml', 'w') as f:
+with open("mirage.flatpak.yaml", "w") as f:
     f.write(yaml.dump(base, sort_keys=False, indent=2))
-    #json.dump(base, f, indent=4)

packaging/flatpak/flatpak-pip-generator

@@ -1,175 +0,0 @@
-#!/usr/bin/env python3
-
-# From https://github.com/flatpak/flatpak-builder-tools/tree/master/pip
-
-__license__ = 'MIT'
-
-import argparse
-import json
-import hashlib
-import os
-import subprocess
-import tempfile
-import urllib.request
-import shlex
-from collections import OrderedDict
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument('packages', nargs='*')
-parser.add_argument('--python2', action='store_true',
-                    help='Look for a Python 2 package')
-parser.add_argument('--cleanup', choices=['scripts', 'all'],
-                    help='Select what to clean up after build')
-parser.add_argument('--requirements-file',
-                    help='Specify requirements.txt file')
-parser.add_argument('--build-only', action='store_const',
-                    dest='cleanup', const='all',
-                    help='Clean up all files after build')
-parser.add_argument('--no-build-isolation',
-                    help='https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support')
-parser.add_argument('--output',
-                    help='Specify output file name')
-opts = parser.parse_args()
-
-
-def get_pypi_url(name: str, filename: str) -> str:
-    url = 'https://pypi.org/pypi/{}/json'.format(name)
-    print('Extracting download url for', name)
-    with urllib.request.urlopen(url) as response:
-        body = json.loads(response.read().decode('utf-8'))
-        for release in body['releases'].values():
-            for source in release:
-                if source['filename'] == filename:
-                    return source['url']
-        else:
-            raise Exception('Failed to extract url from {}'.format(url))
-
-
-def get_package_name(filename: str) -> str:
-    if filename.endswith(('bz2', 'gz', 'xz', 'zip')):
-        segments = filename.split("-")
-        if len(segments) == 2:
-            return segments[0]
-        return "-".join(segments[:len(segments) - 1])
-    elif filename.endswith('whl'):
-        segments = filename.split("-")
-        if len(segments) == 5:
-            return segments[0]
-        return "-".join(segments[:len(segments) - 4])
-    else:
-        raise Exception(
-            'Downloaded filename: {} does not end with bz2, gz, xz, zip, or whl'.format(filename)
-        )
-
-def get_file_hash(filename: str) -> str:
-    sha = hashlib.sha256()
-    print('Generating hash for', filename)
-    with open(filename, 'rb') as f:
-        while True:
-            data = f.read(1024 * 1024 * 32)
-            if not data:
-                break
-            sha.update(data)
-        return sha.hexdigest()
-
-
-if not opts.packages and not opts.requirements_file:
-    exit("Please specifiy either packages or requirements file argument")
-
-packages = []
-if opts.requirements_file and os.path.exists(opts.requirements_file):
-    with open(opts.requirements_file, 'r') as req_file:
-        packages = [package.strip() for package in req_file.readlines()]
-else:
-    packages = opts.packages
-
-
-if opts.python2:
-    pip_executable = 'pip2'
-else:
-    pip_executable = 'pip3'
-
-modules = []
-
-for package in packages:
-    package_name = 'python{}-{}'.format('2' if opts.python2 else '3',
-                                        package.split("=")[0])
-    tempdir_prefix = 'pip-generator-{}-'.format(package_name)
-    with tempfile.TemporaryDirectory(prefix=tempdir_prefix) as tempdir:
-
-        pip_download = [
-            pip_executable,
-            'download',
-            '--dest',
-            tempdir
-        ]
-        pip_command = [
-            pip_executable,
-            'install',
-            '--no-index',
-            '--find-links="file://${PWD}"',
-            '--prefix=${FLATPAK_DEST}',
-            shlex.quote(package)
-        ]
-        if opts.no_build_isolation:
-            pip_command.append('--no-build-isolation')
-        module = OrderedDict([
-            ('name', package_name),
-            ('buildsystem', 'simple'),
-            ('build-commands', [' '.join(pip_command)]),
-            ('sources', []),
-        ])
-
-        if opts.cleanup == 'all':
-            module['cleanup'] = ['*']
-        elif opts.cleanup == 'scripts':
-            module['cleanup'] = ['/bin', '/share/man/man1']
-
-        try:
-            # May download the package twice, the first time it allows pip to
-            # select the preferred package, if the package downloaded is not
-            # platform generic, then it forces pip to download the sdist (using
-            # the --no-binary option).
-            subprocess.run(pip_download + [package], check=True)
-            for filename in os.listdir(tempdir):
-                if not filename.endswith(('gz', 'any.whl')):
-                    os.remove(os.path.join(tempdir, filename))
-                    subprocess.run(pip_download + [
-                        '--no-binary', ':all:', package
-                    ], check=True)
-            for filename in os.listdir(tempdir):
-                name = get_package_name(filename)
-                if name == 'setuptools':  # Already installed
-                    continue
-                sha256 = get_file_hash(os.path.join(tempdir, filename))
-                url = get_pypi_url(name, filename)
-                source = OrderedDict([
-                    ('type', 'file'),
-                    ('url', url),
-                    ('sha256', sha256),
-                ])
-                module['sources'].append(source)
-        except subprocess.CalledProcessError:
-            print("Failed to download {}".format(package))
-            print("Please fix the module manually in the generated file")
-        modules.append(module)
-
-if opts.requirements_file:
-    output_package = opts.output or 'pypi-dependencies'
-else:
-    output_package = opts.output or package_name
-output_filename = output_package + '.json'
-
-if len(modules) == 1:
-    pypi_module = modules[0]
-else:
-    pypi_module = {
-        'name': output_package,
-        'buildsystem': 'simple',
-        'build-commands': [],
-        'modules': modules,
-    }
-
-with open(output_filename, 'w') as output:
-    output.write(json.dumps(pypi_module, indent=4))

packaging/flatpak/generate-flatpak-script.sh

@@ -1,13 +1,17 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
-DIR=`dirname "$(readlink -f "$0")"`
+DIR="$(dirname "$(readlink -f "$0")")"
 
-cd $DIR
+cd "$DIR"
 
 python3 -m venv flatpak-env
-export PATH=$DIR/flatpak-env/bin:$PATH
+export PATH="$DIR/flatpak-env/bin:$PATH"
+
+if [ ! -f flatpak-pip-generator ]; then
+    wget https://raw.githubusercontent.com/flatpak/flatpak-builder-tools/master/pip/flatpak-pip-generator
+fi
 
 cat requirements.flatpak.txt ../../requirements.txt > requirements.txt
 
@@ -17,7 +21,8 @@ flatpak-env/bin/pip install -Ur requirements.txt
 flatpak-env/bin/pip freeze | grep -v PyYAML | grep -v six= | grep -v matrix-nio > flatpak-requirements.txt
 
 # generate flatpak requirements
-flatpak-env/bin/python flatpak-pip-generator --output flatpak-pip --requirements-file=flatpak-requirements.txt
+flatpak-env/bin/python flatpak-pip-generator --output flatpak-pip \
+                       --requirements-file=flatpak-requirements.txt
 
 flatpak-env/bin/pip install PyYAML
 python collector.py