From 3ea97105e49cd4521f03669dc0b3ffc8e1941358 Mon Sep 17 00:00:00 2001 From: Rinigus Date: Wed, 25 Mar 2020 22:27:40 +0200 Subject: [PATCH] Adjust flatpak packaging script according to the feedback --- .gitignore | 1 + mirage.pro | 4 +- packaging/flatpak/README.md | 42 +++-- packaging/flatpak/collector.py | 21 ++- packaging/flatpak/flatpak-pip-generator | 175 ------------------- packaging/flatpak/generate-flatpak-script.sh | 15 +- 6 files changed, 50 insertions(+), 208 deletions(-) delete mode 100755 packaging/flatpak/flatpak-pip-generator diff --git a/.gitignore b/.gitignore index d17a1343..3a1fbba6 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,6 @@ packaging/flatpak/flatpak-env packaging/flatpak/requirements.txt packaging/flatpak/flatpak-requirements.txt packaging/flatpak/flatpak-pip.json +packaging/flatpak/flatpak-pip-generator .flatpak-builder/ flatpak-build/ diff --git a/mirage.pro b/mirage.pro index aeadd466..5b35e717 100644 --- a/mirage.pro +++ b/mirage.pro @@ -44,7 +44,7 @@ dev { shortcuts.files = extra/linux/mirage.desktop icons256.path = $$PREFIX/share/icons/hicolor/256x256/apps - icons256.extra = mkdir -p $(INSTALL_ROOT)/$$PREFIX/share/icons/hicolor/256x256/apps && cp $$PWD/extra/linux/mirage.png $(INSTALL_ROOT)/$$PREFIX/share/icons/hicolor/256x256/apps/$${TARGET}.png + icons256.files = extra/linux/mirage.png INSTALLS += icons256 INSTALLS += executables shortcuts icons256 @@ -107,3 +107,5 @@ for(file, $$list($$glob_filenames(*.py))) { QMAKE_CLEAN *= $$MOC_DIR $$OBJECTS_DIR $$RCC_DIR $$PYCACHE_DIRS $$QRC_FILE QMAKE_CLEAN *= $$BUILD_DIR $$TARGET Makefile mirage.pro.user .qmake.stash QMAKE_CLEAN *= $$glob_filenames(*.pyc, *.qmlc, *.jsc, *.egg-info) +QMAKE_CLEAN *= packaging/flatpak/flatpak-env packaging/flatpak/requirements.txt packaging/flatpak/flatpak-requirements.txt +QMAKE_CLEAN *= packaging/flatpak/flatpak-pip.json .flatpak-builder diff --git a/packaging/flatpak/README.md b/packaging/flatpak/README.md index f30d0b17..28ecf580 100644 --- a/packaging/flatpak/README.md +++ b/packaging/flatpak/README.md @@ -3,38 +3,48 @@ ## Manifest Flatpak packaging manifest is generated by running - -``` -packaging/flatpak/generate-flatpak-script.sh -``` - -from the root of the project. This script requires `libolm` to be -installed on the development PC as it will create Python virtual -environment and install all the requirements in it. +[generate-flatpak-script.sh](generate-flatpak-script.sh). This script +requires `libolm` to be installed on the development PC as it will +create Python virtual environment and install all the requirements in +it. Note that the requirements are taken from -`packaging/flatpak/requirements.flatpak.txt` and the ones specified in -the project' root `requirements.txt`. At the moment of writing, +[requirements.flatpak.txt](requirements.flatpak.txt) and the ones +specified in the project' root +[requirements.txt](../../requirements.txt). At the moment of writing, included packages are `multidict` with the version that installs in Flatpak (newer versions seem to have some issues), `uvloop`, and few packages required for building other packages. In addition, the list of ignored packages is in -`packaging/flatpak/generate-flatpak-script.sh`. +[generate-flatpak-script.sh](generate-flatpak-script.sh). Flatpak manifest is created automatically by -`generate-flatpak-script.sh` using `mirage.flatpak.base.yaml` and -replacing the marked placeholder with Python module dependencies. +[generate-flatpak-script.sh](generate-flatpak-script.sh) using +[mirage.flatpak.base.yaml](mirage.flatpak.base.yaml) and replacing the +marked placeholder with Python module dependencies. ## Building Flatpak -To build flatpak package, you will need flatpak, runtime and SDK (KDE -5.12), and flatpak-builder. +To build flatpak package, you will need flatpak, flatpak-builder, +runtime and SDK (KDE 5.12), and flatpak-builder. flatpak-builder is +usually available from the same repository as flatpak. See +https://flatpak.org/setup/ for setting it up. To install runtimes +(adjust as needed if you prefer system-wide installation): -To build, run +``` +flatpak install --user flathub org.kde.Platform//5.12 org.kde.Sdk//5.12 +``` + +To build, run from the root of the project: ``` flatpak-builder --repo=../flatpak-repo --force-clean flatpak-build packaging/flatpak/mirage.flatpak.yaml ``` +To create bundle, run + +``` +flatpak build-bundle ../flatpak-repo ../mirage.flatpak io.github.mirukana.mirage +``` \ No newline at end of file diff --git a/packaging/flatpak/collector.py b/packaging/flatpak/collector.py index 351100e3..41d7aa94 100644 --- a/packaging/flatpak/collector.py +++ b/packaging/flatpak/collector.py @@ -1,33 +1,32 @@ import json import yaml -with open('mirage.flatpak.base.yaml') as f: +with open("mirage.flatpak.base.yaml") as f: base = yaml.load(f, Loader=yaml.FullLoader) -with open('flatpak-pip.json') as f: - modules = json.load(f)['modules'] +with open("flatpak-pip.json") as f: + modules = json.load(f)["modules"] # set some modules in front as dependencies and dropping matrix-nio # which is declared separately front = [] back = [] for m in modules: - n = m['name'] - if n.startswith('python3-') and \ - n[len('python3-'):] in ['multidict', 'cffi', 'pytest-runner', 'setuptools-scm']: + n = m["name"] + if n.startswith("python3-") and \ + n[len("python3-"):] in ["cffi", "importlib-metadata", "multidict", "pytest-runner", "setuptools-scm"]: front.append(m) else: back.append(m) # replace placeholder with modules phold = None -for i in range(len(base['modules'])): - if base['modules'][i]['name'] == 'PLACEHOLDER PYTHON DEPENDENCIES': +for i in range(len(base["modules"])): + if base["modules"][i]["name"] == "PLACEHOLDER PYTHON DEPENDENCIES": phold = i break -base['modules'] = base['modules'][:i] + front + back + base['modules'][i+1:] +base["modules"] = base["modules"][:i] + front + back + base["modules"][i+1:] -with open('mirage.flatpak.yaml', 'w') as f: +with open("mirage.flatpak.yaml", "w") as f: f.write(yaml.dump(base, sort_keys=False, indent=2)) - #json.dump(base, f, indent=4) diff --git a/packaging/flatpak/flatpak-pip-generator b/packaging/flatpak/flatpak-pip-generator deleted file mode 100755 index fe690d43..00000000 --- a/packaging/flatpak/flatpak-pip-generator +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 - -# From https://github.com/flatpak/flatpak-builder-tools/tree/master/pip - -__license__ = 'MIT' - -import argparse -import json -import hashlib -import os -import subprocess -import tempfile -import urllib.request -import shlex -from collections import OrderedDict - - -parser = argparse.ArgumentParser() -parser.add_argument('packages', nargs='*') -parser.add_argument('--python2', action='store_true', - help='Look for a Python 2 package') -parser.add_argument('--cleanup', choices=['scripts', 'all'], - help='Select what to clean up after build') -parser.add_argument('--requirements-file', - help='Specify requirements.txt file') -parser.add_argument('--build-only', action='store_const', - dest='cleanup', const='all', - help='Clean up all files after build') -parser.add_argument('--no-build-isolation', - help='https://pip.pypa.io/en/stable/reference/pip/#pep-517-and-518-support') -parser.add_argument('--output', - help='Specify output file name') -opts = parser.parse_args() - - -def get_pypi_url(name: str, filename: str) -> str: - url = 'https://pypi.org/pypi/{}/json'.format(name) - print('Extracting download url for', name) - with urllib.request.urlopen(url) as response: - body = json.loads(response.read().decode('utf-8')) - for release in body['releases'].values(): - for source in release: - if source['filename'] == filename: - return source['url'] - else: - raise Exception('Failed to extract url from {}'.format(url)) - - -def get_package_name(filename: str) -> str: - if filename.endswith(('bz2', 'gz', 'xz', 'zip')): - segments = filename.split("-") - if len(segments) == 2: - return segments[0] - return "-".join(segments[:len(segments) - 1]) - elif filename.endswith('whl'): - segments = filename.split("-") - if len(segments) == 5: - return segments[0] - return "-".join(segments[:len(segments) - 4]) - else: - raise Exception( - 'Downloaded filename: {} does not end with bz2, gz, xz, zip, or whl'.format(filename) - ) - -def get_file_hash(filename: str) -> str: - sha = hashlib.sha256() - print('Generating hash for', filename) - with open(filename, 'rb') as f: - while True: - data = f.read(1024 * 1024 * 32) - if not data: - break - sha.update(data) - return sha.hexdigest() - - -if not opts.packages and not opts.requirements_file: - exit("Please specifiy either packages or requirements file argument") - -packages = [] -if opts.requirements_file and os.path.exists(opts.requirements_file): - with open(opts.requirements_file, 'r') as req_file: - packages = [package.strip() for package in req_file.readlines()] -else: - packages = opts.packages - - -if opts.python2: - pip_executable = 'pip2' -else: - pip_executable = 'pip3' - -modules = [] - -for package in packages: - package_name = 'python{}-{}'.format('2' if opts.python2 else '3', - package.split("=")[0]) - tempdir_prefix = 'pip-generator-{}-'.format(package_name) - with tempfile.TemporaryDirectory(prefix=tempdir_prefix) as tempdir: - - pip_download = [ - pip_executable, - 'download', - '--dest', - tempdir - ] - pip_command = [ - pip_executable, - 'install', - '--no-index', - '--find-links="file://${PWD}"', - '--prefix=${FLATPAK_DEST}', - shlex.quote(package) - ] - if opts.no_build_isolation: - pip_command.append('--no-build-isolation') - module = OrderedDict([ - ('name', package_name), - ('buildsystem', 'simple'), - ('build-commands', [' '.join(pip_command)]), - ('sources', []), - ]) - - if opts.cleanup == 'all': - module['cleanup'] = ['*'] - elif opts.cleanup == 'scripts': - module['cleanup'] = ['/bin', '/share/man/man1'] - - try: - # May download the package twice, the first time it allows pip to - # select the preferred package, if the package downloaded is not - # platform generic, then it forces pip to download the sdist (using - # the --no-binary option). - subprocess.run(pip_download + [package], check=True) - for filename in os.listdir(tempdir): - if not filename.endswith(('gz', 'any.whl')): - os.remove(os.path.join(tempdir, filename)) - subprocess.run(pip_download + [ - '--no-binary', ':all:', package - ], check=True) - for filename in os.listdir(tempdir): - name = get_package_name(filename) - if name == 'setuptools': # Already installed - continue - sha256 = get_file_hash(os.path.join(tempdir, filename)) - url = get_pypi_url(name, filename) - source = OrderedDict([ - ('type', 'file'), - ('url', url), - ('sha256', sha256), - ]) - module['sources'].append(source) - except subprocess.CalledProcessError: - print("Failed to download {}".format(package)) - print("Please fix the module manually in the generated file") - modules.append(module) - -if opts.requirements_file: - output_package = opts.output or 'pypi-dependencies' -else: - output_package = opts.output or package_name -output_filename = output_package + '.json' - -if len(modules) == 1: - pypi_module = modules[0] -else: - pypi_module = { - 'name': output_package, - 'buildsystem': 'simple', - 'build-commands': [], - 'modules': modules, - } - -with open(output_filename, 'w') as output: - output.write(json.dumps(pypi_module, indent=4)) diff --git a/packaging/flatpak/generate-flatpak-script.sh b/packaging/flatpak/generate-flatpak-script.sh index 1dafe5ef..6e52c59e 100755 --- a/packaging/flatpak/generate-flatpak-script.sh +++ b/packaging/flatpak/generate-flatpak-script.sh @@ -1,13 +1,17 @@ -#!/bin/bash +#!/usr/bin/env bash set -e -DIR=`dirname "$(readlink -f "$0")"` +DIR="$(dirname "$(readlink -f "$0")")" -cd $DIR +cd "$DIR" python3 -m venv flatpak-env -export PATH=$DIR/flatpak-env/bin:$PATH +export PATH="$DIR/flatpak-env/bin:$PATH" + +if [ ! -f flatpak-pip-generator ]; then + wget https://raw.githubusercontent.com/flatpak/flatpak-builder-tools/master/pip/flatpak-pip-generator +fi cat requirements.flatpak.txt ../../requirements.txt > requirements.txt @@ -17,7 +21,8 @@ flatpak-env/bin/pip install -Ur requirements.txt flatpak-env/bin/pip freeze | grep -v PyYAML | grep -v six= | grep -v matrix-nio > flatpak-requirements.txt # generate flatpak requirements -flatpak-env/bin/python flatpak-pip-generator --output flatpak-pip --requirements-file=flatpak-requirements.txt +flatpak-env/bin/python flatpak-pip-generator --output flatpak-pip \ + --requirements-file=flatpak-requirements.txt flatpak-env/bin/pip install PyYAML python collector.py