#!/usr/bin/python3 -I
"""walnut-data list | import [-p PKG]... [--no-download] GAME [SOURCE...] | remove GAME | install-repo GAME
(walnut-games package, /usr/bin): the player's own game data made into packages and installed, from a
terminal or the Walnut Games app. Runs as the player; only walnut-games-helper (through sudo -n) is root.

Design
- GAME is a catalog id (/usr/share/walnut-games/catalog.json) whose entry has a data_setup (GameDB
  build.toml [data_setup]): the game-data-packager game, the packages, which may download.
- list: every catalog game with a data_setup, its data installed or missing (any one of present_any;
  optional for a game that runs without), whether the configured apt sources have one of its data
  packages (the licensed suite: `walnut-data install-repo GAME`), its title.
- import: each SOURCE as the player has it. SOURCE may be left out when every -p package is in the game's
  `fetch` list (the free shareware or demo gdp downloads entirely: `import -p quake2-demo-data yquake2`);
  gdp then runs with --download and no source (still --no-search and -p); otherwise it is refused. An .iso or .img file is opened read-only with
  `udisksctl loop-setup -r -f` and mounted with `udisksctl mount -b` (the player's own polkit rights; no
  copy), a block device (the disc drive, /dev/sr0) is used at its mount point or mounted the same way,
  and whatever this program mounted or set up is released afterwards. ps3-desktop's udev rule
  (61-walnut-loop.rules, UDISKS_AUTO=0) keeps the desktop from mounting loop devices by itself, so
  there is no race and no pcmanfm error dialog; on a system without it the desktop (gvfs) mounts a new
  loop device within seconds: when udisksctl mount says AlreadyMounted, that mount is used
  (waited for, AUTOMOUNT_WAIT) and unmounted afterwards, which frees the loop device (loop-delete only
  for one still attached). loop-setup needs the desktop session's polkit rights: over ssh it is refused
  (NotAuthorizedCanObtain), and the message says how to run it in the desktop's slice. A directory, an installer or
  archive file, or a mounted disc's path is given to game-data-packager as it is.
  BIN/CUE (a .cue, or a .bin with its .cue next to it; gdp reads neither): the cue sheet is parsed here
  (FILE BINARY|MOTOROLA, one or several files; TRACK MODE1/2352, MODE2/2352, MODE1/2048, MODE2/2336,
  AUDIO; INDEX 01 is where a track starts, the next track's INDEX 00 or 01 where it ends; WAVE files
  and other modes are refused). The first data track's 2048-byte user data (MODE1 after the 16-byte
  sync and header, MODE2 form 1 after the 8-byte subheader too) goes to ~/.cache/walnut-data/disc/
  NAME.iso, cut at the ISO 9660 volume's size, which is then opened like any .iso and removed after.
  When a music package is asked for (the manifest's tracks entry: quake-music, quake2-music), the
  audio tracks it names (first..last) are streamed as WAV (RIFF header, CD-DA's 16-bit little-endian
  stereo 44.1 kHz; MOTOROLA swapped) into oggenc at its default quality, two at a time (no WAV on
  disk), into disc/music/<the manifest's source layout> (id1/music/track02.ogg), the folder gdp is
  given: gdp takes already ripped tracks from there, so it needs no patch for this. Only music packages
  asked for: no ISO is made. A cue sheet is read before anything is written, so a broken one refuses
  at once; the space check counts the ISO and the Ogg files (OGG_ENC_BYTES per second) in the cache.
  MDF/MDS (Alcohol 120%: an .mds, or the .mdf with its .mds next to it): the descriptor is read here,
  version 1.x only (signature "MEDIA DESCRIPTOR", byte 16 = 1): every session's track blocks (points
  1-99; the lead-in entries 0xA0-0xA2 skipped), each track's mode (low four bits: 9 audio, A mode 1, B
  and C mode 2 form 1), sector size (2048, 2336, 2352, or 2448 = 2352 + 96 bytes of interleaved
  subchannel), start offset in the .mdf and length (the extra block's, a CD; a DVD's runs to the file's
  end), the image file from the footer ("*.mdf": the .mds's name). Refused: version 2, a split image
  (number_of_files > 1), mode 2 form 2 or an unknown mode, other sector sizes. The tracks then go the
  BIN/CUE way (ISO extracted, music encoded, the subchannel bytes dropped); a data track of 2048-byte
  sectors at offset 0 is loop-mounted as it is, no copy. An .mdf without its .mds is sniffed: an ISO
  9660 volume at sector 16 is a plain image (loop-mounted), else raw sectors (the sync pattern at the
  start of sectors 0 and 1 says 2352 or 2448; the header's mode byte 1 or 2) are one data track to the
  file's end, cut at the volume's size; anything else is refused (give the .mds).
  The disc in the drive with a music package asked for: its table of contents (CDROMREADTOC* ioctls,
  the player's ACL on /dev/sr0) says whether it has audio tracks; then the device itself is given to gdp
  too, which rips the music with cdparanoia and oggenc (an audio CD that can't be mounted is fine
  then); without audio tracks the music is reported missing, never a rip that would fail gdp's whole
  run.
  game-data-packager (gdp) runs as the player in a transient user unit: `systemd-run --user --pipe
  --wait` with MemoryMax, RuntimeMaxSec and NoNewPrivileges, and PrivateNetwork when it must not
  download and the user manager can make one (checked by a probe: only "lo" in its /proc/net/dev);
  without a user manager it runs directly. Its command: `game-data-packager --no-search --no-compress
  --no-launchers --progress-fd 3 -n -d ~/.cache/walnut-data/out --download|--no-download -p PKG...
  GDPGAME SOURCE...` with TMPDIR=~/.cache/walnut-data/tmp (the PS3's /tmp is a small tmpfs). The
  packages: -p's, else the game's base, its required ones (all of them: gdp makes those the source has;
  a game with only optional packages gets those), each always passed as -p (without -p gdp builds
  whatever else it can, downloads included); the optional ones (expansions, music) are add-ons, imported
  on request with -p from the same source. A deb gdp made that was not asked for fails the import,
  nothing imported. --download only when every one of
  them is among the game's downloads (the official patch files gdp knows), else --no-download.
  A gdp without --progress-fd or --no-launchers (Debian's) runs without them: no progress lines, and
  the helper drops the Doom family's menu files anyway. In the unit, descriptor 3 is a FIFO in the
  cache directory (systemd-run --pipe passes only 0-2).
  Each package gdp made is then `dpkg-deb --fsys-tarfile X.deb | sudo -n walnut-games-helper
  import-data GAME PKG` (the deb unlinked as soon as dpkg-deb has it open, so its space comes back
  when the stream ends): the helper checks every file against its manifest and builds and installs
  the package itself (required packages first, in the catalog's order); a made package that another made
  or installed one replaces (the manifest's replaces: quake-registered and quake-shareware) is left out
  with a log line, so an import never trades a full game for its shareware. out/ and tmp/ are emptied
  before and removed after.
  Free space is checked before gdp runs, from the data manifest's sizes of the packages (T): the most
  each file system holds at one time over the phases (gdp: 3T in the cache, its copy of the files,
  dpkg-deb's temporary data member and the deb (a Quake II import ran out of space at 2T), for an
  installer or archive 3T plus its unpacked files, counted as the packages' data or the installer's
  or archive's own size, the larger (gdp unpacks what the installer holds for any package of the game:
  GOG's Quake III Gold gave baseq3/pak0.pk3 to a Team Arena import, which ran out of space at 4T; the
  unpacked files are what the installer's listing adds up to, whichever packages were asked for, at
  least its own size: GOG's Quake Enhanced, 607 MB, unpacks to 1137 MB), plus
  what the sources put there (a BIN/CUE's ISO and music) and DOWNLOAD_BYTES
  when it may download; streaming: T cache + T staging; the helper's deb: 3T staging, its files, dpkg-deb's
  temporary data member and the deb; installing: T staging +
  T /usr), summed where they share a disk; refused with what is needed and what is free.
  One import at a time per player: import holds an flock on ~/.cache/walnut-data/import.lock for its
  whole run, the file saying "PID GAME"; a second import is refused (busy, exit 2). The Walnut Games
  app reads that lock to show "importing" when it is opened again during an import (import_state).
- Add-ons. After an import, "addon available PKG" says an optional package of the game (not made now, not
  installed) can be made from the same sources: its required files are among theirs, found without
  unpacking by a case-insensitive base name and size against the data manifest (a folder or mount is
  walked, a zip read, an installer listed with `innoextract --list`), and a music package when a source
  has audio tracks (a BIN/CUE or MDS). `import -p PKG GAME SOURCE` makes it. Only a hint: failures to
  look are ignored.
- remove: `walnut-games-helper remove-data GAME` (the game's data packages only). install-repo:
  `walnut-games-helper install-repo-data GAME` (its data from the configured apt sources).
- Output, one item per line, flushed: "gdp step|file|generated|error ..." (gdp's progress lines),
  "log text" (gdp's own output and notes), the helper's lines as they come ("step STEP text", "progress
  N text", "count DONE TOTAL", "log text", "error text"; its "plan" is left out), "error text" for this
  program's own failures. For Walnut Games' bar (walnut_games.Progress) an import also says "plan
  read:B build:B check:B deb:B apt:B ..." (plan_line: the steps with their bytes, check/deb/apt once per
  package imported; first as expected, again with the packages gdp made), "step read ..." when it
  starts on the sources, "count DONE TOTAL" as gdp's file lines reach the packages' files (ReadCount:
  by name and size against the data manifest) and "step build ..." when gdp builds a package.
- Exit codes: 0 done; 2 refused, nothing changed (bad arguments, an unknown game, a missing source,
  the helper refused); 1 failed (gdp or the helper failed, or gdp made no package).
- Partial results. After gdp, the packages asked for are compared with the debs it made. Required means
  the game's required packages, or every -p package when -p was given. For each package that is missing:
    "gdp error needs-download PKG: TEXT"  PKG is one of the game's downloads and downloads were off
        (import with -p PKG alone, or without --no-download, lets gdp fetch its patch files); same shape as
        gdp's own error lines, so the app's "Try again with download" logic reads it
    "warning missing PKG: TEXT"           any other missing package (TEXT is gdp's last error line when it
        gave one, else "game-data-packager did not make it"); a missing optional package is only this
  A plain import (no -p) is playable with any one required package: once one was made (or one is already
  installed) the other required packages are not reported; with -p every package asked for is, and when
  no required package was made they all are (needs-download stays, so the app can offer the download).
  A package in a data_setup `alternatives` group (any one member satisfies it) is not reported when
  another member of its group was made.
  Exit 0 when at least one required package was made (or, for a game with no required package, any
  package) and installed, even with such lines; exit 1 ("error game-data-packager made no package...")
  when none was. The lines come before the helper's output.
- After import, remove and install-repo succeed, `/usr/lib/ps3/desktop/ps3-game-menu --reload
  $XDG_RUNTIME_DIR/ps3-game-menu` runs when it exists and $DISPLAY and $XDG_RUNTIME_DIR are set (failures
  ignored), so the session's menu shows the new launchers like after the app's own installs.
- stdlib only, no Tk. The commands are module constants so that tests/test_walnut_games.py runs it
  with a stubbed gdp, dpkg-deb and helper.
"""
import argparse
import fcntl
import json
import os
import re
import shutil
import stat
import struct
import subprocess
import sys
import threading
import time
import zipfile

CATALOG = "/usr/share/walnut-games/catalog.json"
HELPER = "/usr/lib/ps3/sbin/walnut-games-helper"
SUDO = ["/usr/bin/sudo", "-n"]
GDP = "/usr/games/game-data-packager"
DPKG_DEB, DPKG_QUERY, APT_CACHE = "/usr/bin/dpkg-deb", "/usr/bin/dpkg-query", "/usr/bin/apt-cache"
UDISKSCTL = "/usr/bin/udisksctl"
MENU_TOOL = "/usr/lib/ps3/desktop/ps3-game-menu"
SYSTEMD_RUN = "/usr/bin/systemd-run"
MEMORY_MAX = "192M"       # gdp ~40 MB with one game loaded, plus an unpacker (innoextract, unshield)
RUNTIME_MAX = "3h"        # a CD reads at about 2.3 MB/s
UNIT_PATH = "/usr/local/bin:/usr/bin:/bin:/usr/games"
ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,63}\Z")
PKG_RE = re.compile(r"[a-z0-9][a-z0-9+.-]*[a-z0-9]\Z")
IMAGE_SUFFIXES = (".iso", ".img")
AUTOMOUNT_WAIT = 10       # seconds to wait for the desktop's mount of a loop device it got to first
MANIFEST = "/usr/share/walnut-games/data-manifest.json"
IMPORT_DIR = "/var/lib/walnut-games/import"    # the helper's staging (root's)
USR = "/usr"
SPACE_MARGIN = 64 << 20
SYS_BLOCK = "/sys/block"
INNOEXTRACT = "/usr/bin/innoextract"
LIST_TIMEOUT = 120        # seconds for an installer's file list
LIST_MAX = 300000         # files of a folder or archive looked at for add-ons


class Refused(Exception):
    """exit code 2: nothing was changed"""


class Failed(Exception):
    """exit code 1"""


def say(kind, text=""):
    try:
        sys.stdout.write((kind + " " + text).rstrip() + "\n")
        sys.stdout.flush()
    except (OSError, ValueError):
        pass


def run(cmd, **kw):
    """(returncode, stdout and stderr) of cmd; 127 when it is not there"""
    try:
        p = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
                           errors="replace", **kw)
    except OSError as e:
        return 127, str(e)
    return p.returncode, p.stdout


# ---- the catalog ------------------------------------------------------------------------------

def load_games(path=None):
    with open(path or CATALOG, encoding="utf-8") as f:
        return {g["id"]: g for g in json.load(f)["games"] if isinstance(g.get("id"), str)}


def data_setup(game_id, games):
    if not ID_RE.match(game_id or ""):
        raise Refused("not a game id: %r" % game_id)
    if game_id not in games:
        raise Refused("unknown game: %s (walnut-data list shows the games)" % game_id)
    ds = games[game_id].get("data_setup")
    if not isinstance(ds, dict) or not isinstance(ds.get("gdp"), str):
        raise Refused("%s has no data to set up" % game_id)
    return ds


def names(ds, key):
    return [p for p in ds.get(key) or [] if isinstance(p, str) and PKG_RE.match(p)]


def covered(ds, pkg, made):
    """pkg is in an alternatives group (data_setup: one of its members is enough, e.g. the General MIDI
    or the FM music of Transport Tycoon Deluxe) of which another member is in made: not reported missing"""
    for grp in ds.get("alternatives") or []:
        if isinstance(grp, list) and pkg in grp and any(m in made for m in grp if m != pkg):
            return True
    return False


def wanted_packages(ds, asked):
    """the packages to make: -p's (each one of the game's), else the base: the required packages (a game
    that has only optional ones, like OpenTTD, gets those); the optional ones are add-ons, asked for with -p"""
    known = names(ds, "required") + names(ds, "optional")
    if asked:
        bad = [p for p in asked if p not in known]
        if bad:
            raise Refused("%s: not a data package of this game (%s)" % (", ".join(bad), ", ".join(known)))
        return list(dict.fromkeys(asked))
    return names(ds, "required") or known


def download_choice(ds, pkgs, no_download=False, sourceless=False):
    """--download only when every package is one the game may download for, else --no-download; with no
    SOURCE the packages must be ones gdp fetches entirely (the game's fetch list)"""
    dl = set(names(ds, "fetch" if sourceless else "downloads"))
    return "--download" if pkgs and not no_download and all(p in dl for p in pkgs) else "--no-download"


def installed(pkgs):
    if not pkgs:
        return set()
    rc, out = run([DPKG_QUERY, "-W", "-f", "${Package}\t${db:Status-Abbrev}\n"] + list(pkgs))
    return {n for line in out.splitlines() for n, _, st in [line.partition("\t")] if n in pkgs and st[1:2] == "i"}


def candidates(pkgs):
    if not pkgs:
        return set()
    rc, out = run([APT_CACHE, "policy"] + list(pkgs))
    have, cur = set(), None
    for line in out.splitlines():
        m = re.match(r"(\S+):\s*$", line)
        if m:
            cur = m.group(1)
        elif cur and (m := re.match(r"\s+Candidate:\s*(\S+)", line)) and m.group(1) != "(none)":
            have.add(cur)
    return have


def cmd_list(games):
    rows = []
    all_names = sorted({p for g in games.values() if isinstance(g.get("data_setup"), dict)
                        for k in ("present_any", "required", "optional") for p in names(g["data_setup"], k)})
    inst, repo = installed(all_names), candidates(all_names)
    for gid in sorted(games):
        ds = games[gid].get("data_setup")
        if not isinstance(ds, dict):
            continue
        any_of = names(ds, "present_any") or names(ds, "required")
        opt = names(ds, "optional")
        if names(ds, "required"):
            state = "present" if inst & set(any_of) else "missing"
        else:
            state = "optional-present" if inst & set(opt) else "optional"
        avail = "repo" if repo & set(any_of) and state == "missing" else "-"
        rows.append((gid, state, avail, games[gid].get("title") or gid))
    w = max([len(r[0]) for r in rows] + [4])
    for r in rows:
        print("%-*s  %-16s  %-4s  %s" % (w, r[0], r[1], r[2], r[3]))
    return 0


# ---- sources ----------------------------------------------------------------------------------

def source_kind(path):
    """'image' (an .iso/.img file: loop-mounted), 'device' (a block device: its mount point), 'path'
    (a directory or any other file, as it is); Refused when it is not there"""
    try:
        st = os.stat(path)
    except OSError:
        raise Refused("no such file or folder: %s" % path)
    if stat.S_ISBLK(st.st_mode):
        return "device"
    if stat.S_ISREG(st.st_mode) and path.lower().endswith(IMAGE_SUFFIXES):
        return "image"
    return "path"


def unescape_mount(s):
    return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), s)


def mount_point(dev, mounts="/proc/self/mounts"):
    """where block device dev is mounted, or None"""
    try:
        real = os.path.realpath(dev)
        with open(mounts) as f:
            for line in f:
                parts = line.split()
                if len(parts) >= 2 and os.path.realpath(unescape_mount(parts[0])) == real:
                    return unescape_mount(parts[1])
    except OSError:
        pass
    return None


def already_mounted_at(out):
    """the mount point in udisksctl's AlreadyMounted error ("... is already mounted at `/run/media/u/X'.")"""
    m = re.search(r"already mounted at [`'\"]?(/.+?)['\"]?\.?\s*$", out.strip(), re.M)
    return m.group(1) if m else None


def loop_attached(dev, sys_block=None):
    """whether loop device dev still has a backing file"""
    return os.path.exists(os.path.join(sys_block or SYS_BLOCK, os.path.basename(dev), "loop", "backing_file"))


class Sources:
    """the SOURCE arguments as paths gdp reads; loop devices and mounts made here released by close(), the
    files made from a BIN/CUE (the data track's ISO, the music's Ogg files) removed by it. music: the
    manifest's tracks entries of the music packages asked for (music_entries)."""

    def __init__(self, base=None, music=None):
        self.mounted, self.loops, self.made = [], [], []
        self.base, self.music = base, list(music or [])
        self.only_music = False

    def plan(self, path):
        """(kind, path, extra): what open() will do with a SOURCE and the bytes it adds to the cache while
        gdp runs (the ISO and Ogg files of a BIN/CUE, gdp's WAV of a CD rip); Refused when it is not there.
        A cue sheet is read here, so a broken one refuses before anything is written."""
        path = os.path.abspath(path)
        isfile = os.path.isfile(path)
        cue = cue_for(path) if isfile else None
        if cue:
            tracks = parse_cue(cue)
            return ("disc", (cue, tracks), self.disc_extra(tracks))
        mds = mds_for(path) if isfile else None
        if mds is not None:
            tracks = parse_mds(mds) if mds else sniff_mdf(path)
            if tracks == "iso":
                return ("image", path, 0)          # 2048-byte sectors: loop-mounted as it is
            data = next((t for t in tracks if not t.audio), None)
            if data and data.layout == (2048, 0) and data.start == 0 and not self.music:
                return ("image", data.path, 0)
            return ("disc", (mds or path, tracks), self.disc_extra(tracks))
        kind = source_kind(path)
        return (kind, path, CD_RIP_WAV_BYTES + MUSIC_TRACK_BYTES if kind == "device" and self.music else 0)

    def disc_extra(self, tracks):
        """the bytes a BIN/CUE's or MDF's tracks add to the cache: the data track's ISO and the music"""
        data = next((t for t in tracks if not t.audio), None)
        extra = data.sectors * 2048 if data else 0
        for m in self.music:
            extra += sum(int(track_seconds(t) * OGG_ENC_BYTES) for t in tracks
                         if t.audio and m["first"] <= t.number <= m["last"])
        return extra

    def open(self, plan):
        """[paths] for gdp from a plan"""
        kind, what, _ = plan
        if kind == "disc":
            return self.add_disc(*what)
        if kind == "device":
            return self.add_device(what)
        if kind == "image":
            return [self.open_image(what)]
        return [self.add(what)]

    def add_disc(self, cue, tracks):
        """a BIN/CUE or MDF/MDS (cue: the sheet or descriptor): its first data track as an ISO in the cache,
        opened like any image; the music packages' audio tracks encoded to Ogg Vorbis in a folder laid out
        as gdp looks for ripped tracks (id1/music/track02.ogg), which gdp is given too"""
        name = os.path.splitext(os.path.basename(cue))[0]
        work = os.path.join(self.base or cache_dir(), "disc")
        shutil.rmtree(work, ignore_errors=True)
        os.makedirs(work, 0o700)
        self.made.append(work)
        paths = []
        data = next((t for t in tracks if not t.audio), None)
        if data and not self.only_music:
            iso = os.path.join(work, re.sub(r"[^A-Za-z0-9._-]", "_", name) + ".iso")
            say("gdp", "step extracting the data track of %s" % os.path.basename(cue))
            n = write_iso(data, iso)
            say("log", "track %d of %s (%s) written as %s, %s" % (data.number, os.path.basename(cue), data.mode, iso,
                                                                  human(n)))
            paths.append(self.add(iso))
        audio = [t for t in tracks if t.audio]
        for m in self.music:
            have = [t for t in audio if m["first"] <= t.number <= m["last"]]
            if not have:
                say("log", "%s has no audio tracks %d-%d for the music" % (os.path.basename(cue), m["first"], m["last"]))
                continue
            music = os.path.join(work, "music")
            encode_tracks(have, m["source"], music, m["first"], m["last"])
            if music not in paths:
                paths.append(music)
        if not paths:
            raise Refused("%s has nothing to import: no data track%s" % (cue, " and no music tracks" if self.music else ""))
        return paths

    def add_device(self, dev):
        """the disc in drive dev: its mount point (for the data), and the device itself for gdp to rip the
        music from when a music package is asked for and the disc has audio tracks"""
        paths, why = [], None
        if not self.only_music:
            try:
                paths.append(self.mount(dev))
            except Refused as e:
                if not self.music:
                    raise
                why = e
        if self.music:
            audio = disc_audio_tracks(dev)
            if audio:
                say("log", "the disc in %s has audio tracks %s: game-data-packager rips the music (cdparanoia)"
                    % (dev, ", ".join(map(str, audio))))
                paths.append(dev)
            else:
                say("log", "the disc in %s has no audio tracks for the music" % dev)
        if not paths:
            raise why or Refused("nothing to read on the disc in %s" % dev)
        return paths

    def mount(self, dev, ours=False):
        """dev's mount point, mounted with udisksctl when it is not; ours: dev is a loop device set up
        here, so whoever mounted it (the desktop's automounter does, within seconds) it is unmounted
        by close(), which frees the loop device too"""
        mp = mount_point(dev)
        if mp:
            if ours:
                self.mounted.append(dev)
            return mp
        rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" not in out and os.path.exists(dev + "p1"):     # an image with a partition table
            dev = dev + "p1"
            mp = mount_point(dev)
            if mp:
                if ours:
                    self.mounted.append(dev)
                return mp
            rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" in out:
            # mounted meanwhile by the desktop (gvfs mounts a new loop device by itself): use that mount
            mp = self.wait_mount(dev) or already_mounted_at(out)
            if mp and os.path.isdir(mp):
                if ours:
                    self.mounted.append(dev)
                say("log", "%s is mounted at %s" % (dev, mp))
                return mp
        if rc != 0:
            raise Refused("cannot mount %s: %s" % (dev, (out.strip().splitlines() or ["?"])[-1]))
        self.mounted.append(dev)
        mp = mount_point(dev)
        if not mp:
            m = re.search(r" at (.+?)\.?\s*$", out.strip())
            mp = m.group(1) if m else None
        if not mp or not os.path.isdir(mp):
            raise Refused("%s was mounted but its folder is not known: %s" % (dev, out.strip()))
        say("log", "mounted %s at %s" % (dev, mp))
        return mp

    @staticmethod
    def wait_mount(dev):
        end = time.monotonic() + AUTOMOUNT_WAIT
        while True:
            mp = mount_point(dev)
            if mp or time.monotonic() >= end:
                return mp
            time.sleep(0.2)

    def add(self, path):
        path = os.path.abspath(path)
        kind = source_kind(path)
        if kind == "path":
            return path
        if kind == "device":
            return self.mount(path)
        return self.open_image(path)

    def open_image(self, path):
        """an image of 2048-byte sectors (.iso, .img, a plain .mdf) set up read-only as a loop device and
        mounted; its mount point"""
        rc, out = run([UDISKSCTL, "loop-setup", "-r", "-f", path])
        m = re.search(r" as (/dev/\S+?)\.?\s*$", out.strip())
        if rc != 0 and "NotAuthorized" in out:
            raise Refused("opening the disc image %s needs the desktop session's rights (polkit): run walnut-data "
                          "in a terminal on the Walnut desktop or from Walnut Games; over ssh: systemd-run --user "
                          "--scope --slice=ps3-desktop.slice walnut-data import ...; or open the image in the file "
                          "manager and give its folder" % path)
        if rc != 0 or not m:
            raise Refused("cannot open %s: %s" % (path, (out.strip().splitlines() or ["?"])[-1]))
        self.loops.append(m.group(1))
        say("log", "opened %s as %s" % (path, m.group(1)))
        return self.mount(m.group(1), ours=True)

    def close(self):
        for dev in reversed(self.mounted):
            run([UDISKSCTL, "unmount", "-b", dev])
        for dev in reversed(self.loops):
            # unmounting frees a loop device udisks set up (autoclear); loop-delete only for one still there
            if loop_attached(dev):
                run([UDISKSCTL, "loop-delete", "-b", dev])
        for d in self.made:
            shutil.rmtree(d, ignore_errors=True)
        self.mounted, self.loops, self.made = [], [], []


# ---- BIN/CUE images ---------------------------------------------------------------------------

# a track's sector: (bytes per sector in the .bin, offset of the 2048 bytes of user data or None for audio)
TRACK_MODES = {"MODE1/2352": (2352, 16), "MODE2/2352": (2352, 24), "MODE1/2048": (2048, 0),
               "MODE2/2336": (2336, 8), "AUDIO": (2352, None)}
CUE_FILE_TYPES = ("BINARY", "MOTOROLA")       # MOTOROLA: big-endian audio samples
SYNC = b"\x00" + b"\xff" * 10 + b"\x00"
CUE_MAX = 1 << 20                             # bytes of a cue sheet read
CHUNK_SECTORS = 256
OGG_ENC_BYTES = 24000                         # bytes per second of Vorbis at oggenc's default (q3, ~14 KB/s) and more
DOWNLOAD_BYTES = 128 << 20                    # official patch files gdp may download and unpack (Quake II 3.20: 38 MB)
MUSIC_TRACK_BYTES = 12 << 20                  # a ripped track's Ogg file, for the space check (~6 minutes)
CD_RIP_WAV_BYTES = 110 << 20                  # gdp's rip.wav of one track (10 minutes of CD audio)
OGGENC, NICE = "/usr/bin/oggenc", "/usr/bin/nice"
OGGENC_JOBS = 2                               # the PS3's PPE has two threads


class Track:
    """one track of a cue sheet or an MDS descriptor: number, mode, its file and file type, the byte range in
    that file; layout: (bytes per sector, offset of the user data or None for audio), by default the cue
    mode's (an MDF's 2448-byte sectors carry 96 bytes of subchannel after the 2352)"""

    def __init__(self, number, mode, path, ftype, layout=None):
        self.number, self.mode, self.path, self.ftype = number, mode, path, ftype
        self.layout = layout or TRACK_MODES[mode]
        self.indexes = {}                    # index number: frame (1/75 s) in the file
        self.start = self.end = 0

    @property
    def sector(self):
        return self.layout[0]

    @property
    def audio(self):
        return self.layout[1] is None

    @property
    def sectors(self):
        return (self.end - self.start) // self.sector


def msf_frames(text):
    m = re.fullmatch(r"(\d{1,3}):(\d{2}):(\d{2})", text)
    if not m or int(m.group(2)) > 59 or int(m.group(3)) > 74:
        raise ValueError(text)
    return (int(m.group(1)) * 60 + int(m.group(2))) * 75 + int(m.group(3))


def cue_file(base, name, sheet="the cue sheet"):
    """the cue's FILE name (or an MDS's image file) as a path next to it (its directory, then a
    case-insensitive match: cue sheets from Windows often differ in case from the files)"""
    path = os.path.join(base, name.replace("\\", "/"))
    if os.path.isfile(path):
        return path
    d, want = os.path.dirname(path), os.path.basename(path).lower()
    try:
        for n in os.listdir(d):
            if n.lower() == want and os.path.isfile(os.path.join(d, n)):
                return os.path.join(d, n)
    except OSError:
        pass
    raise Refused("%s names %s, which is not next to it" % (sheet, name))


def parse_cue(path, size_of=os.path.getsize):
    """[Track] of a cue sheet, each with its byte range in its file: from its INDEX 01 to the next track's
    INDEX 00 (its pregap) or 01 in the same file, the last one of a file to the file's end. Offsets add up
    track by track, so a file mixing sector sizes works. Refused for what can't be read."""
    try:
        with open(path, "rb") as f:
            raw = f.read(CUE_MAX + 1)
    except OSError as e:
        raise Refused("cannot read %s: %s" % (path, e.strerror))
    if len(raw) > CUE_MAX or b"\0" in raw:
        raise Refused("%s is not a cue sheet" % path)
    text = raw.decode("utf-8-sig", "strict") if raw[:3] == b"\xef\xbb\xbf" else raw.decode("latin-1")
    base, tracks, cur_file = os.path.dirname(os.path.abspath(path)), [], None
    for n, line in enumerate(text.splitlines(), 1):
        line = line.strip()
        word = line.split(None, 1)[0].upper() if line else ""
        try:
            if word == "FILE":
                m = re.fullmatch(r'FILE\s+(?:"([^"]+)"|(\S+))\s+(\S+)', line, re.I)
                if not m:
                    raise ValueError(line)
                ftype = m.group(3).upper()
                if ftype not in CUE_FILE_TYPES:
                    raise Refused("%s: FILE type %s is not supported (only %s)" % (path, ftype, ", ".join(CUE_FILE_TYPES)))
                cur_file = (cue_file(base, m.group(1) or m.group(2)), ftype)
            elif word == "TRACK":
                m = re.fullmatch(r"TRACK\s+(\d{1,2})\s+(\S+)", line, re.I)
                if not m or cur_file is None:
                    raise ValueError(line)
                mode = m.group(2).upper()
                if mode not in TRACK_MODES:
                    raise Refused("%s: track %s is %s, which is not supported" % (path, m.group(1), mode))
                tracks.append(Track(int(m.group(1)), mode, *cur_file))
            elif word == "INDEX":
                m = re.fullmatch(r"INDEX\s+(\d{1,2})\s+(\S+)", line, re.I)
                if not m or not tracks:
                    raise ValueError(line)
                tracks[-1].indexes[int(m.group(1))] = msf_frames(m.group(2))
        except ValueError:
            raise Refused("%s line %d is not a cue sheet line: %s" % (path, n, line[:80]))
    if not tracks:
        raise Refused("%s has no tracks: not a cue sheet" % path)
    by_file = {}
    for t in tracks:
        if 1 not in t.indexes:
            raise Refused("%s: track %d has no INDEX 01" % (path, t.number))
        by_file.setdefault(t.path, []).append(t)
    for fpath, ts in by_file.items():
        pos, frame, size = 0, 0, size_of(fpath)
        for i, t in enumerate(ts):
            # bytes up to this track's INDEX 01 (frames before it were the previous track's sector size)
            prev = ts[i - 1].sector if i else t.sector
            t.start = pos + (t.indexes[1] - frame) * prev
            pos, frame = t.start, t.indexes[1]
        for i, t in enumerate(ts):
            if i + 1 < len(ts):
                nxt = ts[i + 1]
                t.end = t.start + (min(nxt.indexes.values()) - t.indexes[1]) * t.sector
            else:
                t.end = t.start + (size - t.start) // t.sector * t.sector
            if t.end <= t.start or t.end > size:
                raise Refused("%s: track %d is outside %s" % (path, t.number, os.path.basename(fpath)))
    return tracks


def iso_volume_bytes(pvd):
    """the ISO 9660 volume size from its primary volume descriptor (sector 16), or None"""
    if len(pvd) >= 88 and pvd[:6] == b"\x01CD001":
        blocks, block = int.from_bytes(pvd[80:84], "little"), int.from_bytes(pvd[128:130], "little") or 2048
        return blocks * block
    return None


def write_iso(track, dest):
    """the 2048-byte user data of a data track's sectors written to dest (cut to the ISO 9660 volume's
    size when the track holds one); its size. Refused when the sectors are not the mode the cue says."""
    size, off = track.layout
    n, written, limit = track.sectors, 0, None
    with open(track.path, "rb") as src, open(dest, "wb") as out:
        src.seek(track.start)
        done = 0
        while done < n and (limit is None or written < limit):
            k = min(CHUNK_SECTORS, n - done)
            buf = src.read(k * size)
            if len(buf) < k * size:
                raise Refused("%s ends early" % track.path)
            if done == 0 and size >= 2352:
                if buf[:12] != SYNC or buf[15] not in (1, 2):
                    raise Refused("track %d of %s is not %s data" % (track.number, os.path.basename(track.path),
                                                                      track.mode))
            data = b"".join(buf[i * size + off:i * size + off + 2048] for i in range(k))
            if limit is None and done <= 16 < done + k:
                limit = iso_volume_bytes(data[(16 - done) * 2048:(17 - done) * 2048])
            if limit is not None:
                data = data[:max(0, limit - written)]
            out.write(data)
            written += len(data)
            done += k
    return written


def wav_header(nbytes):
    """a RIFF/WAVE header for nbytes of 16-bit stereo 44.1 kHz PCM"""
    return (b"RIFF" + (36 + nbytes).to_bytes(4, "little") + b"WAVEfmt " + (16).to_bytes(4, "little")
            + (1).to_bytes(2, "little") + (2).to_bytes(2, "little") + (44100).to_bytes(4, "little")
            + (176400).to_bytes(4, "little") + (4).to_bytes(2, "little") + (16).to_bytes(2, "little")
            + b"data" + nbytes.to_bytes(4, "little"))


def write_wav(track, out):
    """an audio track as a WAV stream to the binary file object out: CD-DA in a .bin is 16-bit little-endian
    stereo (a MOTOROLA file is big-endian: swapped; an MDF's subchannel bytes after each 2352 left out); the
    bytes of audio written"""
    size = track.sector
    n = track.sectors * 2352
    out.write(wav_header(n))
    left = track.sectors * size
    with open(track.path, "rb") as src:
        src.seek(track.start)
        while left:
            buf = src.read(min(left, CHUNK_SECTORS * size))
            if not buf or len(buf) % size:
                raise Refused("%s ends early" % track.path)
            left -= len(buf)
            if size != 2352:
                buf = b"".join(buf[i:i + 2352] for i in range(0, len(buf), size))
            if track.ftype == "MOTOROLA":
                b = bytearray(buf)
                b[0::2], b[1::2] = buf[1::2], buf[0::2]
                buf = bytes(b)
            out.write(buf)
    return n


def track_seconds(track):
    return track.sectors * 2352 / 176400


def encode_tracks(tracks, layout, dest, first, last, jobs=OGGENC_JOBS):
    """the audio tracks numbered first..last encoded to Ogg Vorbis (oggenc, its default quality, as
    game-data-packager's own rip does) at dest/(layout % number), the WAV streamed on oggenc's stdin (no
    WAV on disk); jobs encoders at a time. [paths]; Failed when oggenc is missing or fails."""
    if not os.path.exists(OGGENC):
        raise Failed("encoding the CD's music needs oggenc: sudo apt install vorbis-tools")
    want = [t for t in tracks if t.audio and first <= t.number <= last]
    running, made = [], []

    def feed(p, t, errors):
        try:
            write_wav(t, p.stdin)
        except BrokenPipeError:
            pass
        except (OSError, Refused) as e:
            errors.append(str(e))
        finally:
            try:
                p.stdin.close()
            except OSError:
                pass

    def finish(p, th, t, path, errors):
        th.join()
        rc = p.wait()
        if rc != 0 or errors:
            raise Failed("encoding track %d failed: %s" % (t.number, errors[0] if errors else "oggenc exit %d" % rc))
        made.append(path)

    try:
        for t in want:
            path = os.path.join(dest, layout % t.number)
            os.makedirs(os.path.dirname(path), 0o700, exist_ok=True)
            secs = int(track_seconds(t))
            say("gdp", "step encoding the CD's music: track %d of %d (%d:%02d)" % (t.number, last, secs // 60, secs % 60))
            p = subprocess.Popen([NICE, "-n", "10", OGGENC, "-Q", "-o", path, "-"], stdin=subprocess.PIPE,
                                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            errors = []
            th = threading.Thread(target=feed, args=(p, t, errors), daemon=True)
            th.start()
            running.append((p, th, t, path, errors))
            while len(running) >= jobs:
                finish(*running.pop(0))
        while running:
            finish(*running.pop(0))
    finally:
        for p, th, _, _, _ in running:
            p.kill()
            p.wait()
            th.join()
    return made


def cue_for(path):
    """the cue sheet of a SOURCE: path itself (.cue), or the .cue next to a .bin with its name; None for
    anything else"""
    low = path.lower()
    if low.endswith(".cue"):
        return path
    if low.endswith(".bin"):
        stem = os.path.splitext(os.path.basename(path))[0].lower()
        d = os.path.dirname(path)
        try:
            for n in sorted(os.listdir(d)):
                if n.lower() == stem + ".cue":
                    return os.path.join(d, n)
        except OSError:
            pass
        raise Refused("%s needs its .cue sheet next to it (it says where the tracks are)" % path)
    return None


# ---- Alcohol 120% MDF/MDS images ------------------------------------------------------------

MDS_SIGNATURE = b"MEDIA DESCRIPTOR"
MDS_MAX = 1 << 20                             # bytes of an .mds read
MDS_DVD = 0x10                                # the medium type's DVD bit (0x10 DVD-ROM, 0x12 DVD-R)
# a track block's mode, its low four bits (Alcohol sets 0xA0 or 0xE0 above them): 0xC is mode 2 form 1;
# 0x0 (unknown) and 0xD (mode 2 form 2) are refused
MDS_MODES = {0x9: "AUDIO", 0xA: "MODE1", 0xB: "MODE2", 0xC: "MODE2"}
# (mode, bytes per sector in the .mdf): the offset of the 2048 bytes of user data, None for audio; 2448 is
# 2352 + 96 bytes of interleaved P-W subchannel
MDS_LAYOUTS = {("AUDIO", 2352): None, ("AUDIO", 2448): None, ("MODE1", 2048): 0, ("MODE1", 2352): 16,
               ("MODE1", 2448): 16, ("MODE2", 2336): 8, ("MODE2", 2352): 24, ("MODE2", 2448): 24}


def mds_image_name(mds, raw, footer):
    """the image file an MDS footer names, next to the .mds: "*.mdf" is the .mds's own name with that
    extension; a footer at 0 means *.mdf"""
    name = "*.mdf"
    if footer:
        off, wide = struct.unpack_from("<II", raw, footer)
        if wide:
            end = off
            while raw[end:end + 2] not in (b"\0\0", b"", b"\0"):
                end += 2
            name = raw[off:end].decode("utf-16-le", "replace")
        else:
            end = raw.find(b"\0", off)
            name = raw[off:end if end >= 0 else len(raw)].decode("latin-1")
    m = re.fullmatch(r"\*\.(\w+)", name)
    if m:
        name = os.path.splitext(os.path.basename(mds))[0] + "." + m.group(1)
    if not name or "/" in name.replace("\\", "/"):
        raise Refused("%s names no image file it can use: %r" % (mds, name))
    return cue_file(os.path.dirname(os.path.abspath(mds)), name, "the .mds")


def parse_mds(path, size_of=os.path.getsize):
    """[Track] of an Alcohol 120% descriptor (version 1.x: "MEDIA DESCRIPTOR" 1.3, 1.5), each with its byte
    range in the .mdf: from the track block's start offset (an MDF holds no pregaps) for the extra block's
    length in sectors (a DVD: to the file's end); its lead-in entries (points 0xA0-0xA2) left out. Refused
    for what isn't read: version 2 (.mdx-style), a split image (.md0, .md1...), mode 2 form 2, an unknown
    mode, another sector size."""
    try:
        with open(path, "rb") as f:
            raw = f.read(MDS_MAX + 1)
    except OSError as e:
        raise Refused("cannot read %s: %s" % (path, e.strerror))
    if len(raw) > MDS_MAX or raw[:16] != MDS_SIGNATURE:
        raise Refused("%s is not an Alcohol 120%% .mds descriptor" % path)
    if raw[16] != 1:
        raise Refused("%s is a version %d.%d descriptor: only version 1 (Alcohol 120%%'s .mds) is read"
                      % (path, raw[16], raw[17]))
    tracks = []
    try:
        medium, sessions = struct.unpack_from("<HH", raw, 18)
        sessions_at = struct.unpack_from("<I", raw, 80)[0]
        for s_i in range(sessions):
            so = sessions_at + 24 * s_i
            blocks, blocks_at = raw[so + 10], struct.unpack_from("<I", raw, so + 20)[0]
            for b_i in range(blocks):
                o = blocks_at + 80 * b_i
                mode_byte, point = raw[o], raw[o + 4]
                if not 1 <= point <= 99:
                    continue
                extra_at, size = struct.unpack_from("<IH", raw, o + 12)
                start, files, footer = struct.unpack_from("<QII", raw, o + 40)
                kind = MDS_MODES.get(mode_byte & 0x0F)
                if kind is None:
                    raise Refused("%s: track %d has mode 0x%02x (%s), which is not supported" % (
                        path, point, mode_byte, "mode 2 form 2" if mode_byte & 0x0F == 0xD else "unknown"))
                if (kind, size) not in MDS_LAYOUTS:
                    raise Refused("%s: track %d is %s in %d-byte sectors, which is not supported"
                                  % (path, point, kind, size))
                if files > 1:
                    raise Refused("%s: track %d is split over %d files (.md0, .md1...), which is not supported"
                                  % (path, point, files))
                image = mds_image_name(path, raw, footer)
                t = Track(point, "%s/%d" % (kind, size), image, "BINARY", (size, MDS_LAYOUTS[(kind, size)]))
                t.start = start
                if medium & MDS_DVD or not extra_at:
                    t.end = None               # to the next track in the file, or the file's end
                else:
                    t.end = start + struct.unpack_from("<I", raw, extra_at + 4)[0] * size
                tracks.append(t)
    except (struct.error, IndexError):
        raise Refused("%s is cut short: not a whole .mds descriptor" % path)
    if not tracks:
        raise Refused("%s has no tracks" % path)
    tracks.sort(key=lambda t: t.number)
    sizes = {}
    for i, t in enumerate(tracks):
        size = sizes.setdefault(t.path, size_of(t.path))
        if t.end is None:
            nxt = [u.start for u in tracks[i + 1:] if u.path == t.path and u.start > t.start]
            stop = min(nxt) if nxt else size
            t.end = t.start + (stop - t.start) // t.sector * t.sector
        if t.end <= t.start or t.end > size:
            raise Refused("%s: track %d is outside %s" % (path, t.number, os.path.basename(t.path)))
    return tracks


def sniff_mdf(path):
    """a lone .mdf (no .mds next to it): "iso" when it holds 2048-byte sectors (an ISO 9660 volume at
    sector 16: a plain image), else [one data Track] from its raw sectors (2352, or 2448 with subchannel:
    the sync pattern at the start of the first two sectors; mode 1 or 2 from the first header) to the
    file's end, which write_iso cuts at the volume's size; Refused for anything else"""
    try:
        with open(path, "rb") as f:
            head = f.read(2 * 2448)
            f.seek(16 * 2048)
            pvd = f.read(6)
        size = os.path.getsize(path)
    except OSError as e:
        raise Refused("cannot read %s: %s" % (path, e.strerror))
    if pvd == b"\x01CD001":
        return "iso"
    sizes = [n for n in (2352, 2448) if head[:12] == SYNC and head[n:n + 12] == SYNC]
    if len(sizes) != 1 or head[15] not in (1, 2):
        raise Refused("%s is neither a plain image (ISO 9660 in 2048-byte sectors) nor raw CD sectors of a "
                      "data track: give its .mds" % path)
    n = sizes[0]
    kind = "MODE1" if head[15] == 1 else "MODE2"
    t = Track(1, "%s/%d" % (kind, n), path, "BINARY", (n, MDS_LAYOUTS[(kind, n)]))
    t.end = size // n * n
    return [t]


def mds_for(path):
    """the descriptor of a SOURCE: path itself (.mds), the .mds next to an .mdf with its name, or "" for an
    .mdf without one; None for anything else"""
    low = path.lower()
    if low.endswith(".mds"):
        return path
    if low.endswith(".mdf"):
        stem = os.path.splitext(os.path.basename(path))[0].lower()
        d = os.path.dirname(path)
        try:
            for n in sorted(os.listdir(d)):
                if n.lower() == stem + ".mds" and os.path.isfile(os.path.join(d, n)):
                    return os.path.join(d, n)
        except OSError:
            pass
        return ""
    return None


# ---- the CD in the drive: its audio tracks ------------------------------------------------------

CDROMREADTOCHDR, CDROMREADTOCENTRY, CDROM_LBA, CDROM_DATA_TRACK = 0x5305, 0x5306, 0x01, 0x04


def disc_audio_tracks(dev):
    """the audio track numbers of the CD in drive dev (its table of contents, linux/cdrom.h ioctls), [] for
    a disc without audio, None when it can't be read (no disc, not a CD drive)"""
    try:
        fd = os.open(dev, os.O_RDONLY | os.O_NONBLOCK | os.O_CLOEXEC)
    except OSError:
        return None
    try:
        hdr = fcntl.ioctl(fd, CDROMREADTOCHDR, bytes(2))
        first, last = hdr[0], hdr[1]
        audio = []
        for n in range(first, last + 1):
            # struct cdrom_tocentry: u8 track, u8 adr:4 ctrl:4, u8 format, union addr (int), u8 datamode
            e = fcntl.ioctl(fd, CDROMREADTOCENTRY, bytes([n, 0, CDROM_LBA]) + bytes(9))
            # the bit fields adr:4 ctrl:4 share a byte: ctrl is its high half on little-endian, low on big
            ctrl = e[1] >> 4 if sys.byteorder == "little" else e[1] & 0x0F
            if not ctrl & CDROM_DATA_TRACK:
                audio.append(n)
        return audio
    except OSError:
        return None
    finally:
        os.close(fd)


# ---- game-data-packager -----------------------------------------------------------------------

def cache_dir():
    base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache")
    return os.path.join(base, "walnut-data")


LOCK_NAME = "import.lock"


def take_import_lock(base, game_id):
    """the descriptor of base/import.lock, flocked and saying "PID GAME"; Refused (busy) when another
    import holds it"""
    fd = os.open(os.path.join(base, LOCK_NAME), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
    try:
        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except BlockingIOError:
        held = import_state(base)
        os.close(fd)
        raise Refused("busy: another import is running%s" % (" (%s)" % held[1] if held else ""))
    os.ftruncate(fd, 0)
    os.write(fd, ("%d %s\n" % (os.getpid(), game_id)).encode())
    return fd


def import_state(base=None):
    """(pid, game id) of the import running now (its lock held), or None"""
    try:
        fd = os.open(os.path.join(base or cache_dir(), LOCK_NAME), os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
    except OSError:
        return None
    try:
        try:
            fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
        except BlockingIOError:
            pid, game = os.read(fd, 200).decode("utf-8", "replace").split()[:2]
            return int(pid), game
        fcntl.flock(fd, fcntl.LOCK_UN)
        return None
    except ValueError:
        return None
    finally:
        os.close(fd)


def fresh_dirs(base):
    out, tmp = os.path.join(base, "out"), os.path.join(base, "tmp")
    for d in (out, tmp):
        shutil.rmtree(d, ignore_errors=True)
        os.makedirs(d, 0o700)
    return out, tmp


def gdp_features():
    """(--progress-fd, --no-launchers) as this gdp's --help lists them"""
    rc, out = run([GDP, "--help"])
    if rc == 127:
        raise Failed("game-data-packager is not installed (sudo apt install game-data-packager)")
    return "--progress-fd" in out, "--no-launchers" in out


def gdp_command(ds, pkgs, sources, out, download, progress_fd=None, no_launchers=True):
    cmd = [GDP, "--no-search", "--no-compress"]
    if no_launchers:
        cmd.append("--no-launchers")
    if progress_fd is not None:
        cmd += ["--progress-fd", str(progress_fd)]
    cmd += ["-n", "-d", out, download]
    for p in pkgs:
        cmd += ["-p", p]
    return cmd + [ds["gdp"]] + list(sources)


def user_manager():
    """whether systemd-run --user can start a unit here"""
    rc, _ = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "/bin/true"])
    return rc == 0


def private_network_works():
    """whether the user manager can give a unit a network namespace of its own (user namespaces)"""
    rc, out = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "-p", "PrivateNetwork=yes",
                   "/bin/cat", "/proc/net/dev"])
    ifaces = [l.split(":")[0].strip() for l in out.splitlines() if ":" in l]
    return rc == 0 and ifaces == ["lo"]


def unit_command(cmd, tmp, fifo, private_net):
    """cmd in a transient user unit; descriptor 3 opened on fifo inside it when fifo is given"""
    unit = [SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect",
            "-p", "MemoryMax=" + MEMORY_MAX, "-p", "RuntimeMaxSec=" + RUNTIME_MAX, "-p", "NoNewPrivileges=yes",
            "--setenv=TMPDIR=" + tmp, "--setenv=PATH=" + UNIT_PATH, "--setenv=LC_ALL=C.UTF-8"]
    if private_net:
        unit += ["-p", "PrivateNetwork=yes"]
    if fifo:
        return unit + ["/bin/sh", "-c", 'exec 3>"$0" && exec "$@"', fifo] + cmd
    return unit + cmd


def relay_progress(fd, errors, generated, count=None):
    """gdp's --progress-fd lines as 'gdp ...'; its error lines kept in errors, the debs it says it made in
    generated. With count (a ReadCount): a "count DONE TOTAL" line after each file line that counted, and
    "step build ..." when gdp starts building a package (its "step building NAME")"""
    with os.fdopen(fd, "rb") as f:
        for raw in f:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("gdp", line)
                if line.startswith("error "):
                    errors.append(line[6:])
                elif line.startswith("generated "):
                    generated.append(line[10:])
                elif count is not None and line.startswith("file "):
                    name, _, size = line[5:].rpartition(" ")
                    if size.isdigit() and count.file(name, int(size)):
                        say("count", "%d %d" % (count.done, count.total))
                elif count is not None and line.startswith("step building "):
                    say("step", "build building the package " + line[len("step building "):])


def run_gdp(cmd, tmp, base, sandbox, private_net, progress, count=None):
    """run gdp (cmd has '--progress-fd FD' to be filled in); its return code, its error lines (and a
    no-space one for an ENOSPC in its output) and the debs its progress lines say it made (count: the
    ReadCount its file lines go to)"""
    errors, generated = [], []
    env = dict(os.environ, TMPDIR=tmp)
    fifo = reader = keep = None
    if progress and sandbox:
        fifo = os.path.join(base, "progress.fifo")
        try:
            os.unlink(fifo)
        except FileNotFoundError:
            pass
        os.mkfifo(fifo, 0o600)
        rfd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK)
        keep = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)    # no EOF before gdp has even opened it
        os.set_blocking(rfd, True)
        full = unit_command([("3" if a == "FD" else a) for a in cmd], tmp, fifo, private_net)
        pass_fds = ()
    elif progress:
        rfd, wfd = os.pipe()
        full = [(str(wfd) if a == "FD" else a) for a in cmd]
        pass_fds = (wfd,)
    else:
        rfd = None
        full = unit_command(cmd, tmp, None, private_net) if sandbox else cmd
        pass_fds = ()
    if rfd is not None:
        reader = threading.Thread(target=relay_progress, args=(rfd, errors, generated, count), daemon=True)
        reader.start()
    try:
        p = subprocess.Popen(full, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             env=env, pass_fds=pass_fds)
    finally:
        for fd in pass_fds:
            os.close(fd)
    with p.stdout:
        for raw in p.stdout:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("log", line)
                if "No space left on device" in line:
                    errors.append("no-space " + line)
    rc = p.wait()
    if keep is not None:
        os.close(keep)
    if reader:
        reader.join(10)
    if fifo:
        try:
            os.unlink(fifo)
        except OSError:
            pass
    return rc, errors, generated


def built_debs(out, pkgs):
    """[(package, deb path)] gdp made in out, in the order of pkgs"""
    found = []
    files = sorted(os.listdir(out)) if os.path.isdir(out) else []
    for p in pkgs:
        for n in files:
            if n.startswith(p + "_") and n.endswith(".deb"):
                found.append((p, os.path.join(out, n)))
                break
    return found


def helper_lines(stream):
    rc_error = None
    for raw in stream:
        line = raw.decode("utf-8", "replace").rstrip()
        if line:
            kind = line.split(" ", 1)[0]
            if kind == "plan":
                continue      # the import's own plan (plan_line) has the helper's steps
            print(line if kind in ("progress", "log", "error", "status", "step", "count") else "log " + line,
                  flush=True)
            if kind == "error":
                rc_error = line
    return rc_error


def import_deb(game_id, pkg, deb):
    """dpkg-deb --fsys-tarfile deb | sudo -n walnut-games-helper import-data game_id pkg: its exit code"""
    say("log", "importing %s" % pkg)
    # the deb unlinked once open: its space comes back as the stream ends (the helper's staging needs it)
    fd = os.open(deb, os.O_RDONLY | os.O_CLOEXEC)
    os.unlink(deb)
    try:
        tar = subprocess.Popen([DPKG_DEB, "--fsys-tarfile", "/dev/fd/%d" % fd], stdin=subprocess.DEVNULL,
                               stdout=subprocess.PIPE, pass_fds=(fd,))
    finally:
        os.close(fd)
    try:
        h = subprocess.Popen(SUDO + [HELPER, "import-data", game_id, pkg], stdin=tar.stdout, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError:
        tar.kill()
        tar.wait()
        raise
    tar.stdout.close()      # the helper has it; dpkg-deb gets SIGPIPE if the helper stops reading
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    trc = tar.wait()
    if rc == 0 and trc != 0:
        say("error", "dpkg-deb --fsys-tarfile %s failed (%d)" % (os.path.basename(deb), trc))
        return 1
    return rc


def load_manifest(path=None):
    try:
        with open(path or MANIFEST, encoding="utf-8") as f:
            return json.load(f)["packages"]
    except (OSError, ValueError, KeyError, TypeError):
        return None


def data_bytes(pkgs, path=None):
    """the bytes of the packages' files by the data manifest (each file at its largest size; a music
    package's tracks MUSIC_TRACK_BYTES each, their real size being unknown), or None when the manifest
    can't be read"""
    man = load_manifest(path)
    try:
        return sum(sum(max(a["size"] for a in alts) for alts in man[p]["files"].values())
                   + (music_tracks(man[p]) or 0) * MUSIC_TRACK_BYTES for p in pkgs if p in man)
    except (ValueError, KeyError, TypeError):
        return None


def music_tracks(entry):
    """the number of CD tracks of a music package's manifest entry, or None for another package"""
    t = entry.get("tracks") if isinstance(entry, dict) else None
    if isinstance(t, dict) and isinstance(t.get("first"), int) and isinstance(t.get("last"), int):
        return t["last"] - t["first"] + 1
    return None


def music_entries(pkgs, path=None):
    """{package: its manifest tracks entry} of the music packages (CD audio, ripped or encoded) among pkgs"""
    man = load_manifest(path) or {}
    out = {}
    for p in pkgs:
        e = man.get(p)
        if music_tracks(e) and isinstance(e["tracks"].get("source"), str) and e["tracks"]["source"].count("%02d") == 1:
            out[p] = e["tracks"]
    return out


class ReadCount:
    """gdp's "file NAME BYTES" lines (one per file it checks or unpacks) counted against the data manifest:
    a file of one of the packages (its base name, any case, at one of its sizes) counts once, at its size;
    total: each such file at its largest size, a file two of the packages share (quake-shareware's and
    quake-registered's pak0.pak) once. Music tracks are not counted (their size is unknown)."""

    def __init__(self, pkgs, man):
        self.slot, self.size, self.seen, self.done = {}, {}, set(), 0
        for p in pkgs:
            for path, alts in ((man or {}).get(p) or {}).get("files", {}).items():
                sizes = sorted({a["size"] for a in alts if isinstance(a.get("size"), int)})
                if not sizes:
                    continue
                slot = (os.path.basename(path).lower(), sizes[-1])
                self.size[slot] = sizes[-1]
                for n in sizes:
                    self.slot.setdefault((slot[0], n), slot)
        self.total = sum(self.size.values())

    def file(self, name, size):
        """a file line: True when it counted (a new file of the packages)"""
        slot = self.slot.get((os.path.basename(name.replace("//", "/")).lower(), size))
        if slot is None or slot in self.seen:
            return False
        self.seen.add(slot)
        self.done = min(self.total, self.done + size)
        return True


def package_bytes(pkg, man):
    """the bytes of one package's files as the helper's checked copy counts them (each at its largest size, a
    music track at the helper's expected size)"""
    e = (man or {}).get(pkg) or {}
    try:
        return sum(max(a["size"] for a in alts) for alts in e.get("files", {}).values()) + \
            (music_tracks(e) or 0) * MUSIC_TRACK_BYTES
    except (ValueError, KeyError, TypeError):
        return 0


def plan_line(read_total, imports, man):
    """the import's steps for Walnut Games' bar (walnut_games.Progress): gdp reads the source (read: the
    bytes ReadCount expects) and builds the debs (build), then for each package imported the helper's
    checked copy (check), its own deb (deb) and apt (apt), each with the package's bytes"""
    items = ["read:%d" % read_total, "build:%d" % read_total]
    for p in imports:
        b = package_bytes(p, man)
        items += ["check:%d" % b, "deb:%d" % b, "apt:%d" % b]
    return " ".join(items)


def expected_imports(ds, pkgs, asked):
    """the packages an import is expected to bring, for its first plan: every -p package; for a plain
    import the first of the game's base (gdp makes those the source has, one of them is enough), and the
    real list follows once gdp is done"""
    return list(pkgs) if asked else list(pkgs[:1])


def existing(path):
    while not os.path.exists(path) and os.path.dirname(path) != path:
        path = os.path.dirname(path)
    return path


def space_needs(total, archive, cache, var, usr, dev=None, extra=0, unpack=None):
    """{device: (bytes, [paths])}: the most an import has on each file system at one time. Its phases:
    gdp in the cache (its copy of the files, dpkg-deb's temporary data member, the deb, an installer's or
    archive's unpacked files (unpack bytes, by default total), and extra: a BIN/CUE's ISO and music, the
    downloads),
    the deb streamed to the helper's staging (the deb is unlinked as dpkg-deb opens it: gone once the
    stream ends), the helper's own deb made from its staging (with dpkg-deb's temporary copy of the
    data member), then dpkg installing it into /usr"""
    dev = dev or (lambda p: os.stat(existing(p)).st_dev)
    phases = [[(cache, 3 * total + ((total if unpack is None else unpack) if archive else 0) + extra)], [(cache, total), (var, total)], [(var, 3 * total)],
              [(var, total), (usr, total)]]
    need = {}
    for phase in phases:
        now = {}
        for path, n in phase:
            b, paths = now.get(dev(path), (0, []))
            now[dev(path)] = (b + n, paths + [path])
        for d, (b, paths) in now.items():
            old = need.get(d, (0, []))
            need[d] = (max(old[0], b), list(dict.fromkeys(old[1] + paths)))
    return need


def human(n):
    for unit, size in (("GB", 1 << 30), ("MB", 1 << 20)):
        if n >= size:
            return "%.1f %s" % (n / size, unit)
    return "%d KB" % max(1, n >> 10)


def check_space(total, archive, cache, var=IMPORT_DIR, usr=USR, free=None, extra=0, unpack=None):
    """Refused (with what is needed and what is free) when a file system can't hold the import; extra: the
    bytes the sources put in the cache while gdp runs (a BIN/CUE's ISO and Ogg files)"""
    free = free or (lambda p: (lambda st: st.f_bavail * st.f_frsize)(os.statvfs(existing(p))))
    for d, (need, paths) in space_needs(total, archive, cache, var, usr, extra=extra, unpack=unpack).items():
        have = free(paths[0])
        if have < need + SPACE_MARGIN:
            raise Refused("not enough space: importing this data needs about %s free on the disk of %s, and %s is free"
                          % (human(need + SPACE_MARGIN), ", ".join(paths), human(have)))


GDP_HINTS = {
    "not-found": "no files of this game were found there",
    "unknown-version": "this copy is a version game-data-packager doesn't know",
    "unsuitable": "this copy is a release game-data-packager refuses",
    "needs-download": "some files have to be downloaded: connect to the network, or add those files",
    "no-space": "not enough free space",
}


def gdp_hint(error):
    """why gdp made no package, from its last error line 'KIND TEXT' (unsuitable: its own reason, the text
    'FILE RELEASE: REASON')"""
    kind, _, text = (error or "").partition(" ")
    if kind == "unsuitable":
        head, sep, reason = text.partition(": ")
        release = head.partition(" ")[2]
        if sep and release:
            return "this copy is %s, which can't be used: %s" % (release, reason)
    return GDP_HINTS.get(kind)


def reload_menu():
    """ps3-game-menu --reload, like the app does after an install or remove; needs the desktop session,
    failures ignored"""
    rt = os.environ.get("XDG_RUNTIME_DIR")
    if not (rt and os.environ.get("DISPLAY") and os.path.exists(MENU_TOOL)):
        return
    try:
        subprocess.run([MENU_TOOL, "--reload", os.path.join(rt, "ps3-game-menu")], stdin=subprocess.DEVNULL,
                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60)
    except (OSError, subprocess.SubprocessError):
        pass


def report_missing(ds, pkgs, asked, debs, download, errors):
    """say what gdp did not make (see Partial results in the header); (required packages, those missing)"""
    made = {p for p, _ in debs}
    required = list(pkgs) if asked else [p for p in pkgs if p in names(ds, "required")] or list(pkgs)
    why = errors[-1] if errors else ""
    # a plain import: any one required package makes the game playable, so once one was made (or is
    # installed) the other required ones are not reported
    real = names(ds, "required")
    playable = not asked and any(p in made for p in real)
    if not asked and not playable and real:
        playable = bool(installed(real))
    for p in pkgs:
        if p in made or covered(ds, p, made) or (playable and p in real):
            continue
        if p in names(ds, "downloads") and download == "--no-download":
            say("gdp", "error needs-download %s: its official patch files have to be downloaded" % p)
        else:
            say("warning", "missing %s: %s" % (p, why or "game-data-packager did not make it"))
    return required, [p for p in required if p not in made]


def cmd_import(game_id, sources, asked, no_download, games):
    ds = data_setup(game_id, games)
    pkgs = wanted_packages(ds, asked)
    if not pkgs:
        raise Refused("%s names no data packages" % game_id)
    if not sources:
        if not asked or no_download or not all(p in names(ds, "fetch") for p in pkgs):
            free = names(ds, "fetch")
            raise Refused("no SOURCE given: a disc, an .iso, an installer or a folder%s" % (
                " (only %s can be downloaded without one: import -p %s %s)" % (", ".join(free), free[0], game_id)
                if free else ""))
    download = download_choice(ds, pkgs, no_download, not sources)
    if download == "--no-download" and names(ds, "downloads") and not no_download:
        say("log", "downloads off: import with -p %s alone to let it fetch its patch files"
            % " -p ".join(names(ds, "downloads")))
    progress, no_launchers = gdp_features()
    base = cache_dir()
    os.makedirs(base, 0o700, exist_ok=True)
    lock = take_import_lock(base, game_id)
    out, tmp = fresh_dirs(base)
    music = music_entries(pkgs)
    src = Sources(base, music.values())
    src.only_music = bool(music) and all(p in music for p in pkgs)
    try:
        plans = [src.plan(s) for s in sources]
        total = data_bytes(pkgs)
        if total:
            # an installer or archive: unpacked first
            archive = any(kind == "path" and os.path.isfile(what) for kind, what, _ in plans)
            # an installer is unpacked whole, whichever packages were asked for: what its listing adds up
            # to (GOG's Quake Enhanced, 607 MB, unpacks to 1137 MB), at least its own size
            unpack = max(total, sum(max(os.path.getsize(what), unpacked_bytes(what)) for kind, what, _ in plans
                                    if kind == "path" and os.path.isfile(what)))
            check_space(total, archive, base, unpack=unpack, extra=sum(extra for _, _, extra in plans)
                        + (DOWNLOAD_BYTES if download == "--download" else 0))
        man = load_manifest() or {}
        count = ReadCount(pkgs, man)
        say("plan", plan_line(count.total, expected_imports(ds, pkgs, asked), man))
        say("step", "read reading the files" + (" of %s" % ", ".join(os.path.basename(x.rstrip("/")) or x
                                                                       for x in sources) if sources else ""))
        paths = list(dict.fromkeys(p for plan in plans for p in src.open(plan)))
        sandbox = user_manager()
        private_net = sandbox and download == "--no-download" and private_network_works()
        if not sandbox:
            say("log", "no systemd user manager: game-data-packager runs without its sandbox")
        cmd = gdp_command(ds, pkgs, paths, out, download, "FD" if progress else None, no_launchers)
        say("log", "running game-data-packager %s for %s" % (ds["gdp"], " ".join(pkgs)))
        rc, errors, generated = run_gdp(cmd, tmp, base, sandbox, private_net, progress, count)
        if rc != 0:
            # a failed gdp may leave a deb it was still writing (ENOSPC): only those it said it made count
            for n in (os.listdir(out) if os.path.isdir(out) else []):
                if n.endswith(".deb") and os.path.join(out, n) not in generated:
                    say("log", "game-data-packager failed (exit %d): %s dropped, it may be cut short" % (rc, n))
                    os.unlink(os.path.join(out, n))
        debs = built_debs(out, pkgs)
        extra = sorted(n for n in (os.listdir(out) if os.path.isdir(out) else [])
                       if n.endswith(".deb") and os.path.join(out, n) not in [d for _, d in debs])
        if extra:
            raise Failed("game-data-packager also made %s, which was not asked for: nothing was imported"
                         % ", ".join(extra))
        required, missing_required = report_missing(ds, pkgs, asked, debs, download, errors)
        if not debs or len(missing_required) == len(required):
            hint = gdp_hint(next((e for e in errors if e.startswith("no-space ")), errors[-1] if errors else None))
            raise Failed("game-data-packager made no package%s" % (": " + hint if hint else
                                                                    " (exit %d)" % rc if rc else ""))
        addons = find_addons(ds, [p for p, _ in debs], paths, plans)    # while the disc or image is there
        src.close()            # the disc or image is no longer needed
        for pkg, by in superseded([p for p, _ in debs], ds).items():
            say("log", "%s not imported: %s takes its place" % (pkg, by))
            debs = [(p, d) for p, d in debs if p != pkg]
        say("plan", plan_line(count.total, [p for p, _ in debs], man))
        worst = 0
        for pkg, deb in debs:
            rc = import_deb(game_id, pkg, deb)
            if rc != 0:
                worst = rc if worst in (0, 2) else worst
        if worst == 2:
            raise Refused("the helper refused the data")
        if worst:
            raise Failed("the helper failed")
        for pkg in addons:
            say("addon", "available " + pkg)
        reload_menu()
        return 0
    finally:
        src.close()
        for d in (out, tmp):
            shutil.rmtree(d, ignore_errors=True)
        os.close(lock)


def superseded(made, ds, manifest=None):
    """{package: the package that replaces it} for the made packages another made or installed package of the
    game replaces (the data manifest's replaces: quake-registered replaces quake-shareware), which are not
    imported: a plain import of a full Quake CD makes both, and the shareware's import would take an installed
    full game away (its Conflicts) before the full one came back"""
    man = manifest if manifest is not None else (load_manifest() or {})
    have = installed(names(ds, "required") + names(ds, "optional"))
    out = {}
    for p in made:
        for q in list(made) + sorted(have):
            e = man.get(q)
            if q != p and isinstance(e, dict) and p in (e.get("replaces") or []):
                out[p] = q + ("" if q in made else " (installed)")
                break
    return out


def size_bytes(num, unit):
    """(bytes, tolerance) of a size as innoextract prints it: '1234', '180.8 MiB'; the tolerance is one unit of
    the last digit shown, since it may cut the digits off instead of rounding (496862633 bytes: '473 MiB')"""
    mult = {"": 1, "B": 1, "KIB": 1 << 10, "MIB": 1 << 20, "GIB": 1 << 30}.get(unit.upper(), None)
    if mult is None:
        return None
    dec = len(num.partition(".")[2])
    return int(float(num) * mult), (0 if mult == 1 and not dec else mult * 10 ** -dec)


def listed_files(path):
    """[(lower-case base name, (bytes, tolerance) or None)] of the files a source holds, without unpacking
    them: a folder is walked, a zip (GOG's .sh installers carry one) read, an installer listed by
    `innoextract --list`; [] when it can't be looked at"""
    out = []
    if os.path.isdir(path):
        for root, _, files in os.walk(path):
            for n in files:
                try:
                    out.append((n.lower(), (os.stat(os.path.join(root, n)).st_size, 0)))
                except OSError:
                    pass
            if len(out) > LIST_MAX:
                break
        return out
    try:
        if zipfile.is_zipfile(path):
            with zipfile.ZipFile(path) as z:
                return [(i.filename.replace("\\", "/").rpartition("/")[2].lower(), (i.file_size, 0))
                        for i in z.infolist()[:LIST_MAX] if not i.is_dir()]
    except (OSError, ValueError, zipfile.BadZipFile):
        pass
    try:
        rc, text = run([INNOEXTRACT, "--list", path], timeout=LIST_TIMEOUT)
    except subprocess.TimeoutExpired:
        return []
    for line in text.splitlines()[:LIST_MAX]:
        m = re.match(r'\s*-?\s*"([^"]+)"(?:.*\((\d+(?:\.\d+)?)\s*([KMG]i?B|B)?\)\s*$)?', line)
        if rc == 0 and m:
            out.append((m.group(1).replace("\\", "/").rpartition("/")[2].lower(),
                        size_bytes(m.group(2), m.group(3) or "") if m.group(2) else None))
    return out


def unpacked_bytes(path):
    """the bytes an installer's or archive's files take unpacked, summed from its listing (listed_files);
    0 when it can't be listed"""
    return sum(sz[0] for _, sz in listed_files(path) if sz)


def has_file(listed, name, sizes):
    """a listed file called name (case-insensitively) of one of these sizes (an unknown size matches)"""
    base = name.rpartition("/")[2].lower()
    for n, sz in listed:
        if n == base and (sz is None or not sizes or any(abs(sz[0] - s) <= sz[1] for s in sizes)):
            return True
    return False


def find_addons(ds, made, paths, plans, manifest=None):
    """the optional packages of the game (not made now, not installed) whose required files the sources
    also hold, found by file names and sizes against the data manifest (cheap: nothing is unpacked or
    read); a music package when a source has audio tracks. Never raises: an add-on is only a hint."""
    try:
        man = manifest if manifest is not None else load_manifest()
        pending = [p for p in names(ds, "optional") if p not in made and not covered(ds, p, made)]
        pending = [p for p in pending if p not in installed(pending)]
        if not man or not pending:
            return []
        listed = None
        audio = any(kind == "disc" and any(t.audio for t in what[1]) for kind, what, _ in plans)
        found = []
        for pkg in pending:
            e = man.get(pkg)
            if not isinstance(e, dict):
                continue
            if music_tracks(e):
                if audio:
                    found.append(pkg)
                continue
            if listed is None:
                listed = [f for p in paths for f in listed_files(p)]
            need = [r for r in e.get("required") or [] if isinstance(r, str)] or list(e.get("files") or [])
            if need and all(has_file(listed, r, [a["size"] for a in e["files"].get(r, []) if "size" in a])
                            for r in need):
                found.append(pkg)
        return found
    except (OSError, ValueError, KeyError, TypeError, AttributeError) as e:
        say("log", "add-ons not looked for: %s" % e)
        return []


def cmd_helper(action, game_id, games):
    data_setup(game_id, games)
    try:
        h = subprocess.Popen(SUDO + [HELPER, action, game_id], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError as e:
        raise Failed(str(e))
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    return rc if rc in (0, 1, 2) else 1


def parse(argv):
    ap = argparse.ArgumentParser(prog="walnut-data", description="The player's own game data, made into packages.")
    sub = ap.add_subparsers(dest="cmd", required=True)
    sub.add_parser("list", help="the games that take data, and whether theirs is installed")
    p = sub.add_parser("import", help="make and install the game's data packages from SOURCE (a disc, an .iso, "
                                      "an installer, a folder)")
    p.add_argument("-p", "--package", action="append", default=[], help="only this data package (repeatable)")
    p.add_argument("--no-download", action="store_true", help="never download, even the official patch files")
    p.add_argument("game")
    p.add_argument("sources", nargs="*", metavar="SOURCE", help="optional when every -p package is one gdp "
                   "downloads entirely (the free shareware or demo)")
    for name, text in (("remove", "remove the game's data packages"),
                       ("install-repo", "install the game's data from your package sources")):
        sub.add_parser(name, help=text).add_argument("game")
    return ap.parse_args(argv)


def main(argv=None, catalog=None):
    try:
        a = parse(argv)
    except SystemExit as e:
        return 0 if e.code == 0 else 2
    try:
        games = load_games(catalog)
        if a.cmd == "list":
            return cmd_list(games)
        if a.cmd == "import":
            return cmd_import(a.game, a.sources, a.package, a.no_download, games)
        rc = cmd_helper({"remove": "remove-data", "install-repo": "install-repo-data"}[a.cmd], a.game, games)
        if rc == 0:
            reload_menu()
        return rc
    except Refused as e:
        say("error", str(e))
        return 2
    except Failed as e:
        say("error", str(e))
        return 1
    except (OSError, ValueError, KeyError) as e:
        say("error", "%s: %s" % (type(e).__name__, e))
        return 1
    except KeyboardInterrupt:
        say("error", "interrupted")
        return 1


if __name__ == "__main__":
    sys.exit(main())
