#!/usr/bin/python3 -I
"""walnut-data list | import [-p PKG]... [--no-download] GAME [SOURCE...] | remove GAME | install-repo GAME
(walnut-games package, /usr/bin): the player's own game data made into packages and installed, from a
terminal or the Walnut Games app. Runs as the player; only walnut-games-helper (through sudo -n) is root.

Design
- GAME is a catalog id (/usr/share/walnut-games/catalog.json) whose entry has a data_setup (GameDB
  build.toml [data_setup]): the game-data-packager game, the packages, which may download.
- list: every catalog game with a data_setup, its data installed or missing (any one of present_any;
  optional for a game that runs without), whether the configured apt sources have one of its data
  packages (the licensed suite: `walnut-data install-repo GAME`), its title.
- import: each SOURCE as the player has it. SOURCE may be left out when every -p package is in the game's
  `fetch` list (the free shareware or demo gdp downloads entirely: `import -p quake2-demo-data yquake2`);
  gdp then runs with --download and no source (still --no-search and -p); otherwise it is refused. An .iso or .img file is opened read-only with
  `udisksctl loop-setup -r -f` and mounted with `udisksctl mount -b` (the player's own polkit rights; no
  copy), a block device (the disc drive, /dev/sr0) is used at its mount point or mounted the same way,
  and whatever this program mounted or set up is released afterwards. The desktop (gvfs) mounts a new
  loop device by itself within seconds: when udisksctl mount says AlreadyMounted, that mount is used
  (waited for, AUTOMOUNT_WAIT) and unmounted afterwards, which frees the loop device (loop-delete only
  for one still attached). loop-setup needs the desktop session's polkit rights: over ssh it is refused
  (NotAuthorizedCanObtain), and the message says how to run it in the desktop's slice. A directory, an installer or
  archive file, or a mounted disc's path is given to game-data-packager as it is.
  BIN/CUE (a .cue, or a .bin with its .cue next to it; gdp reads neither): the cue sheet is parsed here
  (FILE BINARY|MOTOROLA, one or several files; TRACK MODE1/2352, MODE2/2352, MODE1/2048, MODE2/2336,
  AUDIO; INDEX 01 is where a track starts, the next track's INDEX 00 or 01 where it ends; WAVE files
  and other modes are refused). The first data track's 2048-byte user data (MODE1 after the 16-byte
  sync and header, MODE2 form 1 after the 8-byte subheader too) goes to ~/.cache/walnut-data/disc/
  NAME.iso, cut at the ISO 9660 volume's size, which is then opened like any .iso and removed after.
  When a music package is asked for (the manifest's tracks entry: quake-music, quake2-music), the
  audio tracks it names (first..last) are streamed as WAV (RIFF header, CD-DA's 16-bit little-endian
  stereo 44.1 kHz; MOTOROLA swapped) into oggenc at its default quality, two at a time (no WAV on
  disk), into disc/music/<the manifest's source layout> (id1/music/track02.ogg), the folder gdp is
  given: gdp takes already ripped tracks from there, so it needs no patch for this. Only music packages
  asked for: no ISO is made. A cue sheet is read before anything is written, so a broken one refuses
  at once; the space check counts the ISO and the Ogg files (OGG_ENC_BYTES per second) in the cache.
  The disc in the drive with a music package asked for: its table of contents (CDROMREADTOC* ioctls,
  the player's ACL on /dev/sr0) says whether it has audio tracks; then the device itself is given to gdp
  too, which rips the music with cdparanoia and oggenc (an audio CD that can't be mounted is fine
  then); without audio tracks the music is reported missing, never a rip that would fail gdp's whole
  run.
  game-data-packager (gdp) runs as the player in a transient user unit: `systemd-run --user --pipe
  --wait` with MemoryMax, RuntimeMaxSec and NoNewPrivileges, and PrivateNetwork when it must not
  download and the user manager can make one (checked by a probe: only "lo" in its /proc/net/dev);
  without a user manager it runs directly. Its command: `game-data-packager --no-search --no-compress
  --no-launchers --progress-fd 3 -n -d ~/.cache/walnut-data/out --download|--no-download -p PKG...
  GDPGAME SOURCE...` with TMPDIR=~/.cache/walnut-data/tmp (the PS3's /tmp is a small tmpfs). The
  packages: -p's, else the game's required and optional ones, each always passed as -p (without -p
  gdp builds whatever else it can, downloads included); a deb gdp made that was not asked for fails
  the import, nothing imported. --download only when every one of
  them is among the game's downloads (the official patch files gdp knows), else --no-download.
  A gdp without --progress-fd or --no-launchers (Debian's) runs without them: no progress lines, and
  the helper drops the Doom family's menu files anyway. In the unit, descriptor 3 is a FIFO in the
  cache directory (systemd-run --pipe passes only 0-2).
  Each package gdp made is then `dpkg-deb --fsys-tarfile X.deb | sudo -n walnut-games-helper
  import-data GAME PKG` (the deb unlinked as soon as dpkg-deb has it open, so its space comes back
  when the stream ends): the helper checks every file against its manifest and builds and installs
  the package itself (required packages first, in the catalog's order). out/ and tmp/ are emptied
  before and removed after.
  Free space is checked before gdp runs, from the data manifest's sizes of the packages (T): the most
  each file system holds at one time over the phases (gdp: 3T in the cache, its copy of the files,
  dpkg-deb's temporary data member and the deb (a Quake II import ran out of space at 2T), 4T for an
  installer or archive, plus what the sources put there (a BIN/CUE's ISO and music) and DOWNLOAD_BYTES
  when it may download; streaming: T cache + T staging; the helper's deb: 3T staging, its files, dpkg-deb's
  temporary data member and the deb; installing: T staging +
  T /usr), summed where they share a disk; refused with what is needed and what is free.
  One import at a time per player: import holds an flock on ~/.cache/walnut-data/import.lock for its
  whole run, the file saying "PID GAME"; a second import is refused (busy, exit 2). The Walnut Games
  app reads that lock to show "importing" when it is opened again during an import (import_state).
- remove: `walnut-games-helper remove-data GAME` (the game's data packages only). install-repo:
  `walnut-games-helper install-repo-data GAME` (its data from the configured apt sources).
- Output, one item per line, flushed: "gdp step|file|generated|error ..." (gdp's progress lines),
  "log text" (gdp's own output and notes), the helper's lines as they come ("progress N text",
  "log text", "error text"), "error text" for this program's own failures.
- Exit codes: 0 done; 2 refused, nothing changed (bad arguments, an unknown game, a missing source,
  the helper refused); 1 failed (gdp or the helper failed, or gdp made no package).
- Partial results. After gdp, the packages asked for are compared with the debs it made. Required means
  the game's required packages, or every -p package when -p was given. For each package that is missing:
    "gdp error needs-download PKG: TEXT"  PKG is one of the game's downloads and downloads were off
        (import with -p PKG alone, or without --no-download, lets gdp fetch its patch files); same shape as
        gdp's own error lines, so the app's "Try again with download" logic reads it
    "warning missing PKG: TEXT"           any other missing package (TEXT is gdp's last error line when it
        gave one, else "game-data-packager did not make it"); a missing optional package is only this
  Exit 0 when at least one required package was made (or, for a game with no required package, any
  package) and installed, even with such lines; exit 1 ("error game-data-packager made no package...")
  when none was. The lines come before the helper's output.
- After import, remove and install-repo succeed, `/usr/lib/ps3/desktop/ps3-game-menu --reload
  $XDG_RUNTIME_DIR/ps3-game-menu` runs when it exists and $DISPLAY and $XDG_RUNTIME_DIR are set (failures
  ignored), so the session's menu shows the new launchers like after the app's own installs.
- stdlib only, no Tk. The commands are module constants so that tests/test_walnut_games.py runs it
  with a stubbed gdp, dpkg-deb and helper.
"""
import argparse
import fcntl
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import threading
import time

CATALOG = "/usr/share/walnut-games/catalog.json"
HELPER = "/usr/lib/ps3/sbin/walnut-games-helper"
SUDO = ["/usr/bin/sudo", "-n"]
GDP = "/usr/games/game-data-packager"
DPKG_DEB, DPKG_QUERY, APT_CACHE = "/usr/bin/dpkg-deb", "/usr/bin/dpkg-query", "/usr/bin/apt-cache"
UDISKSCTL = "/usr/bin/udisksctl"
MENU_TOOL = "/usr/lib/ps3/desktop/ps3-game-menu"
SYSTEMD_RUN = "/usr/bin/systemd-run"
MEMORY_MAX = "192M"       # gdp ~40 MB with one game loaded, plus an unpacker (innoextract, unshield)
RUNTIME_MAX = "3h"        # a CD reads at about 2.3 MB/s
UNIT_PATH = "/usr/local/bin:/usr/bin:/bin:/usr/games"
ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,63}\Z")
PKG_RE = re.compile(r"[a-z0-9][a-z0-9+.-]*[a-z0-9]\Z")
IMAGE_SUFFIXES = (".iso", ".img")
AUTOMOUNT_WAIT = 10       # seconds to wait for the desktop's mount of a loop device it got to first
MANIFEST = "/usr/share/walnut-games/data-manifest.json"
IMPORT_DIR = "/var/lib/walnut-games/import"    # the helper's staging (root's)
USR = "/usr"
SPACE_MARGIN = 64 << 20
SYS_BLOCK = "/sys/block"


class Refused(Exception):
    """exit code 2: nothing was changed"""


class Failed(Exception):
    """exit code 1"""


def say(kind, text=""):
    try:
        sys.stdout.write((kind + " " + text).rstrip() + "\n")
        sys.stdout.flush()
    except (OSError, ValueError):
        pass


def run(cmd, **kw):
    """(returncode, stdout and stderr) of cmd; 127 when it is not there"""
    try:
        p = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
                           errors="replace", **kw)
    except OSError as e:
        return 127, str(e)
    return p.returncode, p.stdout


# ---- the catalog ------------------------------------------------------------------------------

def load_games(path=None):
    with open(path or CATALOG, encoding="utf-8") as f:
        return {g["id"]: g for g in json.load(f)["games"] if isinstance(g.get("id"), str)}


def data_setup(game_id, games):
    if not ID_RE.match(game_id or ""):
        raise Refused("not a game id: %r" % game_id)
    if game_id not in games:
        raise Refused("unknown game: %s (walnut-data list shows the games)" % game_id)
    ds = games[game_id].get("data_setup")
    if not isinstance(ds, dict) or not isinstance(ds.get("gdp"), str):
        raise Refused("%s has no data to set up" % game_id)
    return ds


def names(ds, key):
    return [p for p in ds.get(key) or [] if isinstance(p, str) and PKG_RE.match(p)]


def wanted_packages(ds, asked):
    """the packages to make: -p's (each one of the game's), else required then optional"""
    known = names(ds, "required") + names(ds, "optional")
    if asked:
        bad = [p for p in asked if p not in known]
        if bad:
            raise Refused("%s: not a data package of this game (%s)" % (", ".join(bad), ", ".join(known)))
        return list(dict.fromkeys(asked))
    return known


def download_choice(ds, pkgs, no_download=False, sourceless=False):
    """--download only when every package is one the game may download for, else --no-download; with no
    SOURCE the packages must be ones gdp fetches entirely (the game's fetch list)"""
    dl = set(names(ds, "fetch" if sourceless else "downloads"))
    return "--download" if pkgs and not no_download and all(p in dl for p in pkgs) else "--no-download"


def installed(pkgs):
    if not pkgs:
        return set()
    rc, out = run([DPKG_QUERY, "-W", "-f", "${Package}\t${db:Status-Abbrev}\n"] + list(pkgs))
    return {n for line in out.splitlines() for n, _, st in [line.partition("\t")] if n in pkgs and st[1:2] == "i"}


def candidates(pkgs):
    if not pkgs:
        return set()
    rc, out = run([APT_CACHE, "policy"] + list(pkgs))
    have, cur = set(), None
    for line in out.splitlines():
        m = re.match(r"(\S+):\s*$", line)
        if m:
            cur = m.group(1)
        elif cur and (m := re.match(r"\s+Candidate:\s*(\S+)", line)) and m.group(1) != "(none)":
            have.add(cur)
    return have


def cmd_list(games):
    rows = []
    all_names = sorted({p for g in games.values() if isinstance(g.get("data_setup"), dict)
                        for k in ("present_any", "required", "optional") for p in names(g["data_setup"], k)})
    inst, repo = installed(all_names), candidates(all_names)
    for gid in sorted(games):
        ds = games[gid].get("data_setup")
        if not isinstance(ds, dict):
            continue
        any_of = names(ds, "present_any") or names(ds, "required")
        opt = names(ds, "optional")
        if names(ds, "required"):
            state = "present" if inst & set(any_of) else "missing"
        else:
            state = "optional-present" if inst & set(opt) else "optional"
        avail = "repo" if repo & set(any_of) and state == "missing" else "-"
        rows.append((gid, state, avail, games[gid].get("title") or gid))
    w = max([len(r[0]) for r in rows] + [4])
    for r in rows:
        print("%-*s  %-16s  %-4s  %s" % (w, r[0], r[1], r[2], r[3]))
    return 0


# ---- sources ----------------------------------------------------------------------------------

def source_kind(path):
    """'image' (an .iso/.img file: loop-mounted), 'device' (a block device: its mount point), 'path'
    (a directory or any other file, as it is); Refused when it is not there"""
    try:
        st = os.stat(path)
    except OSError:
        raise Refused("no such file or folder: %s" % path)
    if stat.S_ISBLK(st.st_mode):
        return "device"
    if stat.S_ISREG(st.st_mode) and path.lower().endswith(IMAGE_SUFFIXES):
        return "image"
    return "path"


def unescape_mount(s):
    return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), s)


def mount_point(dev, mounts="/proc/self/mounts"):
    """where block device dev is mounted, or None"""
    try:
        real = os.path.realpath(dev)
        with open(mounts) as f:
            for line in f:
                parts = line.split()
                if len(parts) >= 2 and os.path.realpath(unescape_mount(parts[0])) == real:
                    return unescape_mount(parts[1])
    except OSError:
        pass
    return None


def already_mounted_at(out):
    """the mount point in udisksctl's AlreadyMounted error ("... is already mounted at `/run/media/u/X'.")"""
    m = re.search(r"already mounted at [`'\"]?(/.+?)['\"]?\.?\s*$", out.strip(), re.M)
    return m.group(1) if m else None


def loop_attached(dev, sys_block=None):
    """whether loop device dev still has a backing file"""
    return os.path.exists(os.path.join(sys_block or SYS_BLOCK, os.path.basename(dev), "loop", "backing_file"))


class Sources:
    """the SOURCE arguments as paths gdp reads; loop devices and mounts made here released by close(), the
    files made from a BIN/CUE (the data track's ISO, the music's Ogg files) removed by it. music: the
    manifest's tracks entries of the music packages asked for (music_entries)."""

    def __init__(self, base=None, music=None):
        self.mounted, self.loops, self.made = [], [], []
        self.base, self.music = base, list(music or [])
        self.only_music = False

    def plan(self, path):
        """(kind, path, extra): what open() will do with a SOURCE and the bytes it adds to the cache while
        gdp runs (the ISO and Ogg files of a BIN/CUE, gdp's WAV of a CD rip); Refused when it is not there.
        A cue sheet is read here, so a broken one refuses before anything is written."""
        path = os.path.abspath(path)
        cue = cue_for(path) if os.path.isfile(path) else None
        if cue:
            tracks = parse_cue(cue)
            data = next((t for t in tracks if not t.audio), None)
            extra = data.sectors * 2048 if data else 0
            for m in self.music:
                extra += sum(int(track_seconds(t) * OGG_ENC_BYTES) for t in tracks
                             if t.audio and m["first"] <= t.number <= m["last"])
            return ("cue", (cue, tracks), extra)
        kind = source_kind(path)
        return (kind, path, CD_RIP_WAV_BYTES + MUSIC_TRACK_BYTES if kind == "device" and self.music else 0)

    def open(self, plan):
        """[paths] for gdp from a plan"""
        kind, what, _ = plan
        if kind == "cue":
            return self.add_cue(*what)
        if kind == "device":
            return self.add_device(what)
        return [self.add(what)]

    def add_cue(self, cue, tracks):
        """a BIN/CUE: its first data track as an ISO in the cache, opened like any image; the music packages'
        audio tracks encoded to Ogg Vorbis in a folder laid out as gdp looks for ripped tracks
        (id1/music/track02.ogg), which gdp is given too"""
        name = os.path.splitext(os.path.basename(cue))[0]
        work = os.path.join(self.base or cache_dir(), "disc")
        shutil.rmtree(work, ignore_errors=True)
        os.makedirs(work, 0o700)
        self.made.append(work)
        paths = []
        data = next((t for t in tracks if not t.audio), None)
        if data and not self.only_music:
            iso = os.path.join(work, re.sub(r"[^A-Za-z0-9._-]", "_", name) + ".iso")
            say("gdp", "step extracting the data track of %s" % os.path.basename(cue))
            n = write_iso(data, iso)
            say("log", "track %d of %s (%s) written as %s, %s" % (data.number, os.path.basename(cue), data.mode, iso,
                                                                  human(n)))
            paths.append(self.add(iso))
        audio = [t for t in tracks if t.audio]
        for m in self.music:
            have = [t for t in audio if m["first"] <= t.number <= m["last"]]
            if not have:
                say("log", "%s has no audio tracks %d-%d for the music" % (os.path.basename(cue), m["first"], m["last"]))
                continue
            music = os.path.join(work, "music")
            encode_tracks(have, m["source"], music, m["first"], m["last"])
            if music not in paths:
                paths.append(music)
        if not paths:
            raise Refused("%s has nothing to import: no data track%s" % (cue, " and no music tracks" if self.music else ""))
        return paths

    def add_device(self, dev):
        """the disc in drive dev: its mount point (for the data), and the device itself for gdp to rip the
        music from when a music package is asked for and the disc has audio tracks"""
        paths, why = [], None
        if not self.only_music:
            try:
                paths.append(self.mount(dev))
            except Refused as e:
                if not self.music:
                    raise
                why = e
        if self.music:
            audio = disc_audio_tracks(dev)
            if audio:
                say("log", "the disc in %s has audio tracks %s: game-data-packager rips the music (cdparanoia)"
                    % (dev, ", ".join(map(str, audio))))
                paths.append(dev)
            else:
                say("log", "the disc in %s has no audio tracks for the music" % dev)
        if not paths:
            raise why or Refused("nothing to read on the disc in %s" % dev)
        return paths

    def mount(self, dev, ours=False):
        """dev's mount point, mounted with udisksctl when it is not; ours: dev is a loop device set up
        here, so whoever mounted it (the desktop's automounter does, within seconds) it is unmounted
        by close(), which frees the loop device too"""
        mp = mount_point(dev)
        if mp:
            if ours:
                self.mounted.append(dev)
            return mp
        rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" not in out and os.path.exists(dev + "p1"):     # an image with a partition table
            dev = dev + "p1"
            mp = mount_point(dev)
            if mp:
                if ours:
                    self.mounted.append(dev)
                return mp
            rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" in out:
            # mounted meanwhile by the desktop (gvfs mounts a new loop device by itself): use that mount
            mp = self.wait_mount(dev) or already_mounted_at(out)
            if mp and os.path.isdir(mp):
                if ours:
                    self.mounted.append(dev)
                say("log", "%s is mounted at %s" % (dev, mp))
                return mp
        if rc != 0:
            raise Refused("cannot mount %s: %s" % (dev, (out.strip().splitlines() or ["?"])[-1]))
        self.mounted.append(dev)
        mp = mount_point(dev)
        if not mp:
            m = re.search(r" at (.+?)\.?\s*$", out.strip())
            mp = m.group(1) if m else None
        if not mp or not os.path.isdir(mp):
            raise Refused("%s was mounted but its folder is not known: %s" % (dev, out.strip()))
        say("log", "mounted %s at %s" % (dev, mp))
        return mp

    @staticmethod
    def wait_mount(dev):
        end = time.monotonic() + AUTOMOUNT_WAIT
        while True:
            mp = mount_point(dev)
            if mp or time.monotonic() >= end:
                return mp
            time.sleep(0.2)

    def add(self, path):
        path = os.path.abspath(path)
        kind = source_kind(path)
        if kind == "path":
            return path
        if kind == "device":
            return self.mount(path)
        rc, out = run([UDISKSCTL, "loop-setup", "-r", "-f", path])
        m = re.search(r" as (/dev/\S+?)\.?\s*$", out.strip())
        if rc != 0 and "NotAuthorized" in out:
            raise Refused("opening the disc image %s needs the desktop session's rights (polkit): run walnut-data "
                          "in a terminal on the Walnut desktop or from Walnut Games; over ssh: systemd-run --user "
                          "--scope --slice=ps3-desktop.slice walnut-data import ...; or open the image in the file "
                          "manager and give its folder" % path)
        if rc != 0 or not m:
            raise Refused("cannot open %s: %s" % (path, (out.strip().splitlines() or ["?"])[-1]))
        self.loops.append(m.group(1))
        say("log", "opened %s as %s" % (path, m.group(1)))
        return self.mount(m.group(1), ours=True)

    def close(self):
        for dev in reversed(self.mounted):
            run([UDISKSCTL, "unmount", "-b", dev])
        for dev in reversed(self.loops):
            # unmounting frees a loop device udisks set up (autoclear); loop-delete only for one still there
            if loop_attached(dev):
                run([UDISKSCTL, "loop-delete", "-b", dev])
        for d in self.made:
            shutil.rmtree(d, ignore_errors=True)
        self.mounted, self.loops, self.made = [], [], []


# ---- BIN/CUE images ---------------------------------------------------------------------------

# a track's sector: (bytes per sector in the .bin, offset of the 2048 bytes of user data or None for audio)
TRACK_MODES = {"MODE1/2352": (2352, 16), "MODE2/2352": (2352, 24), "MODE1/2048": (2048, 0),
               "MODE2/2336": (2336, 8), "AUDIO": (2352, None)}
CUE_FILE_TYPES = ("BINARY", "MOTOROLA")       # MOTOROLA: big-endian audio samples
SYNC = b"\x00" + b"\xff" * 10 + b"\x00"
CUE_MAX = 1 << 20                             # bytes of a cue sheet read
CHUNK_SECTORS = 256
OGG_ENC_BYTES = 24000                         # bytes per second of Vorbis at oggenc's default (q3, ~14 KB/s) and more
DOWNLOAD_BYTES = 128 << 20                    # official patch files gdp may download and unpack (Quake II 3.20: 38 MB)
MUSIC_TRACK_BYTES = 12 << 20                  # a ripped track's Ogg file, for the space check (~6 minutes)
CD_RIP_WAV_BYTES = 110 << 20                  # gdp's rip.wav of one track (10 minutes of CD audio)
OGGENC, NICE = "/usr/bin/oggenc", "/usr/bin/nice"
OGGENC_JOBS = 2                               # the PS3's PPE has two threads


class Track:
    """one track of a cue sheet: number, mode, its file and file type, the byte range in that file"""

    def __init__(self, number, mode, path, ftype):
        self.number, self.mode, self.path, self.ftype = number, mode, path, ftype
        self.indexes = {}                    # index number: frame (1/75 s) in the file
        self.start = self.end = 0

    @property
    def sector(self):
        return TRACK_MODES[self.mode][0]

    @property
    def audio(self):
        return self.mode == "AUDIO"

    @property
    def sectors(self):
        return (self.end - self.start) // self.sector


def msf_frames(text):
    m = re.fullmatch(r"(\d{1,3}):(\d{2}):(\d{2})", text)
    if not m or int(m.group(2)) > 59 or int(m.group(3)) > 74:
        raise ValueError(text)
    return (int(m.group(1)) * 60 + int(m.group(2))) * 75 + int(m.group(3))


def cue_file(base, name):
    """the cue's FILE name as a path next to the cue (its directory, then a case-insensitive match: cue
    sheets from Windows often differ in case from the files)"""
    path = os.path.join(base, name.replace("\\", "/"))
    if os.path.isfile(path):
        return path
    d, want = os.path.dirname(path), os.path.basename(path).lower()
    try:
        for n in os.listdir(d):
            if n.lower() == want and os.path.isfile(os.path.join(d, n)):
                return os.path.join(d, n)
    except OSError:
        pass
    raise Refused("the cue sheet names %s, which is not next to it" % name)


def parse_cue(path, size_of=os.path.getsize):
    """[Track] of a cue sheet, each with its byte range in its file: from its INDEX 01 to the next track's
    INDEX 00 (its pregap) or 01 in the same file, the last one of a file to the file's end. Offsets add up
    track by track, so a file mixing sector sizes works. Refused for what can't be read."""
    try:
        with open(path, "rb") as f:
            raw = f.read(CUE_MAX + 1)
    except OSError as e:
        raise Refused("cannot read %s: %s" % (path, e.strerror))
    if len(raw) > CUE_MAX or b"\0" in raw:
        raise Refused("%s is not a cue sheet" % path)
    text = raw.decode("utf-8-sig", "strict") if raw[:3] == b"\xef\xbb\xbf" else raw.decode("latin-1")
    base, tracks, cur_file = os.path.dirname(os.path.abspath(path)), [], None
    for n, line in enumerate(text.splitlines(), 1):
        line = line.strip()
        word = line.split(None, 1)[0].upper() if line else ""
        try:
            if word == "FILE":
                m = re.fullmatch(r'FILE\s+(?:"([^"]+)"|(\S+))\s+(\S+)', line, re.I)
                if not m:
                    raise ValueError(line)
                ftype = m.group(3).upper()
                if ftype not in CUE_FILE_TYPES:
                    raise Refused("%s: FILE type %s is not supported (only %s)" % (path, ftype, ", ".join(CUE_FILE_TYPES)))
                cur_file = (cue_file(base, m.group(1) or m.group(2)), ftype)
            elif word == "TRACK":
                m = re.fullmatch(r"TRACK\s+(\d{1,2})\s+(\S+)", line, re.I)
                if not m or cur_file is None:
                    raise ValueError(line)
                mode = m.group(2).upper()
                if mode not in TRACK_MODES:
                    raise Refused("%s: track %s is %s, which is not supported" % (path, m.group(1), mode))
                tracks.append(Track(int(m.group(1)), mode, *cur_file))
            elif word == "INDEX":
                m = re.fullmatch(r"INDEX\s+(\d{1,2})\s+(\S+)", line, re.I)
                if not m or not tracks:
                    raise ValueError(line)
                tracks[-1].indexes[int(m.group(1))] = msf_frames(m.group(2))
        except ValueError:
            raise Refused("%s line %d is not a cue sheet line: %s" % (path, n, line[:80]))
    if not tracks:
        raise Refused("%s has no tracks: not a cue sheet" % path)
    by_file = {}
    for t in tracks:
        if 1 not in t.indexes:
            raise Refused("%s: track %d has no INDEX 01" % (path, t.number))
        by_file.setdefault(t.path, []).append(t)
    for fpath, ts in by_file.items():
        pos, frame, size = 0, 0, size_of(fpath)
        for i, t in enumerate(ts):
            # bytes up to this track's INDEX 01 (frames before it were the previous track's sector size)
            prev = ts[i - 1].sector if i else t.sector
            t.start = pos + (t.indexes[1] - frame) * prev
            pos, frame = t.start, t.indexes[1]
        for i, t in enumerate(ts):
            if i + 1 < len(ts):
                nxt = ts[i + 1]
                t.end = t.start + (min(nxt.indexes.values()) - t.indexes[1]) * t.sector
            else:
                t.end = t.start + (size - t.start) // t.sector * t.sector
            if t.end <= t.start or t.end > size:
                raise Refused("%s: track %d is outside %s" % (path, t.number, os.path.basename(fpath)))
    return tracks


def iso_volume_bytes(pvd):
    """the ISO 9660 volume size from its primary volume descriptor (sector 16), or None"""
    if len(pvd) >= 88 and pvd[:6] == b"\x01CD001":
        blocks, block = int.from_bytes(pvd[80:84], "little"), int.from_bytes(pvd[128:130], "little") or 2048
        return blocks * block
    return None


def write_iso(track, dest):
    """the 2048-byte user data of a data track's sectors written to dest (cut to the ISO 9660 volume's
    size when the track holds one); its size. Refused when the sectors are not the mode the cue says."""
    size, off = TRACK_MODES[track.mode]
    n, written, limit = track.sectors, 0, None
    with open(track.path, "rb") as src, open(dest, "wb") as out:
        src.seek(track.start)
        done = 0
        while done < n and (limit is None or written < limit):
            k = min(CHUNK_SECTORS, n - done)
            buf = src.read(k * size)
            if len(buf) < k * size:
                raise Refused("%s ends early" % track.path)
            if done == 0 and size == 2352:
                if buf[:12] != SYNC or buf[15] not in (1, 2):
                    raise Refused("track %d of %s is not %s data" % (track.number, os.path.basename(track.path),
                                                                      track.mode))
            data = b"".join(buf[i * size + off:i * size + off + 2048] for i in range(k))
            if limit is None and done <= 16 < done + k:
                limit = iso_volume_bytes(data[(16 - done) * 2048:(17 - done) * 2048])
            if limit is not None:
                data = data[:max(0, limit - written)]
            out.write(data)
            written += len(data)
            done += k
    return written


def wav_header(nbytes):
    """a RIFF/WAVE header for nbytes of 16-bit stereo 44.1 kHz PCM"""
    return (b"RIFF" + (36 + nbytes).to_bytes(4, "little") + b"WAVEfmt " + (16).to_bytes(4, "little")
            + (1).to_bytes(2, "little") + (2).to_bytes(2, "little") + (44100).to_bytes(4, "little")
            + (176400).to_bytes(4, "little") + (4).to_bytes(2, "little") + (16).to_bytes(2, "little")
            + b"data" + nbytes.to_bytes(4, "little"))


def write_wav(track, out):
    """an audio track as a WAV stream to the binary file object out: CD-DA in a .bin is 16-bit little-endian
    stereo (a MOTOROLA file is big-endian: swapped); the bytes of audio written"""
    n = track.end - track.start
    out.write(wav_header(n))
    left = n
    with open(track.path, "rb") as src:
        src.seek(track.start)
        while left:
            buf = src.read(min(left, CHUNK_SECTORS * 2352))
            if not buf:
                raise Refused("%s ends early" % track.path)
            if track.ftype == "MOTOROLA":
                b = bytearray(buf)
                b[0::2], b[1::2] = buf[1::2], buf[0::2]
                buf = bytes(b)
            out.write(buf)
            left -= len(buf)
    return n


def track_seconds(track):
    return (track.end - track.start) / 176400


def encode_tracks(tracks, layout, dest, first, last, jobs=OGGENC_JOBS):
    """the audio tracks numbered first..last encoded to Ogg Vorbis (oggenc, its default quality, as
    game-data-packager's own rip does) at dest/(layout % number), the WAV streamed on oggenc's stdin (no
    WAV on disk); jobs encoders at a time. [paths]; Failed when oggenc is missing or fails."""
    if not os.path.exists(OGGENC):
        raise Failed("encoding the CD's music needs oggenc: sudo apt install vorbis-tools")
    want = [t for t in tracks if t.audio and first <= t.number <= last]
    running, made = [], []

    def feed(p, t, errors):
        try:
            write_wav(t, p.stdin)
        except BrokenPipeError:
            pass
        except (OSError, Refused) as e:
            errors.append(str(e))
        finally:
            try:
                p.stdin.close()
            except OSError:
                pass

    def finish(p, th, t, path, errors):
        th.join()
        rc = p.wait()
        if rc != 0 or errors:
            raise Failed("encoding track %d failed: %s" % (t.number, errors[0] if errors else "oggenc exit %d" % rc))
        made.append(path)

    try:
        for t in want:
            path = os.path.join(dest, layout % t.number)
            os.makedirs(os.path.dirname(path), 0o700, exist_ok=True)
            secs = int(track_seconds(t))
            say("gdp", "step encoding the CD's music: track %d of %d (%d:%02d)" % (t.number, last, secs // 60, secs % 60))
            p = subprocess.Popen([NICE, "-n", "10", OGGENC, "-Q", "-o", path, "-"], stdin=subprocess.PIPE,
                                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
            errors = []
            th = threading.Thread(target=feed, args=(p, t, errors), daemon=True)
            th.start()
            running.append((p, th, t, path, errors))
            while len(running) >= jobs:
                finish(*running.pop(0))
        while running:
            finish(*running.pop(0))
    finally:
        for p, th, _, _, _ in running:
            p.kill()
            p.wait()
            th.join()
    return made


def cue_for(path):
    """the cue sheet of a SOURCE: path itself (.cue), or the .cue next to a .bin with its name; None for
    anything else"""
    low = path.lower()
    if low.endswith(".cue"):
        return path
    if low.endswith(".bin"):
        stem = os.path.splitext(os.path.basename(path))[0].lower()
        d = os.path.dirname(path)
        try:
            for n in sorted(os.listdir(d)):
                if n.lower() == stem + ".cue":
                    return os.path.join(d, n)
        except OSError:
            pass
        raise Refused("%s needs its .cue sheet next to it (it says where the tracks are)" % path)
    return None


# ---- the CD in the drive: its audio tracks ------------------------------------------------------

CDROMREADTOCHDR, CDROMREADTOCENTRY, CDROM_LBA, CDROM_DATA_TRACK = 0x5305, 0x5306, 0x01, 0x04


def disc_audio_tracks(dev):
    """the audio track numbers of the CD in drive dev (its table of contents, linux/cdrom.h ioctls), [] for
    a disc without audio, None when it can't be read (no disc, not a CD drive)"""
    try:
        fd = os.open(dev, os.O_RDONLY | os.O_NONBLOCK | os.O_CLOEXEC)
    except OSError:
        return None
    try:
        hdr = fcntl.ioctl(fd, CDROMREADTOCHDR, bytes(2))
        first, last = hdr[0], hdr[1]
        audio = []
        for n in range(first, last + 1):
            # struct cdrom_tocentry: u8 track, u8 adr:4 ctrl:4, u8 format, union addr (int), u8 datamode
            e = fcntl.ioctl(fd, CDROMREADTOCENTRY, bytes([n, 0, CDROM_LBA]) + bytes(9))
            # the bit fields adr:4 ctrl:4 share a byte: ctrl is its high half on little-endian, low on big
            ctrl = e[1] >> 4 if sys.byteorder == "little" else e[1] & 0x0F
            if not ctrl & CDROM_DATA_TRACK:
                audio.append(n)
        return audio
    except OSError:
        return None
    finally:
        os.close(fd)


# ---- game-data-packager -----------------------------------------------------------------------

def cache_dir():
    base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache")
    return os.path.join(base, "walnut-data")


LOCK_NAME = "import.lock"


def take_import_lock(base, game_id):
    """the descriptor of base/import.lock, flocked and saying "PID GAME"; Refused (busy) when another
    import holds it"""
    fd = os.open(os.path.join(base, LOCK_NAME), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
    try:
        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except BlockingIOError:
        held = import_state(base)
        os.close(fd)
        raise Refused("busy: another import is running%s" % (" (%s)" % held[1] if held else ""))
    os.ftruncate(fd, 0)
    os.write(fd, ("%d %s\n" % (os.getpid(), game_id)).encode())
    return fd


def import_state(base=None):
    """(pid, game id) of the import running now (its lock held), or None"""
    try:
        fd = os.open(os.path.join(base or cache_dir(), LOCK_NAME), os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
    except OSError:
        return None
    try:
        try:
            fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
        except BlockingIOError:
            pid, game = os.read(fd, 200).decode("utf-8", "replace").split()[:2]
            return int(pid), game
        fcntl.flock(fd, fcntl.LOCK_UN)
        return None
    except ValueError:
        return None
    finally:
        os.close(fd)


def fresh_dirs(base):
    out, tmp = os.path.join(base, "out"), os.path.join(base, "tmp")
    for d in (out, tmp):
        shutil.rmtree(d, ignore_errors=True)
        os.makedirs(d, 0o700)
    return out, tmp


def gdp_features():
    """(--progress-fd, --no-launchers) as this gdp's --help lists them"""
    rc, out = run([GDP, "--help"])
    if rc == 127:
        raise Failed("game-data-packager is not installed (sudo apt install game-data-packager)")
    return "--progress-fd" in out, "--no-launchers" in out


def gdp_command(ds, pkgs, sources, out, download, progress_fd=None, no_launchers=True):
    cmd = [GDP, "--no-search", "--no-compress"]
    if no_launchers:
        cmd.append("--no-launchers")
    if progress_fd is not None:
        cmd += ["--progress-fd", str(progress_fd)]
    cmd += ["-n", "-d", out, download]
    for p in pkgs:
        cmd += ["-p", p]
    return cmd + [ds["gdp"]] + list(sources)


def user_manager():
    """whether systemd-run --user can start a unit here"""
    rc, _ = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "/bin/true"])
    return rc == 0


def private_network_works():
    """whether the user manager can give a unit a network namespace of its own (user namespaces)"""
    rc, out = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "-p", "PrivateNetwork=yes",
                   "/bin/cat", "/proc/net/dev"])
    ifaces = [l.split(":")[0].strip() for l in out.splitlines() if ":" in l]
    return rc == 0 and ifaces == ["lo"]


def unit_command(cmd, tmp, fifo, private_net):
    """cmd in a transient user unit; descriptor 3 opened on fifo inside it when fifo is given"""
    unit = [SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect",
            "-p", "MemoryMax=" + MEMORY_MAX, "-p", "RuntimeMaxSec=" + RUNTIME_MAX, "-p", "NoNewPrivileges=yes",
            "--setenv=TMPDIR=" + tmp, "--setenv=PATH=" + UNIT_PATH, "--setenv=LC_ALL=C.UTF-8"]
    if private_net:
        unit += ["-p", "PrivateNetwork=yes"]
    if fifo:
        return unit + ["/bin/sh", "-c", 'exec 3>"$0" && exec "$@"', fifo] + cmd
    return unit + cmd


def relay_progress(fd, errors, generated):
    """gdp's --progress-fd lines as 'gdp ...'; its error lines kept in errors, the debs it says it made in
    generated"""
    with os.fdopen(fd, "rb") as f:
        for raw in f:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("gdp", line)
                if line.startswith("error "):
                    errors.append(line[6:])
                elif line.startswith("generated "):
                    generated.append(line[10:])


def run_gdp(cmd, tmp, base, sandbox, private_net, progress):
    """run gdp (cmd has '--progress-fd FD' to be filled in); its return code, its error lines (and a
    no-space one for an ENOSPC in its output) and the debs its progress lines say it made"""
    errors, generated = [], []
    env = dict(os.environ, TMPDIR=tmp)
    fifo = reader = keep = None
    if progress and sandbox:
        fifo = os.path.join(base, "progress.fifo")
        try:
            os.unlink(fifo)
        except FileNotFoundError:
            pass
        os.mkfifo(fifo, 0o600)
        rfd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK)
        keep = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)    # no EOF before gdp has even opened it
        os.set_blocking(rfd, True)
        full = unit_command([("3" if a == "FD" else a) for a in cmd], tmp, fifo, private_net)
        pass_fds = ()
    elif progress:
        rfd, wfd = os.pipe()
        full = [(str(wfd) if a == "FD" else a) for a in cmd]
        pass_fds = (wfd,)
    else:
        rfd = None
        full = unit_command(cmd, tmp, None, private_net) if sandbox else cmd
        pass_fds = ()
    if rfd is not None:
        reader = threading.Thread(target=relay_progress, args=(rfd, errors, generated), daemon=True)
        reader.start()
    try:
        p = subprocess.Popen(full, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             env=env, pass_fds=pass_fds)
    finally:
        for fd in pass_fds:
            os.close(fd)
    with p.stdout:
        for raw in p.stdout:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("log", line)
                if "No space left on device" in line:
                    errors.append("no-space " + line)
    rc = p.wait()
    if keep is not None:
        os.close(keep)
    if reader:
        reader.join(10)
    if fifo:
        try:
            os.unlink(fifo)
        except OSError:
            pass
    return rc, errors, generated


def built_debs(out, pkgs):
    """[(package, deb path)] gdp made in out, in the order of pkgs"""
    found = []
    files = sorted(os.listdir(out)) if os.path.isdir(out) else []
    for p in pkgs:
        for n in files:
            if n.startswith(p + "_") and n.endswith(".deb"):
                found.append((p, os.path.join(out, n)))
                break
    return found


def helper_lines(stream):
    rc_error = None
    for raw in stream:
        line = raw.decode("utf-8", "replace").rstrip()
        if line:
            kind = line.split(" ", 1)[0]
            print(line if kind in ("progress", "log", "error", "status") else "log " + line, flush=True)
            if kind == "error":
                rc_error = line
    return rc_error


def import_deb(game_id, pkg, deb):
    """dpkg-deb --fsys-tarfile deb | sudo -n walnut-games-helper import-data game_id pkg: its exit code"""
    say("log", "importing %s" % pkg)
    # the deb unlinked once open: its space comes back as the stream ends (the helper's staging needs it)
    fd = os.open(deb, os.O_RDONLY | os.O_CLOEXEC)
    os.unlink(deb)
    try:
        tar = subprocess.Popen([DPKG_DEB, "--fsys-tarfile", "/dev/fd/%d" % fd], stdin=subprocess.DEVNULL,
                               stdout=subprocess.PIPE, pass_fds=(fd,))
    finally:
        os.close(fd)
    try:
        h = subprocess.Popen(SUDO + [HELPER, "import-data", game_id, pkg], stdin=tar.stdout, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError:
        tar.kill()
        tar.wait()
        raise
    tar.stdout.close()      # the helper has it; dpkg-deb gets SIGPIPE if the helper stops reading
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    trc = tar.wait()
    if rc == 0 and trc != 0:
        say("error", "dpkg-deb --fsys-tarfile %s failed (%d)" % (os.path.basename(deb), trc))
        return 1
    return rc


def load_manifest(path=None):
    try:
        with open(path or MANIFEST, encoding="utf-8") as f:
            return json.load(f)["packages"]
    except (OSError, ValueError, KeyError, TypeError):
        return None


def data_bytes(pkgs, path=None):
    """the bytes of the packages' files by the data manifest (each file at its largest size; a music
    package's tracks MUSIC_TRACK_BYTES each, their real size being unknown), or None when the manifest
    can't be read"""
    man = load_manifest(path)
    try:
        return sum(sum(max(a["size"] for a in alts) for alts in man[p]["files"].values())
                   + (music_tracks(man[p]) or 0) * MUSIC_TRACK_BYTES for p in pkgs if p in man)
    except (ValueError, KeyError, TypeError):
        return None


def music_tracks(entry):
    """the number of CD tracks of a music package's manifest entry, or None for another package"""
    t = entry.get("tracks") if isinstance(entry, dict) else None
    if isinstance(t, dict) and isinstance(t.get("first"), int) and isinstance(t.get("last"), int):
        return t["last"] - t["first"] + 1
    return None


def music_entries(pkgs, path=None):
    """{package: its manifest tracks entry} of the music packages (CD audio, ripped or encoded) among pkgs"""
    man = load_manifest(path) or {}
    out = {}
    for p in pkgs:
        e = man.get(p)
        if music_tracks(e) and isinstance(e["tracks"].get("source"), str) and e["tracks"]["source"].count("%02d") == 1:
            out[p] = e["tracks"]
    return out


def existing(path):
    while not os.path.exists(path) and os.path.dirname(path) != path:
        path = os.path.dirname(path)
    return path


def space_needs(total, archive, cache, var, usr, dev=None, extra=0):
    """{device: (bytes, [paths])}: the most an import has on each file system at one time. Its phases:
    gdp in the cache (its copy of the files, dpkg-deb's temporary data member, the deb, an installer's or
    archive's unpacked files, and extra: a BIN/CUE's ISO and music, the downloads),
    the deb streamed to the helper's staging (the deb is unlinked as dpkg-deb opens it: gone once the
    stream ends), the helper's own deb made from its staging (with dpkg-deb's temporary copy of the
    data member), then dpkg installing it into /usr"""
    dev = dev or (lambda p: os.stat(existing(p)).st_dev)
    phases = [[(cache, (4 if archive else 3) * total + extra)], [(cache, total), (var, total)], [(var, 3 * total)],
              [(var, total), (usr, total)]]
    need = {}
    for phase in phases:
        now = {}
        for path, n in phase:
            b, paths = now.get(dev(path), (0, []))
            now[dev(path)] = (b + n, paths + [path])
        for d, (b, paths) in now.items():
            old = need.get(d, (0, []))
            need[d] = (max(old[0], b), list(dict.fromkeys(old[1] + paths)))
    return need


def human(n):
    for unit, size in (("GB", 1 << 30), ("MB", 1 << 20)):
        if n >= size:
            return "%.1f %s" % (n / size, unit)
    return "%d KB" % max(1, n >> 10)


def check_space(total, archive, cache, var=IMPORT_DIR, usr=USR, free=None, extra=0):
    """Refused (with what is needed and what is free) when a file system can't hold the import; extra: the
    bytes the sources put in the cache while gdp runs (a BIN/CUE's ISO and Ogg files)"""
    free = free or (lambda p: (lambda st: st.f_bavail * st.f_frsize)(os.statvfs(existing(p))))
    for d, (need, paths) in space_needs(total, archive, cache, var, usr, extra=extra).items():
        have = free(paths[0])
        if have < need + SPACE_MARGIN:
            raise Refused("not enough space: importing this data needs about %s free on the disk of %s, and %s is free"
                          % (human(need + SPACE_MARGIN), ", ".join(paths), human(have)))


GDP_HINTS = {
    "not-found": "no files of this game were found there",
    "unknown-version": "this copy is a version game-data-packager doesn't know",
    "unsuitable": "this copy is a release game-data-packager refuses",
    "needs-download": "some files have to be downloaded: connect to the network, or add those files",
    "no-space": "not enough free space",
}


def gdp_hint(error):
    """why gdp made no package, from its last error line 'KIND TEXT' (unsuitable: its own reason, the text
    'FILE RELEASE: REASON')"""
    kind, _, text = (error or "").partition(" ")
    if kind == "unsuitable":
        head, sep, reason = text.partition(": ")
        release = head.partition(" ")[2]
        if sep and release:
            return "this copy is %s, which can't be used: %s" % (release, reason)
    return GDP_HINTS.get(kind)


def reload_menu():
    """ps3-game-menu --reload, like the app does after an install or remove; needs the desktop session,
    failures ignored"""
    rt = os.environ.get("XDG_RUNTIME_DIR")
    if not (rt and os.environ.get("DISPLAY") and os.path.exists(MENU_TOOL)):
        return
    try:
        subprocess.run([MENU_TOOL, "--reload", os.path.join(rt, "ps3-game-menu")], stdin=subprocess.DEVNULL,
                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60)
    except (OSError, subprocess.SubprocessError):
        pass


def report_missing(ds, pkgs, asked, debs, download, errors):
    """say what gdp did not make (see Partial results in the header); (required packages, those missing)"""
    made = {p for p, _ in debs}
    required = list(pkgs) if asked else [p for p in pkgs if p in names(ds, "required")] or list(pkgs)
    why = errors[-1] if errors else ""
    for p in pkgs:
        if p in made:
            continue
        if p in names(ds, "downloads") and download == "--no-download":
            say("gdp", "error needs-download %s: its official patch files have to be downloaded" % p)
        else:
            say("warning", "missing %s: %s" % (p, why or "game-data-packager did not make it"))
    return required, [p for p in required if p not in made]


def cmd_import(game_id, sources, asked, no_download, games):
    ds = data_setup(game_id, games)
    pkgs = wanted_packages(ds, asked)
    if not pkgs:
        raise Refused("%s names no data packages" % game_id)
    if not sources:
        if not asked or no_download or not all(p in names(ds, "fetch") for p in pkgs):
            free = names(ds, "fetch")
            raise Refused("no SOURCE given: a disc, an .iso, an installer or a folder%s" % (
                " (only %s can be downloaded without one: import -p %s %s)" % (", ".join(free), free[0], game_id)
                if free else ""))
    download = download_choice(ds, pkgs, no_download, not sources)
    if download == "--no-download" and names(ds, "downloads") and not no_download:
        say("log", "downloads off: import with -p %s alone to let it fetch its patch files"
            % " -p ".join(names(ds, "downloads")))
    progress, no_launchers = gdp_features()
    base = cache_dir()
    os.makedirs(base, 0o700, exist_ok=True)
    lock = take_import_lock(base, game_id)
    out, tmp = fresh_dirs(base)
    music = music_entries(pkgs)
    src = Sources(base, music.values())
    src.only_music = bool(music) and all(p in music for p in pkgs)
    try:
        plans = [src.plan(s) for s in sources]
        total = data_bytes(pkgs)
        if total:
            # an installer or archive: unpacked first
            archive = any(kind == "path" and os.path.isfile(what) for kind, what, _ in plans)
            check_space(total, archive, base, extra=sum(extra for _, _, extra in plans)
                        + (DOWNLOAD_BYTES if download == "--download" else 0))
        paths = list(dict.fromkeys(p for plan in plans for p in src.open(plan)))
        sandbox = user_manager()
        private_net = sandbox and download == "--no-download" and private_network_works()
        if not sandbox:
            say("log", "no systemd user manager: game-data-packager runs without its sandbox")
        cmd = gdp_command(ds, pkgs, paths, out, download, "FD" if progress else None, no_launchers)
        say("log", "running game-data-packager %s for %s" % (ds["gdp"], " ".join(pkgs)))
        rc, errors, generated = run_gdp(cmd, tmp, base, sandbox, private_net, progress)
        if rc != 0:
            # a failed gdp may leave a deb it was still writing (ENOSPC): only those it said it made count
            for n in (os.listdir(out) if os.path.isdir(out) else []):
                if n.endswith(".deb") and os.path.join(out, n) not in generated:
                    say("log", "game-data-packager failed (exit %d): %s dropped, it may be cut short" % (rc, n))
                    os.unlink(os.path.join(out, n))
        debs = built_debs(out, pkgs)
        extra = sorted(n for n in (os.listdir(out) if os.path.isdir(out) else [])
                       if n.endswith(".deb") and os.path.join(out, n) not in [d for _, d in debs])
        if extra:
            raise Failed("game-data-packager also made %s, which was not asked for: nothing was imported"
                         % ", ".join(extra))
        required, missing_required = report_missing(ds, pkgs, asked, debs, download, errors)
        if not debs or len(missing_required) == len(required):
            hint = gdp_hint(next((e for e in errors if e.startswith("no-space ")), errors[-1] if errors else None))
            raise Failed("game-data-packager made no package%s" % (": " + hint if hint else
                                                                    " (exit %d)" % rc if rc else ""))
        src.close()            # the disc or image is no longer needed
        worst = 0
        for pkg, deb in debs:
            rc = import_deb(game_id, pkg, deb)
            if rc != 0:
                worst = rc if worst in (0, 2) else worst
        if worst == 2:
            raise Refused("the helper refused the data")
        if worst:
            raise Failed("the helper failed")
        reload_menu()
        return 0
    finally:
        src.close()
        for d in (out, tmp):
            shutil.rmtree(d, ignore_errors=True)
        os.close(lock)


def cmd_helper(action, game_id, games):
    data_setup(game_id, games)
    try:
        h = subprocess.Popen(SUDO + [HELPER, action, game_id], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError as e:
        raise Failed(str(e))
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    return rc if rc in (0, 1, 2) else 1


def parse(argv):
    ap = argparse.ArgumentParser(prog="walnut-data", description="The player's own game data, made into packages.")
    sub = ap.add_subparsers(dest="cmd", required=True)
    sub.add_parser("list", help="the games that take data, and whether theirs is installed")
    p = sub.add_parser("import", help="make and install the game's data packages from SOURCE (a disc, an .iso, "
                                      "an installer, a folder)")
    p.add_argument("-p", "--package", action="append", default=[], help="only this data package (repeatable)")
    p.add_argument("--no-download", action="store_true", help="never download, even the official patch files")
    p.add_argument("game")
    p.add_argument("sources", nargs="*", metavar="SOURCE", help="optional when every -p package is one gdp "
                   "downloads entirely (the free shareware or demo)")
    for name, text in (("remove", "remove the game's data packages"),
                       ("install-repo", "install the game's data from your package sources")):
        sub.add_parser(name, help=text).add_argument("game")
    return ap.parse_args(argv)


def main(argv=None, catalog=None):
    try:
        a = parse(argv)
    except SystemExit as e:
        return 0 if e.code == 0 else 2
    try:
        games = load_games(catalog)
        if a.cmd == "list":
            return cmd_list(games)
        if a.cmd == "import":
            return cmd_import(a.game, a.sources, a.package, a.no_download, games)
        rc = cmd_helper({"remove": "remove-data", "install-repo": "install-repo-data"}[a.cmd], a.game, games)
        if rc == 0:
            reload_menu()
        return rc
    except Refused as e:
        say("error", str(e))
        return 2
    except Failed as e:
        say("error", str(e))
        return 1
    except (OSError, ValueError, KeyError) as e:
        say("error", "%s: %s" % (type(e).__name__, e))
        return 1
    except KeyboardInterrupt:
        say("error", "interrupted")
        return 1


if __name__ == "__main__":
    sys.exit(main())
