#!/usr/bin/python3 -I
"""walnut-data list | import [-p PKG]... [--no-download] GAME SOURCE... | remove GAME | install-repo GAME
(walnut-games package, /usr/bin): the player's own game data made into packages and installed, from a
terminal or the Walnut Games app. Runs as the player; only walnut-games-helper (through sudo -n) is root.

Design
- GAME is a catalog id (/usr/share/walnut-games/catalog.json) whose entry has a data_setup (GameDB
  build.toml [data_setup]): the game-data-packager game, the packages, which may download.
- list: every catalog game with a data_setup, its data installed or missing (any one of present_any;
  optional for a game that runs without), whether the configured apt sources have one of its data
  packages (the licensed suite: `walnut-data install-repo GAME`), its title.
- import: each SOURCE as the player has it. An .iso or .img file is opened read-only with
  `udisksctl loop-setup -r -f` and mounted with `udisksctl mount -b` (the player's own polkit rights; no
  copy), a block device (the disc drive, /dev/sr0) is used at its mount point or mounted the same way,
  and whatever this program mounted or set up is released afterwards. The desktop (gvfs) mounts a new
  loop device by itself within seconds: when udisksctl mount says AlreadyMounted, that mount is used
  (waited for, AUTOMOUNT_WAIT) and unmounted afterwards, which frees the loop device (loop-delete only
  for one still attached). loop-setup needs the desktop session's polkit rights: over ssh it is refused
  (NotAuthorizedCanObtain), and the message says how to run it in the desktop's slice. A directory, an installer or
  archive file, or a mounted disc's path is given to game-data-packager as it is.
  game-data-packager (gdp) runs as the player in a transient user unit: `systemd-run --user --pipe
  --wait` with MemoryMax, RuntimeMaxSec and NoNewPrivileges, and PrivateNetwork when it must not
  download and the user manager can make one (checked by a probe: only "lo" in its /proc/net/dev);
  without a user manager it runs directly. Its command: `game-data-packager --no-search --no-compress
  --no-launchers --progress-fd 3 -n -d ~/.cache/walnut-data/out --download|--no-download -p PKG...
  GDPGAME SOURCE...` with TMPDIR=~/.cache/walnut-data/tmp (the PS3's /tmp is a small tmpfs). The
  packages: -p's, else the game's required and optional ones, each always passed as -p (without -p
  gdp builds whatever else it can, downloads included); a deb gdp made that was not asked for fails
  the import, nothing imported. --download only when every one of
  them is among the game's downloads (the official patch files gdp knows), else --no-download.
  A gdp without --progress-fd or --no-launchers (Debian's) runs without them: no progress lines, and
  the helper drops the Doom family's menu files anyway. In the unit, descriptor 3 is a FIFO in the
  cache directory (systemd-run --pipe passes only 0-2).
  Each package gdp made is then `dpkg-deb --fsys-tarfile X.deb | sudo -n walnut-games-helper
  import-data GAME PKG` (the deb unlinked as soon as dpkg-deb has it open, so its space comes back
  when the stream ends): the helper checks every file against its manifest and builds and installs
  the package itself (required packages first, in the catalog's order). out/ and tmp/ are emptied
  before and removed after.
  Free space is checked before gdp runs, from the data manifest's sizes of the packages (T): the most
  each file system holds at one time over the phases (gdp: 2T in the cache, 3T for an installer or
  archive; streaming: T cache + T staging; the helper's deb: 3T staging, its files, dpkg-deb's
  temporary data member and the deb; installing: T staging +
  T /usr), summed where they share a disk; refused with what is needed and what is free.
  One import at a time per player: import holds an flock on ~/.cache/walnut-data/import.lock for its
  whole run, the file saying "PID GAME"; a second import is refused (busy, exit 2). The Walnut Games
  app reads that lock to show "importing" when it is opened again during an import (import_state).
- remove: `walnut-games-helper remove-data GAME` (the game's data packages only). install-repo:
  `walnut-games-helper install-repo-data GAME` (its data from the configured apt sources).
- Output, one item per line, flushed: "gdp step|file|generated|error ..." (gdp's progress lines),
  "log text" (gdp's own output and notes), the helper's lines as they come ("progress N text",
  "log text", "error text"), "error text" for this program's own failures.
- Exit codes: 0 done; 2 refused, nothing changed (bad arguments, an unknown game, a missing source,
  the helper refused); 1 failed (gdp or the helper failed, or gdp made no package).
- stdlib only, no Tk. The commands are module constants so that tests/test_walnut_games.py runs it
  with a stubbed gdp, dpkg-deb and helper.
"""
import argparse
import fcntl
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import threading
import time

CATALOG = "/usr/share/walnut-games/catalog.json"
HELPER = "/usr/lib/ps3/sbin/walnut-games-helper"
SUDO = ["/usr/bin/sudo", "-n"]
GDP = "/usr/games/game-data-packager"
DPKG_DEB, DPKG_QUERY, APT_CACHE = "/usr/bin/dpkg-deb", "/usr/bin/dpkg-query", "/usr/bin/apt-cache"
UDISKSCTL = "/usr/bin/udisksctl"
SYSTEMD_RUN = "/usr/bin/systemd-run"
MEMORY_MAX = "192M"       # gdp ~40 MB with one game loaded, plus an unpacker (innoextract, unshield)
RUNTIME_MAX = "3h"        # a CD reads at about 2.3 MB/s
UNIT_PATH = "/usr/local/bin:/usr/bin:/bin:/usr/games"
ID_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,63}\Z")
PKG_RE = re.compile(r"[a-z0-9][a-z0-9+.-]*[a-z0-9]\Z")
IMAGE_SUFFIXES = (".iso", ".img")
AUTOMOUNT_WAIT = 10       # seconds to wait for the desktop's mount of a loop device it got to first
MANIFEST = "/usr/share/walnut-games/data-manifest.json"
IMPORT_DIR = "/var/lib/walnut-games/import"    # the helper's staging (root's)
USR = "/usr"
SPACE_MARGIN = 64 << 20
SYS_BLOCK = "/sys/block"


class Refused(Exception):
    """exit code 2: nothing was changed"""


class Failed(Exception):
    """exit code 1"""


def say(kind, text=""):
    try:
        sys.stdout.write((kind + " " + text).rstrip() + "\n")
        sys.stdout.flush()
    except (OSError, ValueError):
        pass


def run(cmd, **kw):
    """(returncode, stdout and stderr) of cmd; 127 when it is not there"""
    try:
        p = subprocess.run(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
                           errors="replace", **kw)
    except OSError as e:
        return 127, str(e)
    return p.returncode, p.stdout


# ---- the catalog ------------------------------------------------------------------------------

def load_games(path=None):
    with open(path or CATALOG, encoding="utf-8") as f:
        return {g["id"]: g for g in json.load(f)["games"] if isinstance(g.get("id"), str)}


def data_setup(game_id, games):
    if not ID_RE.match(game_id or ""):
        raise Refused("not a game id: %r" % game_id)
    if game_id not in games:
        raise Refused("unknown game: %s (walnut-data list shows the games)" % game_id)
    ds = games[game_id].get("data_setup")
    if not isinstance(ds, dict) or not isinstance(ds.get("gdp"), str):
        raise Refused("%s has no data to set up" % game_id)
    return ds


def names(ds, key):
    return [p for p in ds.get(key) or [] if isinstance(p, str) and PKG_RE.match(p)]


def wanted_packages(ds, asked):
    """the packages to make: -p's (each one of the game's), else required then optional"""
    known = names(ds, "required") + names(ds, "optional")
    if asked:
        bad = [p for p in asked if p not in known]
        if bad:
            raise Refused("%s: not a data package of this game (%s)" % (", ".join(bad), ", ".join(known)))
        return list(dict.fromkeys(asked))
    return known


def download_choice(ds, pkgs, no_download=False):
    """--download only when every package is one the game may download for, else --no-download"""
    dl = set(names(ds, "downloads"))
    return "--download" if pkgs and not no_download and all(p in dl for p in pkgs) else "--no-download"


def installed(pkgs):
    if not pkgs:
        return set()
    rc, out = run([DPKG_QUERY, "-W", "-f", "${Package}\t${db:Status-Abbrev}\n"] + list(pkgs))
    return {n for line in out.splitlines() for n, _, st in [line.partition("\t")] if n in pkgs and st[1:2] == "i"}


def candidates(pkgs):
    if not pkgs:
        return set()
    rc, out = run([APT_CACHE, "policy"] + list(pkgs))
    have, cur = set(), None
    for line in out.splitlines():
        m = re.match(r"(\S+):\s*$", line)
        if m:
            cur = m.group(1)
        elif cur and (m := re.match(r"\s+Candidate:\s*(\S+)", line)) and m.group(1) != "(none)":
            have.add(cur)
    return have


def cmd_list(games):
    rows = []
    all_names = sorted({p for g in games.values() if isinstance(g.get("data_setup"), dict)
                        for k in ("present_any", "required", "optional") for p in names(g["data_setup"], k)})
    inst, repo = installed(all_names), candidates(all_names)
    for gid in sorted(games):
        ds = games[gid].get("data_setup")
        if not isinstance(ds, dict):
            continue
        any_of = names(ds, "present_any") or names(ds, "required")
        opt = names(ds, "optional")
        if names(ds, "required"):
            state = "present" if inst & set(any_of) else "missing"
        else:
            state = "optional-present" if inst & set(opt) else "optional"
        avail = "repo" if repo & set(any_of) and state == "missing" else "-"
        rows.append((gid, state, avail, games[gid].get("title") or gid))
    w = max([len(r[0]) for r in rows] + [4])
    for r in rows:
        print("%-*s  %-16s  %-4s  %s" % (w, r[0], r[1], r[2], r[3]))
    return 0


# ---- sources ----------------------------------------------------------------------------------

def source_kind(path):
    """'image' (an .iso/.img file: loop-mounted), 'device' (a block device: its mount point), 'path'
    (a directory or any other file, as it is); Refused when it is not there"""
    try:
        st = os.stat(path)
    except OSError:
        raise Refused("no such file or folder: %s" % path)
    if stat.S_ISBLK(st.st_mode):
        return "device"
    if stat.S_ISREG(st.st_mode) and path.lower().endswith(IMAGE_SUFFIXES):
        return "image"
    return "path"


def unescape_mount(s):
    return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), s)


def mount_point(dev, mounts="/proc/self/mounts"):
    """where block device dev is mounted, or None"""
    try:
        real = os.path.realpath(dev)
        with open(mounts) as f:
            for line in f:
                parts = line.split()
                if len(parts) >= 2 and os.path.realpath(unescape_mount(parts[0])) == real:
                    return unescape_mount(parts[1])
    except OSError:
        pass
    return None


def already_mounted_at(out):
    """the mount point in udisksctl's AlreadyMounted error ("... is already mounted at `/run/media/u/X'.")"""
    m = re.search(r"already mounted at [`'\"]?(/.+?)['\"]?\.?\s*$", out.strip(), re.M)
    return m.group(1) if m else None


def loop_attached(dev, sys_block=None):
    """whether loop device dev still has a backing file"""
    return os.path.exists(os.path.join(sys_block or SYS_BLOCK, os.path.basename(dev), "loop", "backing_file"))


class Sources:
    """the SOURCE arguments as paths gdp reads; loop devices and mounts made here released by close()"""

    def __init__(self):
        self.mounted, self.loops = [], []

    def mount(self, dev, ours=False):
        """dev's mount point, mounted with udisksctl when it is not; ours: dev is a loop device set up
        here, so whoever mounted it (the desktop's automounter does, within seconds) it is unmounted
        by close(), which frees the loop device too"""
        mp = mount_point(dev)
        if mp:
            if ours:
                self.mounted.append(dev)
            return mp
        rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" not in out and os.path.exists(dev + "p1"):     # an image with a partition table
            dev = dev + "p1"
            mp = mount_point(dev)
            if mp:
                if ours:
                    self.mounted.append(dev)
                return mp
            rc, out = run([UDISKSCTL, "mount", "-b", dev])
        if rc != 0 and "AlreadyMounted" in out:
            # mounted meanwhile by the desktop (gvfs mounts a new loop device by itself): use that mount
            mp = self.wait_mount(dev) or already_mounted_at(out)
            if mp and os.path.isdir(mp):
                if ours:
                    self.mounted.append(dev)
                say("log", "%s is mounted at %s" % (dev, mp))
                return mp
        if rc != 0:
            raise Refused("cannot mount %s: %s" % (dev, (out.strip().splitlines() or ["?"])[-1]))
        self.mounted.append(dev)
        mp = mount_point(dev)
        if not mp:
            m = re.search(r" at (.+?)\.?\s*$", out.strip())
            mp = m.group(1) if m else None
        if not mp or not os.path.isdir(mp):
            raise Refused("%s was mounted but its folder is not known: %s" % (dev, out.strip()))
        say("log", "mounted %s at %s" % (dev, mp))
        return mp

    @staticmethod
    def wait_mount(dev):
        end = time.monotonic() + AUTOMOUNT_WAIT
        while True:
            mp = mount_point(dev)
            if mp or time.monotonic() >= end:
                return mp
            time.sleep(0.2)

    def add(self, path):
        path = os.path.abspath(path)
        kind = source_kind(path)
        if kind == "path":
            return path
        if kind == "device":
            return self.mount(path)
        rc, out = run([UDISKSCTL, "loop-setup", "-r", "-f", path])
        m = re.search(r" as (/dev/\S+?)\.?\s*$", out.strip())
        if rc != 0 and "NotAuthorized" in out:
            raise Refused("opening the disc image %s needs the desktop session's rights (polkit): run walnut-data "
                          "in a terminal on the Walnut desktop or from Walnut Games; over ssh: systemd-run --user "
                          "--scope --slice=ps3-desktop.slice walnut-data import ...; or open the image in the file "
                          "manager and give its folder" % path)
        if rc != 0 or not m:
            raise Refused("cannot open %s: %s" % (path, (out.strip().splitlines() or ["?"])[-1]))
        self.loops.append(m.group(1))
        say("log", "opened %s as %s" % (path, m.group(1)))
        return self.mount(m.group(1), ours=True)

    def close(self):
        for dev in reversed(self.mounted):
            run([UDISKSCTL, "unmount", "-b", dev])
        for dev in reversed(self.loops):
            # unmounting frees a loop device udisks set up (autoclear); loop-delete only for one still there
            if loop_attached(dev):
                run([UDISKSCTL, "loop-delete", "-b", dev])
        self.mounted, self.loops = [], []


# ---- game-data-packager -----------------------------------------------------------------------

def cache_dir():
    base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache")
    return os.path.join(base, "walnut-data")


LOCK_NAME = "import.lock"


def take_import_lock(base, game_id):
    """the descriptor of base/import.lock, flocked and saying "PID GAME"; Refused (busy) when another
    import holds it"""
    fd = os.open(os.path.join(base, LOCK_NAME), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
    try:
        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except BlockingIOError:
        held = import_state(base)
        os.close(fd)
        raise Refused("busy: another import is running%s" % (" (%s)" % held[1] if held else ""))
    os.ftruncate(fd, 0)
    os.write(fd, ("%d %s\n" % (os.getpid(), game_id)).encode())
    return fd


def import_state(base=None):
    """(pid, game id) of the import running now (its lock held), or None"""
    try:
        fd = os.open(os.path.join(base or cache_dir(), LOCK_NAME), os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
    except OSError:
        return None
    try:
        try:
            fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
        except BlockingIOError:
            pid, game = os.read(fd, 200).decode("utf-8", "replace").split()[:2]
            return int(pid), game
        fcntl.flock(fd, fcntl.LOCK_UN)
        return None
    except ValueError:
        return None
    finally:
        os.close(fd)


def fresh_dirs(base):
    out, tmp = os.path.join(base, "out"), os.path.join(base, "tmp")
    for d in (out, tmp):
        shutil.rmtree(d, ignore_errors=True)
        os.makedirs(d, 0o700)
    return out, tmp


def gdp_features():
    """(--progress-fd, --no-launchers) as this gdp's --help lists them"""
    rc, out = run([GDP, "--help"])
    if rc == 127:
        raise Failed("game-data-packager is not installed (sudo apt install game-data-packager)")
    return "--progress-fd" in out, "--no-launchers" in out


def gdp_command(ds, pkgs, sources, out, download, progress_fd=None, no_launchers=True):
    cmd = [GDP, "--no-search", "--no-compress"]
    if no_launchers:
        cmd.append("--no-launchers")
    if progress_fd is not None:
        cmd += ["--progress-fd", str(progress_fd)]
    cmd += ["-n", "-d", out, download]
    for p in pkgs:
        cmd += ["-p", p]
    return cmd + [ds["gdp"]] + list(sources)


def user_manager():
    """whether systemd-run --user can start a unit here"""
    rc, _ = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "/bin/true"])
    return rc == 0


def private_network_works():
    """whether the user manager can give a unit a network namespace of its own (user namespaces)"""
    rc, out = run([SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect", "-p", "PrivateNetwork=yes",
                   "/bin/cat", "/proc/net/dev"])
    ifaces = [l.split(":")[0].strip() for l in out.splitlines() if ":" in l]
    return rc == 0 and ifaces == ["lo"]


def unit_command(cmd, tmp, fifo, private_net):
    """cmd in a transient user unit; descriptor 3 opened on fifo inside it when fifo is given"""
    unit = [SYSTEMD_RUN, "--user", "--pipe", "--wait", "--quiet", "--collect",
            "-p", "MemoryMax=" + MEMORY_MAX, "-p", "RuntimeMaxSec=" + RUNTIME_MAX, "-p", "NoNewPrivileges=yes",
            "--setenv=TMPDIR=" + tmp, "--setenv=PATH=" + UNIT_PATH, "--setenv=LC_ALL=C.UTF-8"]
    if private_net:
        unit += ["-p", "PrivateNetwork=yes"]
    if fifo:
        return unit + ["/bin/sh", "-c", 'exec 3>"$0" && exec "$@"', fifo] + cmd
    return unit + cmd


def relay_progress(fd, errors):
    """gdp's --progress-fd lines as 'gdp ...'; its error lines kept in errors"""
    with os.fdopen(fd, "rb") as f:
        for raw in f:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("gdp", line)
                if line.startswith("error "):
                    errors.append(line[6:])


def run_gdp(cmd, tmp, base, sandbox, private_net, progress):
    """run gdp (cmd has '--progress-fd FD' to be filled in); its return code and its error lines"""
    errors = []
    env = dict(os.environ, TMPDIR=tmp)
    fifo = reader = keep = None
    if progress and sandbox:
        fifo = os.path.join(base, "progress.fifo")
        try:
            os.unlink(fifo)
        except FileNotFoundError:
            pass
        os.mkfifo(fifo, 0o600)
        rfd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK)
        keep = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)    # no EOF before gdp has even opened it
        os.set_blocking(rfd, True)
        full = unit_command([("3" if a == "FD" else a) for a in cmd], tmp, fifo, private_net)
        pass_fds = ()
    elif progress:
        rfd, wfd = os.pipe()
        full = [(str(wfd) if a == "FD" else a) for a in cmd]
        pass_fds = (wfd,)
    else:
        rfd = None
        full = unit_command(cmd, tmp, None, private_net) if sandbox else cmd
        pass_fds = ()
    if rfd is not None:
        reader = threading.Thread(target=relay_progress, args=(rfd, errors), daemon=True)
        reader.start()
    try:
        p = subprocess.Popen(full, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             env=env, pass_fds=pass_fds)
    finally:
        for fd in pass_fds:
            os.close(fd)
    with p.stdout:
        for raw in p.stdout:
            line = raw.decode("utf-8", "replace").rstrip()
            if line:
                say("log", line)
    rc = p.wait()
    if keep is not None:
        os.close(keep)
    if reader:
        reader.join(10)
    if fifo:
        try:
            os.unlink(fifo)
        except OSError:
            pass
    return rc, errors


def built_debs(out, pkgs):
    """[(package, deb path)] gdp made in out, in the order of pkgs"""
    found = []
    files = sorted(os.listdir(out)) if os.path.isdir(out) else []
    for p in pkgs:
        for n in files:
            if n.startswith(p + "_") and n.endswith(".deb"):
                found.append((p, os.path.join(out, n)))
                break
    return found


def helper_lines(stream):
    rc_error = None
    for raw in stream:
        line = raw.decode("utf-8", "replace").rstrip()
        if line:
            kind = line.split(" ", 1)[0]
            print(line if kind in ("progress", "log", "error", "status") else "log " + line, flush=True)
            if kind == "error":
                rc_error = line
    return rc_error


def import_deb(game_id, pkg, deb):
    """dpkg-deb --fsys-tarfile deb | sudo -n walnut-games-helper import-data game_id pkg: its exit code"""
    say("log", "importing %s" % pkg)
    # the deb unlinked once open: its space comes back as the stream ends (the helper's staging needs it)
    fd = os.open(deb, os.O_RDONLY | os.O_CLOEXEC)
    os.unlink(deb)
    try:
        tar = subprocess.Popen([DPKG_DEB, "--fsys-tarfile", "/dev/fd/%d" % fd], stdin=subprocess.DEVNULL,
                               stdout=subprocess.PIPE, pass_fds=(fd,))
    finally:
        os.close(fd)
    try:
        h = subprocess.Popen(SUDO + [HELPER, "import-data", game_id, pkg], stdin=tar.stdout, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError:
        tar.kill()
        tar.wait()
        raise
    tar.stdout.close()      # the helper has it; dpkg-deb gets SIGPIPE if the helper stops reading
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    trc = tar.wait()
    if rc == 0 and trc != 0:
        say("error", "dpkg-deb --fsys-tarfile %s failed (%d)" % (os.path.basename(deb), trc))
        return 1
    return rc


def data_bytes(pkgs, path=None):
    """the bytes of the packages' files by the data manifest (each file at its largest size), or None
    when the manifest can't be read"""
    try:
        with open(path or MANIFEST, encoding="utf-8") as f:
            man = json.load(f)["packages"]
        return sum(max(a["size"] for a in alts) for p in pkgs if p in man for alts in man[p]["files"].values())
    except (OSError, ValueError, KeyError, TypeError):
        return None


def existing(path):
    while not os.path.exists(path) and os.path.dirname(path) != path:
        path = os.path.dirname(path)
    return path


def space_needs(total, archive, cache, var, usr, dev=None):
    """{device: (bytes, [paths])}: the most an import has on each file system at one time. Its phases:
    gdp in the cache (its unpacked copy, the deb, and an installer's or archive's unpacked files),
    the deb streamed to the helper's staging (the deb is unlinked as dpkg-deb opens it: gone once the
    stream ends), the helper's own deb made from its staging (with dpkg-deb's temporary copy of the
    data member), then dpkg installing it into /usr"""
    dev = dev or (lambda p: os.stat(existing(p)).st_dev)
    phases = [[(cache, (3 if archive else 2) * total)], [(cache, total), (var, total)], [(var, 3 * total)],
              [(var, total), (usr, total)]]
    need = {}
    for phase in phases:
        now = {}
        for path, n in phase:
            b, paths = now.get(dev(path), (0, []))
            now[dev(path)] = (b + n, paths + [path])
        for d, (b, paths) in now.items():
            old = need.get(d, (0, []))
            need[d] = (max(old[0], b), list(dict.fromkeys(old[1] + paths)))
    return need


def human(n):
    for unit, size in (("GB", 1 << 30), ("MB", 1 << 20)):
        if n >= size:
            return "%.1f %s" % (n / size, unit)
    return "%d KB" % max(1, n >> 10)


def check_space(total, archive, cache, var=IMPORT_DIR, usr=USR, free=None):
    """Refused (with what is needed and what is free) when a file system can't hold the import"""
    free = free or (lambda p: (lambda st: st.f_bavail * st.f_frsize)(os.statvfs(existing(p))))
    for d, (need, paths) in space_needs(total, archive, cache, var, usr).items():
        have = free(paths[0])
        if have < need + SPACE_MARGIN:
            raise Refused("not enough space: importing this data needs about %s free on the disk of %s, and %s is free"
                          % (human(need + SPACE_MARGIN), ", ".join(paths), human(have)))


GDP_HINTS = {
    "not-found": "no files of this game were found there",
    "unknown-version": "this copy is a version game-data-packager doesn't know",
    "unsuitable": "this copy is a release game-data-packager refuses",
    "needs-download": "some files have to be downloaded: connect to the network, or add those files",
    "no-space": "not enough free space",
}


def gdp_hint(error):
    """why gdp made no package, from its last error line 'KIND TEXT' (unsuitable: its own reason, the text
    'FILE RELEASE: REASON')"""
    kind, _, text = (error or "").partition(" ")
    if kind == "unsuitable":
        head, sep, reason = text.partition(": ")
        release = head.partition(" ")[2]
        if sep and release:
            return "this copy is %s, which can't be used: %s" % (release, reason)
    return GDP_HINTS.get(kind)


def cmd_import(game_id, sources, asked, no_download, games):
    ds = data_setup(game_id, games)
    pkgs = wanted_packages(ds, asked)
    if not pkgs:
        raise Refused("%s names no data packages" % game_id)
    if not sources:
        raise Refused("no SOURCE given: a disc, an .iso, an installer or a folder")
    download = download_choice(ds, pkgs, no_download)
    if download == "--no-download" and names(ds, "downloads") and not no_download:
        say("log", "downloads off: import with -p %s alone to let it fetch its patch files"
            % " -p ".join(names(ds, "downloads")))
    progress, no_launchers = gdp_features()
    base = cache_dir()
    os.makedirs(base, 0o700, exist_ok=True)
    lock = take_import_lock(base, game_id)
    out, tmp = fresh_dirs(base)
    src = Sources()
    try:
        paths = [src.add(s) for s in sources]
        total = data_bytes(pkgs)
        if total:
            archive = any(os.path.isfile(p) for p in paths)     # an installer or archive: unpacked first
            check_space(total, archive, base)
        sandbox = user_manager()
        private_net = sandbox and download == "--no-download" and private_network_works()
        if not sandbox:
            say("log", "no systemd user manager: game-data-packager runs without its sandbox")
        cmd = gdp_command(ds, pkgs, paths, out, download, "FD" if progress else None, no_launchers)
        say("log", "running game-data-packager %s for %s" % (ds["gdp"], " ".join(pkgs)))
        rc, errors = run_gdp(cmd, tmp, base, sandbox, private_net, progress)
        debs = built_debs(out, pkgs)
        extra = sorted(n for n in (os.listdir(out) if os.path.isdir(out) else [])
                       if n.endswith(".deb") and os.path.join(out, n) not in [d for _, d in debs])
        if extra:
            raise Failed("game-data-packager also made %s, which was not asked for: nothing was imported"
                         % ", ".join(extra))
        if rc != 0 or not debs:
            hint = gdp_hint(errors[-1] if errors else None)
            raise Failed("game-data-packager made no package%s" % (": " + hint if hint else
                                                                    " (exit %d)" % rc if rc else ""))
        src.close()            # the disc or image is no longer needed
        worst = 0
        for pkg, deb in debs:
            rc = import_deb(game_id, pkg, deb)
            if rc != 0:
                worst = rc if worst in (0, 2) else worst
        if worst == 2:
            raise Refused("the helper refused the data")
        if worst:
            raise Failed("the helper failed")
        return 0
    finally:
        src.close()
        for d in (out, tmp):
            shutil.rmtree(d, ignore_errors=True)
        os.close(lock)


def cmd_helper(action, game_id, games):
    data_setup(game_id, games)
    try:
        h = subprocess.Popen(SUDO + [HELPER, action, game_id], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
    except OSError as e:
        raise Failed(str(e))
    with h.stdout:
        helper_lines(h.stdout)
    rc = h.wait()
    return rc if rc in (0, 1, 2) else 1


def parse(argv):
    ap = argparse.ArgumentParser(prog="walnut-data", description="The player's own game data, made into packages.")
    sub = ap.add_subparsers(dest="cmd", required=True)
    sub.add_parser("list", help="the games that take data, and whether theirs is installed")
    p = sub.add_parser("import", help="make and install the game's data packages from SOURCE (a disc, an .iso, "
                                      "an installer, a folder)")
    p.add_argument("-p", "--package", action="append", default=[], help="only this data package (repeatable)")
    p.add_argument("--no-download", action="store_true", help="never download, even the official patch files")
    p.add_argument("game")
    p.add_argument("sources", nargs="+", metavar="SOURCE")
    for name, text in (("remove", "remove the game's data packages"),
                       ("install-repo", "install the game's data from your package sources")):
        sub.add_parser(name, help=text).add_argument("game")
    return ap.parse_args(argv)


def main(argv=None, catalog=None):
    try:
        a = parse(argv)
    except SystemExit as e:
        return 0 if e.code == 0 else 2
    try:
        games = load_games(catalog)
        if a.cmd == "list":
            return cmd_list(games)
        if a.cmd == "import":
            return cmd_import(a.game, a.sources, a.package, a.no_download, games)
        return cmd_helper({"remove": "remove-data", "install-repo": "install-repo-data"}[a.cmd], a.game, games)
    except Refused as e:
        say("error", str(e))
        return 2
    except Failed as e:
        say("error", str(e))
        return 1
    except (OSError, ValueError, KeyError) as e:
        say("error", "%s: %s" % (type(e).__name__, e))
        return 1
    except KeyboardInterrupt:
        say("error", "interrupted")
        return 1


if __name__ == "__main__":
    sys.exit(main())
