niv update fails with "Could not read the output of git" on https://sourceware.org/git/binutils-gdb.git

[MangoIV]

When trying to update source.{nix,json}, I encounter the following output of niv

Updating all packages
  INFO: new sources.nix available: 27 -> 28
    Please run 'niv init' or add the following line in the nix/sources.nix file:
    # niv: no_update
  Package: nixpkgs
  Package: binutils-gdb
  FATAL: Could not read the output of 'git'.
  command: git ls-remote https://sourceware.org/git/binutils-gdb.git refs/heads/master
  Git produced too much output:
    edf0ae3b9f1ca23daaa55628683edc9a3360286f    refs/heads/master
    9673461f75f44cdd05deb9e33888eebb91c710d9    refs/original/refs/heads/master


  This is a bug. Please create a ticket:
    https://github.com/nmattia/niv/issues/new
  Thanks! I'll buy you a beer.
  Package: rr
  Package: gdb-walkers
  Package: niv
  Package: processor-trace
  FATAL: One or more packages failed to update:
  binutils-gdb: ExitFailure 1
ERROR: ExitFailure 1

niv version: 0.2.22
git version: 2.25.0

sources.json:

Details

{
    "binutils-gdb": {
        "branch": "master",
        "repo": "https://sourceware.org/git/binutils-gdb.git",
        "rev": "49c843e6d2d0577200e7c1d2d02855f21a3a9dde",
        "type": "git"
    },
    "gdb-walkers": {
        "branch": "master",
        "description": "Bring mdb walkers to gdb, also add other helpful commands.",
        "homepage": "",
        "owner": "hardenedapple",
        "repo": "gdb-walkers",
        "rev": "c0701c4c87852bd09e21ca313c48dd4a649cfd0d",
        "sha256": "1sd61a90lg8bkddl8lp15qady1wvbjmhjgm0d3lb813nwimlka9y",
        "type": "tarball",
        "url": "https://github.com/hardenedapple/gdb-walkers/archive/c0701c4c87852bd09e21ca313c48dd4a649cfd0d.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
    "niv": {
        "branch": "master",
        "description": "Easy dependency management for Nix projects",
        "homepage": "https://github.com/nmattia/niv",
        "owner": "nmattia",
        "repo": "niv",
        "rev": "82e5cd1ad3c387863f0545d7591512e76ab0fc41",
        "sha256": "090l219mzc0gi33i3psgph6s2pwsc8qy4lyrqjdj4qzkvmaj65a7",
        "type": "tarball",
        "url": "https://github.com/nmattia/niv/archive/82e5cd1ad3c387863f0545d7591512e76ab0fc41.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
    "nixpkgs": {
        "branch": "nixos-22.11",
        "description": "Nix Packages collection",
        "homepage": "",
        "owner": "NixOS",
        "repo": "nixpkgs",
        "rev": "2d10e73416ec1449ef74aeac7faf2cf8c556ff5a",
        "sha256": "00s89np0sqr3jxxp5h9nrpqy30fy4vsrmis6mmryrrmjqh09lpfv",
        "type": "tarball",
        "url": "https://github.com/NixOS/nixpkgs/archive/2d10e73416ec1449ef74aeac7faf2cf8c556ff5a.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
    "processor-trace": {
        "branch": "master",
        "description": "libipt - an Intel(R) Processor Trace decoder library",
        "homepage": "",
        "owner": "01org",
        "repo": "processor-trace",
        "rev": "c848a85c3104e2f5780741f85de5c9e65476ece2",
        "sha256": "1ml8g6pm2brlcqp90yvgc780xf64d6k2km7fiqs88wvhlwsl7vzf",
        "type": "tarball",
        "url": "https://github.com/01org/processor-trace/archive/c848a85c3104e2f5780741f85de5c9e65476ece2.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    },
    "rr": {
        "branch": "master",
        "description": "Record and Replay Framework",
        "homepage": "http://rr-project.org/",
        "owner": "rr-debugger",
        "repo": "rr",
        "rev": "e77b5f8ca4b360daffd31cf72cb6b093fa9e0b62",
        "sha256": "sha256:1gxphqcv1yw2ffmjp0d2cv0mpccr00pf9jhf44rq57jqdsvlfn2c",
        "type": "tarball",
        "url": "https://github.com/rr-debugger/rr/archive/3f87444659d1f063397fabc7791ed3b13b15c798.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    }
}

sources.nix

Details

# This file has been generated by Niv.

let

  #
  # The fetchers. fetch_<type> fetches specs of type <type>.
  #

  fetch_file = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
      if spec.builtin or true then
        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
      else
        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };

  fetch_tarball = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
      if spec.builtin or true then
        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
      else
        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };

  fetch_git = name: spec:
    let
      ref =
        if spec ? ref then spec.ref else
          if spec ? branch then "refs/heads/${spec.branch}" else
            if spec ? tag then "refs/tags/${spec.tag}" else
              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
      submodules = if spec ? submodules then spec.submodules else false;
      submoduleArg =
        let
          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
          emptyArgWithWarning =
            if submodules == true
            then
              builtins.trace
                (
                  "The niv input \"${name}\" uses submodules "
                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
                  + "does not support them"
                )
                {}
            else {};
        in
          if nixSupportsSubmodules
          then { inherit submodules; }
          else emptyArgWithWarning;
    in
      builtins.fetchGit
        ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);

  fetch_local = spec: spec.path;

  fetch_builtin-tarball = name: throw
    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=tarball -a builtin=true'';

  fetch_builtin-url = name: throw
    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=file -a builtin=true'';

  #
  # Various helpers
  #

  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
  sanitizeName = name:
    (
      concatMapStrings (s: if builtins.isList s then "-" else s)
        (
          builtins.split "[^[:alnum:]+._?=-]+"
            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
        )
    );

  # The set of packages used when specs are fetched using non-builtins.
  mkPkgs = sources: system:
    let
      sourcesNixpkgs =
        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
    in
      if builtins.hasAttr "nixpkgs" sources
      then sourcesNixpkgs
      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
        import <nixpkgs> {}
      else
        abort
          ''
            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
            add a package called "nixpkgs" to your sources.json.
          '';

  # The actual fetching function.
  fetch = pkgs: name: spec:

    if ! builtins.hasAttr "type" spec then
      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
    else if spec.type == "file" then fetch_file pkgs name spec
    else if spec.type == "tarball" then fetch_tarball pkgs name spec
    else if spec.type == "git" then fetch_git name spec
    else if spec.type == "local" then fetch_local spec
    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
    else if spec.type == "builtin-url" then fetch_builtin-url name
    else
      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";

  # If the environment variable NIV_OVERRIDE_${name} is set, then use
  # the path directly as opposed to the fetched source.
  replace = name: drv:
    let
      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
    in
      if ersatz == "" then drv else
        # this turns the string into an actual Nix path (for both absolute and
        # relative paths)
        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";

  # Ports of functions for older nix versions

  # a Nix version of mapAttrs if the built-in doesn't exist
  mapAttrs = builtins.mapAttrs or (
    f: set: with builtins;
    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
  );

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
  concatMapStrings = f: list: concatStrings (map f list);
  concatStrings = builtins.concatStringsSep "";

  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
  optionalAttrs = cond: as: if cond then as else {};

  # fetchTarball version that is compatible between all the versions of Nix
  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchTarball;
    in
      if lessThan nixVersion "1.12" then
        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
      else
        fetchTarball attrs;

  # fetchurl version that is compatible between all the versions of Nix
  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchurl;
    in
      if lessThan nixVersion "1.12" then
        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
      else
        fetchurl attrs;

  # Create the final "sources" from the config
  mkSources = config:
    mapAttrs (
      name: spec:
        if builtins.hasAttr "outPath" spec
        then abort
          "The values in sources.json should not have an 'outPath' attribute"
        else
          spec // { outPath = replace name (fetch config.pkgs name spec); }
    ) config.sources;

  # The "config" used by the fetchers
  mkConfig =
    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
    , system ? builtins.currentSystem
    , pkgs ? mkPkgs sources system
    }: rec {
      # The sources, i.e. the attribute set of spec name to spec
      inherit sources;

      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
      inherit pkgs;
    };

in
mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

Thank you for making this tool!

[MangoIV]

If you don't have time, ping me, I can try fixing this during the weekend. :)

[nmattia]

Thanks! I'll buy you a beer.

One beer for you! And thanks for all the details.

command: git ls-remote https://sourceware.org/git/binutils-gdb.git refs/heads/master
  Git produced too much output:
    edf0ae3b9f1ca23daaa55628683edc9a3360286f    refs/heads/master
    9673461f75f44cdd05deb9e33888eebb91c710d9    refs/original/refs/heads/master

well, that's interesting! I guess this is a mirror of some sort?

I had a quick look and couldn't find a way to have an exact match in git ls-remote. I think we'll have to use a filter:

$ ref="refs/heads/master"
$ git ls-remote --refs https://sourceware.org/git/binutils-gdb.git "$ref" | grep "\t$ref"
edf0ae3b9f1ca23daaa55628683edc9a3360286f	refs/heads/master

This is the commit you'd want, correct?

If you don't have time, ping me, I can try fixing this during the weekend. :)

I'll admit that I haven't written Haskell in a long while, so that'd be much appreciated :p let me know if you get to it otherwise I'll try to push a fix! I think we just need a <&> filter (...) around here:

niv/src/Niv/Git/Cmd.hs

Line 180 in dd67878

sout <- runGit gitArgs