Could not read the output of 'nix-prefetch-url'

[rscorer]

Hi, learning nix etc using nix.dev and ran into this issue when executing

nix-shell -p niv --run "niv init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11"

Initializing
  Creating nix/sources.nix
  Creating nix/sources.json
  Importing 'nixpkgs' ...
  Adding package nixpkgs
    FATAL: Could not read the output of 'nix-prefetch-url'. This is a bug. Please create a
    ticket:

      https://github.com/nmattia/niv/issues/new

    Thanks! I'll buy you a beer.
    command:
    nix-prefetch-url--unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz
    stdout:

    stderr:
    error: failed to extract archive (Write failed)

    FATAL: One or more packages failed to update:
    nixpkgs: ExitFailure 1
  ERROR: ExitFailure 1
ERROR: ExitFailure 1

[rscorer]

$ niv --version
0.2.22

Ubuntu 23.10
Linux nixie 6.5.0-17-generic #17-Ubuntu SMP PREEMPT_DYNAMIC Thu Jan 11 14:01:59 UTC 2024 x86_64 GNU/Linux

On DigitalOcean

Nix installed with The Determinate Nix Installer

[nmattia]

Hi @rscorer, thanks for the report. Does this happen consistently? What happens if you run the failing command?

$ nix-prefetch-url --unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz

[rscorer]

When I remember to delete the local ./nix folder, it happens each time - and I tried various nixOS versions too, all with the same result...

# nix-prefetch-url --unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz
path is '/nix/store/brkw8r1qx8m53zgl585qcrm79chbcb3b-20f65b86b6485decb43c5498780c223571dd56ef.tar.gz'
04n3qgkgj90qzxm8wsgr2pb8q3qvxjrygcchg8sr34p29cb2mv33

Contents of the ./nix folder - maybe this shows how far your tool got?

# tree -a nix
nix
├── sources.json
└── sources.nix

1 directory, 2 files

cat nix/sources.json

{}

cat nix/sources.nix

# This file has been generated by Niv.

let

  #
  # The fetchers. fetch_<type> fetches specs of type <type>.
  #

  fetch_file = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
    if spec.builtin or true then
      builtins_fetchurl { inherit (spec) url sha256; name = name'; }
    else
      pkgs.fetchurl { inherit (spec) url sha256; name = name'; };

  fetch_tarball = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
    if spec.builtin or true then
      builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
    else
      pkgs.fetchzip { name = name'; inherit (spec) url sha256; };

  fetch_git = name: spec:
    let
      ref =
        spec.ref or (
          if spec ? branch then "refs/heads/${spec.branch}" else
          if spec ? tag then "refs/tags/${spec.tag}" else
          abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"
        );
      submodules = spec.submodules or false;
      submoduleArg =
        let
          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
          emptyArgWithWarning =
            if submodules
            then
              builtins.trace
                (
                  "The niv input \"${name}\" uses submodules "
                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
                  + "does not support them"
                )
                { }
            else { };
        in
        if nixSupportsSubmodules
        then { inherit submodules; }
        else emptyArgWithWarning;
    in
    builtins.fetchGit
      ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);

  fetch_local = spec: spec.path;

  fetch_builtin-tarball = name: throw
    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=tarball -a builtin=true'';

  fetch_builtin-url = name: throw
    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=file -a builtin=true'';

  #
  # Various helpers
  #

  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
  sanitizeName = name:
    (
      concatMapStrings (s: if builtins.isList s then "-" else s)
        (
          builtins.split "[^[:alnum:]+._?=-]+"
            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
        )
    );

  # The set of packages used when specs are fetched using non-builtins.
  mkPkgs = sources: system:
    let
      sourcesNixpkgs =
        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
    in
    if builtins.hasAttr "nixpkgs" sources
    then sourcesNixpkgs
    else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
      import <nixpkgs> { }
    else
      abort
        ''
          Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
          add a package called "nixpkgs" to your sources.json.
        '';

  # The actual fetching function.
  fetch = pkgs: name: spec:

    if ! builtins.hasAttr "type" spec then
      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
    else if spec.type == "file" then fetch_file pkgs name spec
    else if spec.type == "tarball" then fetch_tarball pkgs name spec
    else if spec.type == "git" then fetch_git name spec
    else if spec.type == "local" then fetch_local spec
    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
    else if spec.type == "builtin-url" then fetch_builtin-url name
    else
      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";

  # If the environment variable NIV_OVERRIDE_${name} is set, then use
  # the path directly as opposed to the fetched source.
  replace = name: drv:
    let
      saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name;
      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
    in
    if ersatz == "" then drv else
      # this turns the string into an actual Nix path (for both absolute and
      # relative paths)
    if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";

  # Ports of functions for older nix versions

  # a Nix version of mapAttrs if the built-in doesn't exist
  mapAttrs = builtins.mapAttrs or (
    f: set: with builtins;
    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
  );

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
  range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1);

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
  concatMapStrings = f: list: concatStrings (map f list);
  concatStrings = builtins.concatStringsSep "";

  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
  optionalAttrs = cond: as: if cond then as else { };

  # fetchTarball version that is compatible between all the versions of Nix
  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchTarball;
    in
    if lessThan nixVersion "1.12" then
      fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
    else
      fetchTarball attrs;

  # fetchurl version that is compatible between all the versions of Nix
  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchurl;
    in
    if lessThan nixVersion "1.12" then
      fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
    else
      fetchurl attrs;

  # Create the final "sources" from the config
  mkSources = config:
    mapAttrs
      (
        name: spec:
          if builtins.hasAttr "outPath" spec
          then
            abort
              "The values in sources.json should not have an 'outPath' attribute"
          else
            spec // { outPath = replace name (fetch config.pkgs name spec); }
      )
      config.sources;

  # The "config" used by the fetchers
  mkConfig =
    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
    , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile)
    , system ? builtins.currentSystem
    , pkgs ? mkPkgs sources system
    }: rec {
      # The sources, i.e. the attribute set of spec name to spec
      inherit sources;

      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
      inherit pkgs;
    };

in
mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); }

HTH

[nmattia]

@rscorer thanks, I think it might be an issue with the environment set up by nix-shell. Can you try to run the niv command outside of the nix-shell (after having wiped the nix/ folder of course, good point)?

$ $(nix-shell -p niv --run "which niv") init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11

[rscorer]

@nmattia cool, that seems to have worked 👍

$(nix-shell -p niv --run "which niv") init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11
Initializing
  Creating nix/sources.nix
  Creating nix/sources.json
  Importing 'nixpkgs' ...
  Adding package nixpkgs
    Writing new sources file
  Done: Adding package nixpkgs
Done: Initializing

If it would help - I can add your public ssh key to my server so you can poke around and, once you stopped laughing at my mistakes, figure out what I've broken!.
It's a VM running on a Digital Ocean trial, I can send you details etc. I've set it up to learn nix, and have rebuilt it a few times already 😀

cat nix/sources.json

{
    "nixpkgs": {
        "branch": "nixos-23.11",
        "description": "Nix Packages collection & NixOS",
        "homepage": "",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "809cca784b9f72a5ad4b991e0e7bcf8890f9c3a6",
        "sha256": "174d7cbgjyhlin2lm71chgdqs2cprqphnka9w33nlc1qh226ks3l",
        "type": "tarball",
        "url": "https://github.com/nixos/nixpkgs/archive/809cca784b9f72a5ad4b991e0e7bcf8890f9c3a6.tar.gz",
        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
    }
}

cat nix/sources.nix

# This file has been generated by Niv.

let

  #
  # The fetchers. fetch_<type> fetches specs of type <type>.
  #

  fetch_file = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
    if spec.builtin or true then
      builtins_fetchurl { inherit (spec) url sha256; name = name'; }
    else
      pkgs.fetchurl { inherit (spec) url sha256; name = name'; };

  fetch_tarball = pkgs: name: spec:
    let
      name' = sanitizeName name + "-src";
    in
    if spec.builtin or true then
      builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
    else
      pkgs.fetchzip { name = name'; inherit (spec) url sha256; };

  fetch_git = name: spec:
    let
      ref =
        spec.ref or (
          if spec ? branch then "refs/heads/${spec.branch}" else
          if spec ? tag then "refs/tags/${spec.tag}" else
          abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"
        );
      submodules = spec.submodules or false;
      submoduleArg =
        let
          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
          emptyArgWithWarning =
            if submodules
            then
              builtins.trace
                (
                  "The niv input \"${name}\" uses submodules "
                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
                  + "does not support them"
                )
                { }
            else { };
        in
        if nixSupportsSubmodules
        then { inherit submodules; }
        else emptyArgWithWarning;
    in
    builtins.fetchGit
      ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);

  fetch_local = spec: spec.path;

  fetch_builtin-tarball = name: throw
    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=tarball -a builtin=true'';

  fetch_builtin-url = name: throw
    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
        $ niv modify ${name} -a type=file -a builtin=true'';

  #
  # Various helpers
  #

  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
  sanitizeName = name:
    (
      concatMapStrings (s: if builtins.isList s then "-" else s)
        (
          builtins.split "[^[:alnum:]+._?=-]+"
            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
        )
    );

  # The set of packages used when specs are fetched using non-builtins.
  mkPkgs = sources: system:
    let
      sourcesNixpkgs =
        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
    in
    if builtins.hasAttr "nixpkgs" sources
    then sourcesNixpkgs
    else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
      import <nixpkgs> { }
    else
      abort
        ''
          Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
          add a package called "nixpkgs" to your sources.json.
        '';

  # The actual fetching function.
  fetch = pkgs: name: spec:

    if ! builtins.hasAttr "type" spec then
      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
    else if spec.type == "file" then fetch_file pkgs name spec
    else if spec.type == "tarball" then fetch_tarball pkgs name spec
    else if spec.type == "git" then fetch_git name spec
    else if spec.type == "local" then fetch_local spec
    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
    else if spec.type == "builtin-url" then fetch_builtin-url name
    else
      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";

  # If the environment variable NIV_OVERRIDE_${name} is set, then use
  # the path directly as opposed to the fetched source.
  replace = name: drv:
    let
      saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name;
      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
    in
    if ersatz == "" then drv else
      # this turns the string into an actual Nix path (for both absolute and
      # relative paths)
    if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";

  # Ports of functions for older nix versions

  # a Nix version of mapAttrs if the built-in doesn't exist
  mapAttrs = builtins.mapAttrs or (
    f: set: with builtins;
    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
  );

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
  range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1);

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));

  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
  concatMapStrings = f: list: concatStrings (map f list);
  concatStrings = builtins.concatStringsSep "";

  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
  optionalAttrs = cond: as: if cond then as else { };

  # fetchTarball version that is compatible between all the versions of Nix
  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchTarball;
    in
    if lessThan nixVersion "1.12" then
      fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
    else
      fetchTarball attrs;

  # fetchurl version that is compatible between all the versions of Nix
  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
    let
      inherit (builtins) lessThan nixVersion fetchurl;
    in
    if lessThan nixVersion "1.12" then
      fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
    else
      fetchurl attrs;

  # Create the final "sources" from the config
  mkSources = config:
    mapAttrs
      (
        name: spec:
          if builtins.hasAttr "outPath" spec
          then
            abort
              "The values in sources.json should not have an 'outPath' attribute"
          else
            spec // { outPath = replace name (fetch config.pkgs name spec); }
      )
      config.sources;

  # The "config" used by the fetchers
  mkConfig =
    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
    , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile)
    , system ? builtins.currentSystem
    , pkgs ? mkPkgs sources system
    }: rec {
      # The sources, i.e. the attribute set of spec name to spec
      inherit sources;

      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
      inherit pkgs;
    };

in
mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); }

Proof I'm learning 😉 - also, would this shell.nix be interfering with niv?

cat shell.nix

let

nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-23.11";

pkgs = import nixpkgs { config = {}; overlays = []; };

in
pkgs.mkShell {

packages = with pkgs; [

cowsay

lolcat

niv

];
GREETING = "Hello, Nix!";
shellHook = ''

echo $GREETING | cowsay | lolcat

'';

}