[WIP] tensorflow: re-enable build from source

pkgs/build-support/build-bazel-package/default.nix

@@ -1,18 +1,18 @@
-{ stdenv, bazel, cacert, enableNixHacks ? true }:
+{ stdenv, bazel, cacert }:
 
-args@{ name, bazelFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
+args@{ name, bazelFlags ? [], bazelBuildFlags ? [], bazelFetchFlags ? [], bazelTarget, buildAttrs, fetchAttrs, ... }:
 
 let
   fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" ];
   fBuildAttrs = fArgs // buildAttrs;
   fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
 
 in stdenv.mkDerivation (fBuildAttrs // {
-  inherit name bazelFlags bazelTarget;
+  inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
   deps = stdenv.mkDerivation (fFetchAttrs // {
     name = "${name}-deps";
-    inherit bazelFlags bazelTarget;
+    inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
 
     nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
 
@@ -37,7 +37,16 @@ in stdenv.mkDerivation (fBuildAttrs // {
 
       # We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
       # https://github.com/bazelbuild/bazel/issues/6502
-      BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 USER=homeless-shelter bazel --output_base="$bazelOut" --output_user_root="$bazelUserRoot" fetch --loading_phase_threads=1 $bazelFlags $bazelTarget
+      BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
+      USER=homeless-shelter \
+      bazel \
+        --output_base="$bazelOut" \
+        --output_user_root="$bazelUserRoot" \
+        fetch \
+        --loading_phase_threads=1 \
+        $bazelFlags \
+        $bazelFetchFlags \
+        $bazelTarget
 
       runHook postBuild
     '';
@@ -74,12 +83,14 @@ in stdenv.mkDerivation (fBuildAttrs // {
     '';
 
     dontFixup = true;
+    allowedRequisites = [];
+
     outputHashMode = "recursive";
     outputHashAlgo = "sha256";
     outputHash = fetchAttrs.sha256;
   });
 
-  nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (if enableNixHacks then (bazel.override { enableNixHacks = true; }) else bazel) ];
+  nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ];
 
   preHook = fBuildAttrs.preHook or "" + ''
     export bazelOut="$NIX_BUILD_TOP/output"
@@ -99,41 +110,15 @@ in stdenv.mkDerivation (fBuildAttrs // {
   buildPhase = fBuildAttrs.buildPhase or ''
     runHook preBuild
 
-    # Bazel sandboxes the execution of the tools it invokes, so even though we are
-    # calling the correct nix wrappers, the values of the environment variables
-    # the wrappers are expecting will not be set. So instead of relying on the
-    # wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
-    # and related flags.
-    #
-    copts=()
-    host_copts=()
-    for flag in $NIX_CFLAGS_COMPILE; do
-      copts+=( "--copt=$flag" )
-      host_copts+=( "--host_copt=$flag" )
-    done
-    for flag in $NIX_CXXSTDLIB_COMPILE; do
-      copts+=( "--copt=$flag" )
-      host_copts+=( "--host_copt=$flag" )
-    done
-    linkopts=()
-    host_linkopts=()
-    for flag in $NIX_LD_FLAGS; do
-      linkopts+=( "--linkopt=$flag" )
-      host_linkopts+=( "--host_linkopt=$flag" )
-    done
-
     BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
     USER=homeless-shelter \
     bazel \
       --output_base="$bazelOut" \
       --output_user_root="$bazelUserRoot" \
       build \
       -j $NIX_BUILD_CORES \
-      "''${copts[@]}" \
-      "''${host_copts[@]}" \
-      "''${linkopts[@]}" \
-      "''${host_linkopts[@]}" \
       $bazelFlags \
+      $bazelBuildFlags \
       $bazelTarget
 
     runHook postBuild

pkgs/development/interpreters/python/cpython/2.7/atomic_pyc.patch

@@ -0,0 +1,116 @@
+diff --git a/Lib/py_compile.py b/Lib/py_compile.py
+index 978da73d74..3559eb95ca 100644
+--- a/Lib/py_compile.py
++++ b/Lib/py_compile.py
+@@ -120,16 +120,27 @@ def compile(file, cfile=None, dfile=None, doraise=False):
+             return
+     if cfile is None:
+         cfile = file + (__debug__ and 'c' or 'o')
+-    with open(cfile, 'wb') as fc:
+-        fc.write('\0\0\0\0')
+-        if "DETERMINISTIC_BUILD" in os.environ:
++    # Atomically write the pyc/pyo file.  Issue #13146.
++    # id() is used to generate a pseudo-random filename.
++    path_tmp = '{}.{}'.format(cfile, id(cfile))
++    try:
++        with open(path_tmp, 'wb') as fc:
+             fc.write('\0\0\0\0')
+-        else:
+-            wr_long(fc, timestamp)
+-        marshal.dump(codeobject, fc)
+-        fc.flush()
+-        fc.seek(0, 0)
+-        fc.write(MAGIC)
++            if "DETERMINISTIC_BUILD" in os.environ:
++                fc.write('\0\0\0\0')
++            else:
++                wr_long(fc, timestamp)
++            marshal.dump(codeobject, fc)
++            fc.flush()
++            fc.seek(0, 0)
++            fc.write(MAGIC)
++        os.rename(path_tmp, cfile)
++    except OSError:
++        try:
++            os.unlink(path_tmp)
++        except OSError:
++            pass
++        raise
+
+ def main(args=None):
+     """Compile several source files.
+diff --git a/Python/import.c b/Python/import.c
+index 1e31d79279..f78a1efcf0 100644
+--- a/Python/import.c
++++ b/Python/import.c
+@@ -951,6 +951,8 @@ static void
+ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, time_t mtime)
+ {
+     FILE *fp;
++    size_t cpathname_len;
++    char *cpathname_tmp;
+ #ifdef MS_WINDOWS   /* since Windows uses different permissions  */
+     mode_t mode = srcstat->st_mode & ~S_IEXEC;
+     /* Issue #6074: We ensure user write access, so we can delete it later
+@@ -963,11 +965,28 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, t
+     mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH;
+ #endif
+
++#ifdef MS_WINDOWS
+     fp = open_exclusive(cpathname, mode);
++#else
++    /* Under POSIX, we first write to a tmp file and then take advantage
++       of atomic renaming. */
++    cpathname_len = strlen(cpathname);
++    cpathname_tmp = PyMem_MALLOC(cpathname_len + 5);
++    if (cpathname_tmp == NULL) {
++        PyErr_Clear();
++        return;
++    }
++    memcpy(cpathname_tmp, cpathname, cpathname_len);
++    memcpy(cpathname_tmp + cpathname_len, ".tmp", 5);
++    fp = open_exclusive(cpathname_tmp, mode);
++#endif
+     if (fp == NULL) {
+         if (Py_VerboseFlag)
+             PySys_WriteStderr(
+                 "# can't create %s\n", cpathname);
++#ifndef MS_WINDOWS
++        PyMem_FREE(cpathname_tmp);
++#endif
+         return;
+     }
+     PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
+@@ -979,7 +998,12 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, t
+             PySys_WriteStderr("# can't write %s\n", cpathname);
+         /* Don't keep partial file */
+         fclose(fp);
++#ifdef MS_WINDOWS
+         (void) unlink(cpathname);
++#else
++        (void) unlink(cpathname_tmp);
++        PyMem_FREE(cpathname_tmp);
++#endif
+         return;
+     }
+     /* Now write the true mtime (as a 32-bit field) */
+@@ -989,6 +1013,19 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, t
+         PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
+         fflush(fp);
+     }
++    /* Under POSIX, do an atomic rename */
++#ifndef MS_WINDOWS
++    if (rename(cpathname_tmp, cpathname)) {
++        if (Py_VerboseFlag)
++            PySys_WriteStderr("# can't write %s\n", cpathname);
++        /* Don't keep tmp file */
++        fclose(fp);
++        (void) unlink(cpathname_tmp);
++        PyMem_FREE(cpathname_tmp);
++        return;
++    }
++    PyMem_FREE(cpathname_tmp);
++#endif
+     fclose(fp);
+     if (Py_VerboseFlag)
+         PySys_WriteStderr("# wrote %s\n", cpathname);

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -79,6 +79,12 @@ let
         sha256 = "0l9rw6r5r90iybdkp3hhl2pf0h0s1izc68h5d3ywrm92pq32wz57";
       })
 
+      # Fix race-condition during pyc creation. Has a slight backwards
+      # incompatible effect: pyc symlinks will now be overridden
+      # (https://bugs.python.org/issue17222). Included in python >= 3.4,
+      # backported in debian since 2013.
+      # https://bugs.python.org/issue13146
+      ./atomic_pyc.patch
     ] ++ optionals (x11Support && stdenv.isDarwin) [
       ./use-correct-tcl-tk-on-darwin.patch
     ] ++ optionals stdenv.isLinux [

pkgs/development/libraries/science/math/tensorflow/bin.nix

@@ -0,0 +1,72 @@
+{ stdenv
+, fetchurl
+, patchelf
+, cudaSupport ? false, symlinkJoin, cudatoolkit, cudnn, nvidia_x11
+}:
+
+with stdenv.lib;
+let
+  unavailable = throw "libtensorflow is not available for this platform!";
+
+  tfType = if cudaSupport then "gpu" else "cpu";
+
+  system = 
+    if      stdenv.isLinux  then "linux"
+    else if stdenv.isDarwin then "darwin"
+    else unavailable;
+
+  platform =
+    if stdenv.isx86_64 then "x86_64"
+    else unavailable;
+
+  rpath = makeLibraryPath ([stdenv.cc.libc stdenv.cc.cc.lib] ++
+            optionals cudaSupport [ cudatoolkit.out cudatoolkit.lib cudnn nvidia_x11 ]);
+
+  packages = import ./binary-hashes.nix;
+  packageName = "${tfType}-${system}-${platform}";
+  url = packages.${packageName} or unavailable;
+
+  patchLibs =
+    if stdenv.isDarwin
+    then ''
+      install_name_tool -id $out/lib/libtensorflow.dylib $out/lib/libtensorflow.dylib
+      install_name_tool -id $out/lib/libtensorflow_framework.dylib $out/lib/libtensorflow_framework.dylib
+    ''
+    else ''
+      patchelf --set-rpath "${rpath}:$out/lib" $out/lib/libtensorflow.so
+      patchelf --set-rpath "${rpath}" $out/lib/libtensorflow_framework.so
+    '';
+
+in stdenv.mkDerivation rec {
+  pname = "libtensorflow";
+  inherit (packages) version;
+
+  src = fetchurl url;
+
+  # Patch library to use our libc, libstdc++ and others
+  buildCommand = ''
+    mkdir -pv $out
+    tar -C $out -xzf $src
+    chmod -R +w $out
+    ${patchLibs}
+
+    # Write pkgconfig file.
+    mkdir $out/lib/pkgconfig
+    cat > $out/lib/pkgconfig/tensorflow.pc << EOF
+    Name: TensorFlow
+    Version: ${version}
+    Description: Library for computation using data flow graphs for scalable machine learning
+    Requires:
+    Libs: -L$out/lib -ltensorflow
+    Cflags: -I$out/include/tensorflow
+    EOF
+  '';
+
+  meta = {
+    description = "C API for TensorFlow";
+    homepage = https://www.tensorflow.org/install/lang_c;
+    license = licenses.asl20;
+    platforms = [ "x86_64-linux" "x86_64-darwin" ];
+    maintainers = with maintainers; [ basvandijk ];
+  };
+}

pkgs/development/libraries/science/math/tensorflow/binary-hashes.nix

@@ -0,0 +1,15 @@
+{
+version = "1.14.0";
+"cpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "04bi3ijq4sbb8c5vk964zlv0j9mrjnzzxd9q9knq3h273nc1a36k";
+};
+"gpu-linux-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.14.0.tar.gz";
+  sha256 = "1ffnpyj9jjgwxpjfiyjvq4dm3n6nwiksim5jld9zw7fdswh215x6";
+};
+"cpu-darwin-x86_64" = {
+  url = "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.14.0.tar.gz";
+  sha256 = "0zsd5ils1a17j6jzh0c7q1z56fw46gkzybbnms7h2rgg8al0rh92";
+};
+}