build2

Makefile

@@ -26,9 +26,20 @@ $(OBJDIR)/generate_comms_proto: pkg/core/proto/comms.proto
 	cd embly-wrapper-rs && cargo build
 	touch build/generate_comms_proto
 
+build_build_server: $(OBJDIR)/build_build_server
+$(OBJDIR)/build_build_server: pkg/nixbuild/* pkg/nixbuild/**/* cmd/build-server/*
+	cd cmd/build-server \
+	&& docker -l=info build -f ./Dockerfile -t embly/build-server ../.. \
+	&& docker kill embly-build || true \
+	&& docker rm embly-build || true
+	touch build/build_build_server
+
 all:
 	make -j install_embly install_embly_wrapper generate_http_proto generate_comms_proto
 
+generate: generate_comms_proto generate_http_proto
+	cd pkg/nixbuild && go generate
+
 build_embly:
 	make -j install_embly install_embly_wrapper
 
@@ -41,12 +52,18 @@ push_embly_image:
 test:
 	make -j wrapper_test cargo_test go_test
 
+build_server_shell:
+	docker run -it embly/build-server sh
+
+push_build_server: build_build_server
+	docker --config=~/.docker-embly  push embly/build-server
+
 build_examples: build_embly
 	cd examples/mjpeg && embly build
 	cd examples/kv && embly build
 	cd examples/project && embly build
 
-build_hello_world:
+build_hello_world: install_embly build_build_server
 	cd examples/hello-world && embly build
 
 go_test:
@@ -77,6 +94,9 @@ bundle_kv_example: build_embly
 bundle_project_example: build_embly
 	cd examples/project && embly bundle
 
+new_build: install_embly
+	cd examples/hello-world && embly build
+
 run_project_example: build_embly
 	cd examples/project && embly dev
 

app/example/hello/Cargo.toml

@@ -1,6 +1,7 @@
 [package]
 name = "hello"
 version = "0.0.1"
+edition = "2018"
 
 [dependencies]
 embly = "0.0.4"

app/terraform/public_files.hcl

@@ -0,0 +1,6 @@
+
+resource "aws_s3_bucket" "embly_static_files" {
+  bucket = "embly-static"
+  acl    = "public-read"
+}
+

app/terraform/public_files.tf

@@ -0,0 +1,5 @@
+
+resource "aws_s3_bucket" "embly_static_files" {
+  bucket = "embly-static"
+  acl    = "public-read"
+}

cmd/build-server/Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.13-buster AS go
+
+WORKDIR /embly
+
+COPY go.mod go.sum ./
+RUN go mod download
+COPY pkg ./pkg
+COPY cmd ./cmd
+
+RUN cd cmd/build-server \
+    && go build \
+    && strip build-server
+
+FROM nixos/nix as patchelf
+COPY --from=go /embly/cmd/build-server/build-server /usr/bin/build-server
+RUN export LIB="/nix/store/$(ls /nix/store | grep glibc)/lib" \
+    && ls -lah $LIB \
+    && nix-env -iA nixpkgs.patchelf \
+    && patchelf --set-rpath "$LIB" /usr/bin/build-server \
+    && patchelf --set-interpreter "$LIB/ld-linux-x86-64.so.2" /usr/bin/build-server
+
+FROM nixos/nix
+COPY --from=patchelf /usr/bin/build-server /usr/bin/build-server

cmd/build-server/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"embly/pkg/nixbuild"
+	"log"
+)
+
+func main() {
+	builder, err := nixbuild.NewBuildServer()
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println("starting server")
+	if err := builder.StartServer(); err != nil {
+		log.Fatal(err)
+	}
+}

compilers/nix/Dockerfile

@@ -0,0 +1,3 @@
+FROM nixery.dev/shell/wabt/sccache  
+
+

compilers/nix/cctools.nix

@@ -0,0 +1,38 @@
+with import <nixpkgs> { };
+with pkgs;
+stdenv.mkDerivation {
+  name = "cctools";
+  nativeBuildInputs = [ autoconf automake libtool autoreconfHook ];
+  buildInputs = [ libuuid clang ];
+  src = fetchFromGitHub {
+    owner = "tpoechtrager";
+    repo = "cctools-port";
+    rev = "ae83e7fcdeb2b8bd838b6008f1972bd853fa0e76";
+    sha256 = "1zncypdn3a8cqlhzqa975d342yclw5ncm7l3g7rb9n8pp860pcxm";
+  };
+  #   configureFlags = [ "--target=x86_64-apple-darwin" ];
+  postPatch = ''
+    export CC=clang
+    export CXX=clang++
+    cd cctools
+  '';
+}
+
+# with import <nixpkgs> { };
+# with pkgs;
+# (runCommandCC "cctools" {
+#   buildInputs = [ clang ];
+#   src = fetchFromGitHub {
+#     owner = "tpoechtrager";
+#     repo = "cctools-port";
+#     rev = "ae83e7fcdeb2b8bd838b6008f1972bd853fa0e76";
+#     sha256 = "1zncypdn3a8cqlhzqa975d342yclw5ncm7l3g7rb9n8pp860pcxm";
+#   };
+# } ''
+#   cp -r $src/cctools .
+#   chmod -R +w .
+#   cd cctools
+#   export CC=clang
+#   ./configure  --target=x86_64-apple-darwin11
+
+# '')

compilers/nix/crossshell.nix

@@ -0,0 +1,5 @@
+with import <nixpkgs> { crossSystem = { config = "x86_64-apple-darwin"; }; };
+
+mkShell {
+  buildInputs = [ bash ]; # your dependencies here
+}

compilers/nix/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3.5"
+
+services:
+  app:
+    image: nixos/nix
+    volumes:
+      - ./:/root/embly
+    # command: nix-shell -p bash
+    command: sh

compilers/nix/lucetc-0.5.0.nix

@@ -0,0 +1,19 @@
+with import <nixpkgs> { };
+
+let
+  lucetFile = pkgs.fetchurl {
+    url = "https://embly-static.s3.amazonaws.com/lucetc-0.5.0";
+    sha256 = "0lrcbv2agx6k36hwxarm4s0mijp8ikx9041jqr72qgq9dwm0s39z";
+  };
+in (runCommandCC "lucetc-0.5.0" {
+  buildInputs = [ cacert ];
+  propagatedBuildInputs = [ stdenv.cc.cc ];
+} ''
+  cp ${lucetFile} ./lucetc-0.5.0
+  chmod +w lucetc-0.5.0
+  patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" lucetc-0.5.0
+  patchelf --set-rpath ${stdenv.cc.cc.lib}/lib64 lucetc-0.5.0
+  mkdir -p $out/bin
+  mv lucetc-0.5.0 $out/bin/lucetc
+  chmod +x $out/bin/lucetc
+'')

compilers/nix/lucetc.nix

@@ -0,0 +1,19 @@
+with import <nixpkgs> { };
+
+let
+  lucetFile = pkgs.fetchurl {
+    url = "https://embly-static.s3.amazonaws.com/lucetc-0.4.1";
+    sha256 = "0swmsngxh0p3lk5r88s0kp8nk934qkq52nqnjasa2cv08ghr9w3j";
+  };
+in (runCommandCC "lucetc-0.4.1" {
+  buildInputs = [ cacert ];
+  propagatedBuildInputs = [ stdenv.cc.cc ];
+} ''
+  cp ${lucetFile} ./lucetc-0.4.1
+  chmod +w lucetc-0.4.1
+  patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" lucetc-0.4.1
+  patchelf --set-rpath ${stdenv.cc.cc.lib}/lib64 lucetc-0.4.1
+  mkdir -p $out/bin
+  mv lucetc-0.4.1 $out/bin/lucetc
+  chmod +x $out/bin/lucetc
+'')

compilers/nix/overlay.nix

@@ -0,0 +1,83 @@
+self: super:
+
+let
+  src = super.fetchFromGitHub {
+    owner = "mozilla";
+    repo = "nixpkgs-mozilla";
+    # commit from: 2019-05-15
+    rev = "9f35c4b09fd44a77227e79ff0c1b4b6a69dff533";
+    sha256 = "18h0nvh55b5an4gmlgfbvwbyqj91bklf1zymis6lbdh75571qaz0";
+  };
+  lucetc = with super;
+    stdenv.mkDerivation {
+      pname = "lucetc";
+      version = "0.4.1";
+      buildInputs = [ cargo rustc cmake ];
+      src = fetchurl {
+        url = "https://static.crates.io/crates/lucetc/lucetc-0.4.1.crate";
+        sha256 = "06v05s9jma84x3fvpm9xnq1lnjd5pfz2cvvg7mvbg7x9mapz574i";
+      };
+      unpackCmd = "tar -zxvf $src";
+      dontConfigure = true;
+      buildPhase = ''
+        env
+                cargo build --release
+              '';
+      installPhase = ''
+        mkdir -p $out/bin
+        mv ./target/release/lucetc $out/bin
+        strip $out/bin/lucetc
+      '';
+    };
+  lucetc2 = with super;
+    (runCommandCC "lucetc-0.4.1" {
+      buildInputs = [ cacert ];
+      propagatedBuildInputs = [ stdenv.cc.cc ];
+    } ''
+      ${wget}/bin/wget https://embly-static.s3.amazonaws.com/lucetc-0.4.1
+
+      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" lucetc-0.4.1
+      patchelf --set-rpath ${stdenv.cc.cc.lib}/lib64 lucetc-0.4.1
+      mkdir -p $out/bin
+      mv lucetc-0.4.1 $out/bin/lucetc
+      chmod +x $out/bin/lucetc
+    '');
+  wrapper2 = with super;
+    (runCommandCC "embly-wrapper-0.0.2" {
+      buildInputs = [ cacert ];
+      propagatedBuildInputs = [ stdenv.cc.cc ];
+    } ''
+      ${wget}/bin/wget https://embly-static.s3.amazonaws.com/embly-wrapper-0.0.2
+
+      patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" embly-wrapper-0.0.2
+      patchelf --set-rpath ${stdenv.cc.cc.lib}/lib64 embly-wrapper-0.0.2
+      mkdir -p $out/bin
+      mv embly-wrapper-0.0.2 $out/bin/embly-wrapper
+      chmod +x $out/bin/embly-wrapper
+    '');
+  wrapper = with super;
+    stdenv.mkDerivation {
+      pname = "embly-wrapper";
+      version = "0.0.2";
+      buildInputs = [ cargo rustc ];
+      src = fetchurl {
+        url =
+          "https://static.crates.io/crates/embly-wrapper/embly-wrapper-0.0.2.crate";
+        sha256 = "08l0s3cw6h44r2mnx1x4409pzp5wm20mslq1shxl2b0crjhrinnh";
+      };
+      unpackCmd = "tar -zxvf $src";
+      dontConfigure = true;
+      buildPhase = ''
+        cargo build --release
+      '';
+      installPhase = ''
+        mkdir -p $out/bin
+        mv ./target/release/embly-wrapper $out/bin
+        strip $out/bin/embly-wrapper
+      '';
+    };
+in with import "${src.out}/rust-overlay.nix" super super; rec {
+  emblyRust = (rustChannelOfTargets "nightly" "2019-12-30" [ "wasm32-wasi" ]);
+  emblyLucetc = lucetc2;
+  emblyWrapper = wrapper2;
+}

compilers/nix/protoc-osx.nix

@@ -0,0 +1,25 @@
+with import <nixpkgs> { };
+
+(runCommandCC "protoc" { } ''
+  set -x
+  mkdir -p $out/bin $out/lib
+  ls -lah ${protobuf}/lib/
+  ls -lah ${protobuf}/bin/
+  otool -L ${protobuf}/bin/protoc
+  otool -l ${protobuf}/bin/protoc
+  cp ${protobuf}/lib/libprotoc.18.dylib $out/lib/
+  cp ${protobuf}/lib/libprotobuf.18.dylib $out/lib/
+  cp ${protobuf}/bin/protoc $out/bin/
+  chmod +w $out/bin/protoc
+  chmod +w $out/lib/libprotoc.18.dylib
+  chmod +w $out/lib/libprotobuf.18.dylib
+  install_name_tool -delete_rpath /nix/store/vfp33xyy80wmx3c725wsxd0rcvxhanws-swift-corefoundation/Library/Frameworks $out/bin/protoc
+  install_name_tool -add_rpath $out/lib/:${stdenv.cc.cc.lib}/lib64:${zlib}/lib $out/bin/protoc
+  install_name_tool -add_rpath $out/lib/:${stdenv.cc.cc.lib}/lib64:${zlib}/lib $out/lib/libprotoc.18.dylib
+  # strip $out/bin/protoc $out/lib/*
+
+  # patchelf --shrink-rpath $out/bin/protoc
+  # patchelf --shrink-rpath $out/lib/libprotoc.18.dylib
+  # patchelf --shrink-rpath $out/lib/libprotobuf.18.dylib
+  set +x
+'')

compilers/nix/protoc.nix

@@ -0,0 +1,21 @@
+with import <nixpkgs> { };
+
+(runCommandCC "protoc" { } ''
+  set -x
+  mkdir -p $out/bin $out/lib
+
+  cp ${protobuf}/lib/libprotoc.so.18 $out/lib/
+  cp ${protobuf}/lib/libprotobuf.so.18 $out/lib/
+  cp ${protobuf}/bin/protoc $out/bin/
+  chmod +w $out/bin/protoc
+  chmod +w $out/lib/libprotoc.so.18
+  chmod +w $out/lib/libprotobuf.so.18
+  patchelf --set-rpath $out/lib/:${stdenv.cc.cc.lib}/lib64:${zlib}/lib $out/bin/protoc
+  patchelf --set-rpath $out/lib/:${stdenv.cc.cc.lib}/lib64:${zlib}/lib $out/lib/libprotoc.so.18
+  strip $out/bin/protoc $out/lib/*
+
+  patchelf --shrink-rpath $out/bin/protoc
+  patchelf --shrink-rpath $out/lib/libprotoc.so.18
+  patchelf --shrink-rpath $out/lib/libprotobuf.so.18
+  set +x
+'')