Merge pull request #42 from pachyderm/next

v0.4.1 Fix a few things in dev/install

Dockerfile

@@ -1,9 +1,9 @@
-FROM ubuntu:14.10
+FROM ubuntu:15.04
 
 ENV GOPATH /go
 ENV PFS github.com/pachyderm/pfs
 
-RUN apt-get update && apt-get install -y golang btrfs-tools git mercurial && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y golang git mercurial && rm -rf /var/lib/apt/lists/*
 RUN go get github.com/coreos/go-etcd/etcd
 RUN go get code.google.com/p/go-uuid/uuid
 RUN go get github.com/samalba/dockerclient
@@ -13,5 +13,6 @@ ADD . /go/src/$PFS
 RUN go install -race $PFS/services/master && go install $PFS/services/router && go install $PFS/services/webhook && go install $PFS/deploy
 RUN ln $GOPATH/src/$PFS/scripts/pfs-test /usr/local/bin/pfs-test
 RUN ln $GOPATH/src/$PFS/scripts/pfs-bench /usr/local/bin/pfs-bench
+RUN ln $GOPATH/src/$PFS/scripts/btrfs-wrapper /usr/sbin/btrfs
 
 EXPOSE 80

deploy/static/dev/announce-master-0-3.service

@@ -4,7 +4,7 @@ PartOf = master-0-3.service
 BindsTo = master-0-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/0-3 '%H:60321' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/0-3 '%H:62871' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/0-3
 
 [X-Fleet]

deploy/static/dev/announce-master-1-3.service

@@ -4,7 +4,7 @@ PartOf = master-1-3.service
 BindsTo = master-1-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/1-3 '%H:49957' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/1-3 '%H:60711' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/1-3
 
 [X-Fleet]

deploy/static/dev/announce-master-2-3.service

@@ -4,7 +4,7 @@ PartOf = master-2-3.service
 BindsTo = master-2-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/2-3 '%H:55672' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/2-3 '%H:58281' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/2-3
 
 [X-Fleet]

deploy/static/dev/master-0-3.service

@@ -9,18 +9,20 @@ ExecStartPre = -/bin/sh -c "echo $(docker kill master-0-3)"
 ExecStartPre = -/bin/sh -c "echo $(docker rm master-0-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs mkfs.btrfs /var/lib/pfs/data.img)"
+ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs /var/lib/pfs/data.img)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
 ExecStartPre = -/bin/sh -c "echo $(mount /var/lib/pfs/data.img /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-0-3 \
             -v /var/lib/pfs:/var/lib/pfs \
             -v /var/run/docker.sock:/var/run/docker.sock \
+            -v /usr/sbin/btrfs:/usr/sbin/btrfs.host \
+            -v /lib64:/lib64.host \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 60321:80 \
+            -p 62871:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 0-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-0-3)"

deploy/static/dev/master-1-3.service

@@ -9,18 +9,20 @@ ExecStartPre = -/bin/sh -c "echo $(docker kill master-1-3)"
 ExecStartPre = -/bin/sh -c "echo $(docker rm master-1-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs mkfs.btrfs /var/lib/pfs/data.img)"
+ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs /var/lib/pfs/data.img)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
 ExecStartPre = -/bin/sh -c "echo $(mount /var/lib/pfs/data.img /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-1-3 \
             -v /var/lib/pfs:/var/lib/pfs \
             -v /var/run/docker.sock:/var/run/docker.sock \
+            -v /usr/sbin/btrfs:/usr/sbin/btrfs.host \
+            -v /lib64:/lib64.host \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 49957:80 \
+            -p 60711:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 1-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-1-3)"

deploy/static/dev/master-2-3.service

@@ -9,18 +9,20 @@ ExecStartPre = -/bin/sh -c "echo $(docker kill master-2-3)"
 ExecStartPre = -/bin/sh -c "echo $(docker rm master-2-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i `etcdctl get /pfs/registry`/pfs mkfs.btrfs /var/lib/pfs/data.img)"
+ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs /var/lib/pfs/data.img)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
 ExecStartPre = -/bin/sh -c "echo $(mount /var/lib/pfs/data.img /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-2-3 \
             -v /var/lib/pfs:/var/lib/pfs \
             -v /var/run/docker.sock:/var/run/docker.sock \
+            -v /usr/sbin/btrfs:/usr/sbin/btrfs.host \
+            -v /lib64:/lib64.host \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 55672:80 \
+            -p 58281:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 2-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-2-3)"

deploy/static/dev/registry.service

@@ -9,7 +9,7 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm registry)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --name registry \
             -e SETTINGS_FLAVOR=s3 \
-            -e AWS_BUCKET=`etcdctl get /pfs/registry/IMAGE_BUCKET` \
+            -e AWS_BUCKET=`etcdctl get /pfs/creds/IMAGE_BUCKET` \
             -e STORAGE_PATH=/registry \
             -e AWS_KEY=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \

deploy/templates/registry

@@ -9,7 +9,7 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm {{.Name}})"
 ExecStart = /bin/sh -c "echo $(docker run \
             --name registry \
             -e SETTINGS_FLAVOR=s3 \
-            -e AWS_BUCKET=`etcdctl get /pfs/registry/IMAGE_BUCKET` \
+            -e AWS_BUCKET=`etcdctl get /pfs/creds/IMAGE_BUCKET` \
             -e STORAGE_PATH=/registry \
             -e AWS_KEY=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \

deploy/templates/sharded

@@ -9,15 +9,17 @@ ExecStartPre = -/bin/sh -c "echo $(docker kill {{.Name}}-{{.Shard}}-{{.Nshards}}
 ExecStartPre = -/bin/sh -c "echo $(docker rm {{.Name}}-{{.Shard}}-{{.Nshards}})"
 ExecStartPre = /bin/sh -c "echo $(docker pull {{.Container}})"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i {{.Container}} truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(docker run --privileged=true -v /var/lib/pfs:/var/lib/pfs -i {{.Container}} mkfs.btrfs /var/lib/pfs/data.img)"
+ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs /var/lib/pfs/data.img)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
 ExecStartPre = -/bin/sh -c "echo $(mount /var/lib/pfs/data.img /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name {{.Name}}-{{.Shard}}-{{.Nshards}} \
             -v /var/lib/pfs:/var/lib/pfs \
             -v /var/run/docker.sock:/var/run/docker.sock \
+            -v /usr/sbin/btrfs:/usr/sbin/btrfs.host \
+            -v /lib64:/lib64.host \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
             -p {{.Port}}:80 \

lib/mapreduce/mapreduce.go

@@ -13,8 +13,6 @@ import (
 	"sync"
 	"time"
 
-	"testing/iotest"
-
 	"github.com/mitchellh/goamz/aws"
 	"github.com/mitchellh/goamz/s3"
 	"github.com/pachyderm/pfs/lib/btrfs"
@@ -92,11 +90,12 @@ func contains(set []string, val string) bool {
 }
 
 type Job struct {
-	Type    string   `json:"type"`
-	Input   string   `json:"input"`
-	Image   string   `json:"image"`
-	Command []string `json:"command"`
-	Limit   int      `json:"limit"`
+	Type     string   `json:"type"`
+	Input    string   `json:"input"`
+	Image    string   `json:"image"`
+	Command  []string `json:"command"`
+	Limit    int      `json:"limit"`
+	Parallel int      `json:"parallel"`
 }
 
 type materializeInfo struct {
@@ -178,8 +177,13 @@ func Map(job Job, jobPath string, m materializeInfo, host string, shard, modulos
 	defer wg.Wait()
 	client := &http.Client{}
 
+	nGoros := 300
+	if job.Parallel > 0 {
+		nGoros = job.Parallel
+	}
+
 	defer close(files)
-	for i := 0; i < 300; i++ {
+	for i := 0; i < nGoros; i++ {
 		wg.Add(1)
 		go func(wg *sync.WaitGroup) {
 			defer wg.Done()
@@ -200,28 +204,26 @@ func Map(job Job, jobPath string, m materializeInfo, host string, shard, modulos
 					return
 				}
 				defer inFile.Close()
-				inReader := iotest.NewReadLogger("MapIn", inFile)
 
 				var resp *http.Response
 				err = retry(func() error {
 					log.Print("Posting: ", "http://"+path.Join(host, name))
-					resp, err = client.Post("http://"+path.Join(host, name), "application/text", inReader)
+					resp, err = client.Post("http://"+path.Join(host, name), "application/text", inFile)
 					return err
 				}, retries, 200*time.Millisecond)
 				if err != nil {
 					log.Print(err)
 					return
 				}
 				defer resp.Body.Close()
-				outReader := iotest.NewReadLogger("MapOut", resp.Body)
 
 				outFile, err := btrfs.CreateAll(path.Join(m.Out, m.Branch, jobPath, name))
 				if err != nil {
 					log.Print(err)
 					return
 				}
 				defer outFile.Close()
-				if _, err := io.Copy(outFile, outReader); err != nil {
+				if _, err := io.Copy(outFile, resp.Body); err != nil {
 					log.Print(err)
 					return
 				}
@@ -297,21 +299,20 @@ func Reduce(job Job, jobPath string, m materializeInfo, host string, shard, modu
 
 	// Notice we're just passing "host" here. Multicast will fill in the host
 	// field so we don't actually need to specify it.
-	var _reader io.ReadCloser
+	var reader io.ReadCloser
 	err := retry(func() error {
 		req, err := http.NewRequest("GET", "http://host/"+path.Join(job.Input, "file", "*")+"?commit="+m.Commit, nil)
 		if err != nil {
 			return err
 		}
-		_reader, err = route.Multicast(req, "/pfs/master")
+		reader, err = route.Multicast(req, "/pfs/master")
 		return err
 	}, retries, time.Minute)
 	if err != nil {
 		log.Print(err)
 		return
 	}
-	defer _reader.Close()
-	reader := iotest.NewReadLogger("Reduce", _reader)
+	defer reader.Close()
 
 	var resp *http.Response
 	err = retry(func() error {

scripts/btrfs-wrapper

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# This script gets injected in to the docker container as /usr/sbin/btrfs
+
+LD_LIBRARY_PATH=/lib64.host /usr/sbin/btrfs.host $@

scripts/dev-install

@@ -2,7 +2,7 @@
 # Install a git repo on a pfs cluster.
 ###############################################################################
 set -E          # pass trap handlers down to subshells
-#set -x         # execution tracing debug messages
+set -x         # execution tracing debug messages
 
 USAGE="Usage: $0 host"
 
@@ -27,10 +27,12 @@ scp -rqC "$CMD_DIR"/dev-post-receive core@"$REMOTE"/hooks/post-receive
 git remote remove staging
 git remote add staging git://"$HOST"/home/core/"$REPO".git
 
+BUCKET=${IMAGE_BUCKET:-"pachyderm-images"}
+
 # Do an initial run with the post-receive hook to set everything up.
-ssh core@"$HOST" "etcdctl set /pfs/creds/AWS_ACCESS_KEY_ID $AWS_ACCESS_KEY_ID"
-ssh core@"$HOST" "etcdctl set /pfs/creds/AWS_SECRET_ACCESS_KEY $AWS_SECRET_ACCESS_KEY"
-ssh core@"$HOST" "etcdctl set /pfs/registry/IMAGE_BUCKET $IMAGE_BUCKET"
+ssh core@"$HOST" etcdctl set /pfs/creds/AWS_ACCESS_KEY_ID ${AWS_ACCESS_KEY_ID:?"Env: AWS_ACCESS_KEY_ID must be set"}
+ssh core@"$HOST" etcdctl set /pfs/creds/AWS_SECRET_ACCESS_KEY ${AWS_SECRET_ACCESS_KEY:?"Env: AWS_SECRET_ACCESS_KEY must be set"}
+ssh core@"$HOST" etcdctl set /pfs/creds/IMAGE_BUCKET "$BUCKET"
 ssh core@"$HOST" 'cd pfs.git && hooks/post-receive'
 
 echo "Done."