Merge pull request #50 from pachyderm/next

Next

Dockerfile

@@ -9,6 +9,7 @@ RUN go get code.google.com/p/go-uuid/uuid
 RUN go get github.com/samalba/dockerclient
 RUN go get github.com/bitly/go-simplejson
 RUN go get github.com/mitchellh/goamz/...
+RUN go get github.com/go-fsnotify/fsnotify
 ADD . /go/src/$PFS
 RUN go install -race $PFS/services/master && go install $PFS/services/router && go install $PFS/services/webhook && go install $PFS/deploy
 RUN ln $GOPATH/src/$PFS/scripts/pfs-test /usr/local/bin/pfs-test

README.md

@@ -240,12 +240,13 @@ Two guys who love data and communities and both happen to be named Joe. We'd lov
 to chat: joey@pachyderm.io jdoliner@pachyderm.io.
 
 ## How do I hack on pfs?
-You can deploy pfs directly to a CoreOS cluster that's accessible via ssh by running:
+You can run pfs locally using:
 
 ```shell
-scripts/dev-install <coreos-host>
+scripts/dev-launch
 ```
 
-This will deploy pfs and give you a new remote called `staging` so that you can
-push later changes via `git push staging`. The created repo also has a
-post-receive hook that redeploys the cluster.
+This will build a docker image from the working directory, tag it as `pfs` and
+launch it locally using `scripts/launch`.  The only dependencies are Docker >=
+1.5 and btrfs-tools >= 3.14. The script checks for this and gives you
+directions on how to fix it.

deploy/static/dev/announce-master-0-3.service

@@ -4,7 +4,7 @@ PartOf = master-0-3.service
 BindsTo = master-0-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/0-3 '%H:53285' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/0-3 '%H:50620' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/0-3
 
 [X-Fleet]

deploy/static/dev/announce-master-1-3.service

@@ -4,7 +4,7 @@ PartOf = master-1-3.service
 BindsTo = master-1-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/1-3 '%H:59553' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/1-3 '%H:53784' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/1-3
 
 [X-Fleet]

deploy/static/dev/announce-master-2-3.service

@@ -4,7 +4,7 @@ PartOf = master-2-3.service
 BindsTo = master-2-3.service
 
 [Service]
-ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/2-3 '%H:53432' --ttl 60;sleep 45;done"
+ExecStart = /bin/sh -c "while true; do etcdctl set /pfs/master/2-3 '%H:51782' --ttl 60;sleep 45;done"
 ExecStop = /usr/bin/etcdctl rm /pfs/master/2-3
 
 [X-Fleet]

deploy/static/dev/master-0-3.service

@@ -10,9 +10,9 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm master-0-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK`)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img`)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
-ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK` /var/lib/pfs/vol)"
+ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img` /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-0-3 \
@@ -22,7 +22,7 @@ ExecStart = /bin/sh -c "echo $(docker run \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 53285:80 \
+            -p 50620:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 0-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-0-3)"

deploy/static/dev/master-1-3.service

@@ -10,9 +10,9 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm master-1-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK`)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img`)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
-ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK` /var/lib/pfs/vol)"
+ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img` /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-1-3 \
@@ -22,7 +22,7 @@ ExecStart = /bin/sh -c "echo $(docker run \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 59553:80 \
+            -p 53784:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 1-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-1-3)"

deploy/static/dev/master-2-3.service

@@ -10,9 +10,9 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm master-2-3)"
 ExecStartPre = /bin/sh -c "echo $(docker pull `etcdctl get /pfs/registry`/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK`)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img`)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
-ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK` /var/lib/pfs/vol)"
+ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img` /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name master-2-3 \
@@ -22,7 +22,7 @@ ExecStart = /bin/sh -c "echo $(docker run \
             -v /var/run/docker.sock:/var/run/docker.sock \
             -e AWS_ACCESS_KEY_ID=`etcdctl get /pfs/creds/AWS_ACCESS_KEY_ID` \
             -e AWS_SECRET_ACCESS_KEY=`etcdctl get /pfs/creds/AWS_SECRET_ACCESS_KEY` \
-            -p 53432:80 \
+            -p 51782:80 \
             -i `etcdctl get /pfs/registry`/pfs \
             /go/bin/master 2-3)"
 ExecStop = /bin/sh -c "echo $(docker rm -f master-2-3)"

deploy/templates/sharded

@@ -10,9 +10,9 @@ ExecStartPre = -/bin/sh -c "echo $(docker rm {{.Name}}-{{.Shard}}-{{.Nshards}})"
 ExecStartPre = /bin/sh -c "echo $(docker pull {{.Container}})"
 ExecStartPre = -/bin/sh -c "echo $(mkdir /var/lib/pfs)"
 ExecStartPre = -/bin/sh -c "echo $(truncate /var/lib/pfs/data.img -s 10G)"
-ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK`)"
+ExecStartPre = -/bin/sh -c "echo $(mkfs.btrfs `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img`)"
 ExecStartPre = -/bin/sh -c "echo $(mkdir -p /var/lib/pfs/vol)"
-ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK` /var/lib/pfs/vol)"
+ExecStartPre = -/bin/sh -c "echo $(mount `etcdctl get /pfs/config/DISK || echo /var/lib/pfs/data.img` /var/lib/pfs/vol)"
 ExecStart = /bin/sh -c "echo $(docker run \
             --privileged=true \
             --name {{.Name}}-{{.Shard}}-{{.Nshards}} \

lib/btrfs/btrfs.go

@@ -14,9 +14,11 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"code.google.com/p/go-uuid/uuid"
+	"github.com/go-fsnotify/fsnotify"
 	"github.com/pachyderm/pfs/lib/shell"
 )
 
@@ -80,6 +82,10 @@ func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
 	return os.OpenFile(FilePath(name), flag, perm)
 }
 
+func OpenFd(name string, mode int, perm uint32) (int, error) {
+	return syscall.Open(FilePath(name), mode, perm)
+}
+
 func WriteFile(name string, r io.Reader) (int64, error) {
 	f, err := Open(name)
 	if err != nil {
@@ -93,6 +99,10 @@ func Remove(name string) error {
 	return os.Remove(FilePath(name))
 }
 
+func RemoveAll(name string) error {
+	return os.RemoveAll(FilePath(name))
+}
+
 func Rename(oldname, newname string) error {
 	return os.Rename(FilePath(oldname), FilePath(newname))
 }
@@ -159,6 +169,50 @@ func LazyWalk(name string, f func(string) error) error {
 	return nil
 }
 
+func WaitForFile(name string) error {
+	if err := MkdirAll(path.Dir(name)); err != nil {
+		log.Print(err)
+		return err
+	}
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Print(err)
+		return err
+	}
+	defer watcher.Close()
+
+	if err := watcher.Add(FilePath(path.Dir(name))); err != nil {
+		log.Print(err)
+		return err
+	}
+
+	exists, err := FileExists(name)
+	if err != nil {
+		log.Print(err)
+		return err
+	}
+	if exists {
+		log.Print("File exists, returning.")
+		return nil
+	}
+
+	log.Print("File doesn't exist. Waiting for it...")
+	for {
+		select {
+		case event := <-watcher.Events:
+			log.Print(event)
+			if event.Op == fsnotify.Create && event.Name == FilePath(name) {
+				return nil
+			}
+		case err := <-watcher.Errors:
+			log.Print(err)
+			return err
+		}
+	}
+	return nil
+}
+
 func SubvolumeCreate(name string) error {
 	return shell.RunStderr(exec.Command("btrfs", "subvolume", "create", FilePath(name)))
 }