diff --git a/vendor/k8s.io/kubernetes/pkg/api/validation/validation.go b/vendor/k8s.io/kubernetes/pkg/api/validation/validation.go index 33085ce6e198..1fbe7aec996f 100644 --- a/vendor/k8s.io/kubernetes/pkg/api/validation/validation.go +++ b/vendor/k8s.io/kubernetes/pkg/api/validation/validation.go @@ -1781,7 +1781,11 @@ func ValidateVolumeMounts(mounts []api.VolumeMount, volumes sets.String, fldPath } mountpoints.Insert(mnt.MountPath) if len(mnt.SubPath) > 0 { - allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...) + if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("subPath"), "subPath is disabled by feature-gate")) + } else { + allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...) + } } } return allErrs diff --git a/vendor/k8s.io/kubernetes/pkg/api/validation/validation_test.go b/vendor/k8s.io/kubernetes/pkg/api/validation/validation_test.go index a9cb2cb496c4..949e0bbbba77 100644 --- a/vendor/k8s.io/kubernetes/pkg/api/validation/validation_test.go +++ b/vendor/k8s.io/kubernetes/pkg/api/validation/validation_test.go @@ -10144,3 +10144,56 @@ func TestValidateFlexVolumeSource(t *testing.T) { } } } + +func TestValidateDisabledSubpath(t *testing.T) { + utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false") + defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true") + + volumes := []api.Volume{ + {Name: "abc", VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}}, + {Name: "abc-123", VolumeSource: api.VolumeSource{PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}}, + {Name: "123", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/foo/baz"}}}, + } + vols, v1err := ValidateVolumes(volumes, field.NewPath("field")) + if len(v1err) > 0 { + t.Errorf("Invalid test volume - expected success %v", v1err) + return + } + + cases := map[string]struct { + mounts []api.VolumeMount + expectError bool + }{ + "subpath not specified": { + []api.VolumeMount{ + { + Name: "abc-123", + MountPath: "/bab", + }, + }, + false, + }, + "subpath specified": { + []api.VolumeMount{ + { + Name: "abc-123", + MountPath: "/bab", + SubPath: "baz", + }, + }, + true, + }, + } + + for name, test := range cases { + errs := ValidateVolumeMounts(test.mounts, vols, field.NewPath("field")) + + if len(errs) != 0 && !test.expectError { + t.Errorf("test %v failed: %+v", name, errs) + } + + if len(errs) == 0 && test.expectError { + t.Errorf("test %v failed, expected error", name) + } + } +} diff --git a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go index 7920d4bc5cf3..01bc036e2c5c 100644 --- a/vendor/k8s.io/kubernetes/pkg/features/kube_features.go +++ b/vendor/k8s.io/kubernetes/pkg/features/kube_features.go @@ -115,6 +115,20 @@ const ( // // New local storage types to support local storage capacity isolation LocalStorageCapacityIsolation utilfeature.Feature = "LocalStorageCapacityIsolation" + + // owner: @saad-ali + // ga + // + // Allow mounting a subpath of a volume in a container + // Do not remove this feature gate even though it's GA + VolumeSubpath utilfeature.Feature = "VolumeSubpath" + + // owner: @joelsmith + // deprecated: v1.10 + // + // Mount secret, configMap, downwardAPI and projected volumes ReadOnly. Note: this feature + // gate is present only for backward compatability, it will be removed in the 1.11 release. + ReadOnlyAPIDataVolumes utilfeature.Feature = "ReadOnlyAPIDataVolumes" ) func init() { @@ -138,9 +152,13 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS RotateKubeletClientCertificate: {Default: false, PreRelease: utilfeature.Alpha}, PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha}, LocalStorageCapacityIsolation: {Default: false, PreRelease: utilfeature.Alpha}, + VolumeSubpath: {Default: true, PreRelease: utilfeature.GA}, // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side: StreamingProxyRedirects: {Default: true, PreRelease: utilfeature.Beta}, genericfeatures.AdvancedAuditing: {Default: false, PreRelease: utilfeature.Alpha}, + + // features that enable backwards compatability but are scheduled to be removed + ReadOnlyAPIDataVolumes: {Default: true, PreRelease: utilfeature.GA}, } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/BUILD index 05898c6b1d1b..f52e51dabef6 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/BUILD @@ -227,6 +227,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library", "//vendor/k8s.io/client-go/rest:go_default_library", "//vendor/k8s.io/client-go/testing:go_default_library", diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux_test.go index d1cba37ae472..efaf3b3294be 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux_test.go @@ -75,6 +75,18 @@ func (mi *fakeMountInterface) PathIsDevice(pathname string) (bool, error) { return true, nil } +func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) { + return "", nil, nil +} + +func (mi *fakeMountInterface) CleanSubPaths(_, _ string) error { + return nil +} + +func (mi *fakeMountInterface) SafeMakeDir(_, _ string, _ os.FileMode) error { + return nil +} + func fakeContainerMgrMountInt() mount.Interface { return &fakeMountInterface{ []mount.MountPoint{ diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go b/vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go index 7930fed552bd..bccf1ace0193 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/container/helpers.go @@ -48,7 +48,7 @@ type HandlerRunner interface { // RuntimeHelper wraps kubelet to make container runtime // able to get necessary informations like the RunContainerOptions, DNS settings, Host IP. type RuntimeHelper interface { - GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, useClusterFirstPolicy bool, err error) + GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, useClusterFirstPolicy bool, cleanupAction func(), err error) GetClusterDNS(pod *v1.Pod) (dnsServers []string, dnsSearches []string, useClusterFirstPolicy bool, err error) // GetPodCgroupParent returns the the CgroupName identifer, and its literal cgroupfs form on the host // of a pod. diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/container/testing/fake_runtime_helper.go b/vendor/k8s.io/kubernetes/pkg/kubelet/container/testing/fake_runtime_helper.go index 76bbd0a4bc37..703ca80953b2 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/container/testing/fake_runtime_helper.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/container/testing/fake_runtime_helper.go @@ -32,12 +32,12 @@ type FakeRuntimeHelper struct { Err error } -func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { +func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, func(), error) { var opts kubecontainer.RunContainerOptions if len(container.TerminationMessagePath) != 0 { opts.PodContainerDir = f.PodContainerDir } - return &opts, false, nil + return &opts, false, nil, nil } func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) string { diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go b/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go index 162030fe5baa..39b1672a1073 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods.go @@ -40,6 +40,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/v1" @@ -48,6 +49,7 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/api/v1/resource" "k8s.io/kubernetes/pkg/api/v1/validation" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/fieldpath" "k8s.io/kubernetes/pkg/kubelet/cm" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -60,6 +62,7 @@ import ( kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/util" + mountutil "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/util/volumehelper" volumevalidation "k8s.io/kubernetes/pkg/volume/validation" @@ -109,7 +112,7 @@ func (kl *Kubelet) makeDevices(pod *v1.Pod, container *v1.Container) ([]kubecont } // makeMounts determines the mount points for the given container. -func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) { +func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface) ([]kubecontainer.Mount, func(), error) { // Kubernetes only mounts on /etc/hosts if : // - container does not use hostNetwork and // - container is not an infrastructure(pause) container @@ -119,7 +122,8 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h mountEtcHostsFile := !pod.Spec.HostNetwork && len(podIP) > 0 && runtime.GOOS != "windows" glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile) mounts := []kubecontainer.Mount{} - for _, mount := range container.VolumeMounts { + var cleanupAction func() = nil + for i, mount := range container.VolumeMounts { mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath) vol, ok := podVolumes[mount.Name] if !ok || vol.Mounter == nil { @@ -137,25 +141,33 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h } hostPath, err := volume.GetPath(vol.Mounter) if err != nil { - return nil, err + return nil, cleanupAction, err } if mount.SubPath != "" { + if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) { + return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled") + } + if filepath.IsAbs(mount.SubPath) { - return nil, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath) + return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath) } err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath) if err != nil { - return nil, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err) + return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err) } fileinfo, err := os.Lstat(hostPath) if err != nil { - return nil, err + return nil, cleanupAction, err } perm := fileinfo.Mode() - hostPath = filepath.Join(hostPath, mount.SubPath) + volumePath, err := filepath.EvalSymlinks(hostPath) + if err != nil { + return nil, cleanupAction, err + } + hostPath = filepath.Join(volumePath, mount.SubPath) if subPathExists, err := util.FileOrSymlinkExists(hostPath); err != nil { glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath) @@ -164,17 +176,25 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h // incorrect ownership and mode. For example, the sub path directory must have at least g+rwx // when the pod specifies an fsGroup, and if the directory is not created here, Docker will // later auto-create it with the incorrect mode 0750 - if err := os.MkdirAll(hostPath, perm); err != nil { - glog.Errorf("failed to mkdir:%s", hostPath) - return nil, err - } - - // chmod the sub path because umask may have prevented us from making the sub path with the same - // permissions as the mounter path - if err := os.Chmod(hostPath, perm); err != nil { - return nil, err + // Make extra care not to escape the volume! + if err := mounter.SafeMakeDir(hostPath, volumePath, perm); err != nil { + glog.Errorf("failed to mkdir %q: %v", hostPath, err) + return nil, cleanupAction, err } } + hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{ + VolumeMountIndex: i, + Path: hostPath, + VolumeName: mount.Name, + VolumePath: volumePath, + PodDir: podDir, + ContainerName: container.Name, + }) + if err != nil { + // Don't pass detailed error back to the user because it could give information about host filesystem + glog.Errorf("failed to prepare subPath for volumeMount %q of container %q: %v", mount.Name, container.Name, err) + return nil, cleanupAction, fmt.Errorf("failed to prepare subPath for volumeMount %q of container %q", mount.Name, container.Name) + } } // Docker Volume Mounts fail on Windows if it is not of the form C:/ @@ -188,11 +208,13 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h } } + mustMountRO := vol.Mounter.GetAttributes().ReadOnly && utilfeature.DefaultFeatureGate.Enabled(features.ReadOnlyAPIDataVolumes) + mounts = append(mounts, kubecontainer.Mount{ Name: mount.Name, ContainerPath: containerPath, HostPath: hostPath, - ReadOnly: mount.ReadOnly, + ReadOnly: mount.ReadOnly || mustMountRO, SELinuxRelabel: relabelVolume, }) } @@ -200,11 +222,11 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h hostAliases := pod.Spec.HostAliases hostsMount, err := makeHostsMount(podDir, podIP, hostName, hostDomain, hostAliases) if err != nil { - return nil, err + return nil, cleanupAction, err } mounts = append(mounts, *hostsMount) } - return mounts, nil + return mounts, cleanupAction, nil } // makeHostsMount makes the mountpoint for the hosts file that the containers @@ -311,14 +333,14 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) string { // GenerateRunContainerOptions generates the RunContainerOptions, which can be used by // the container runtime to set parameters for launching a container. -func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, error) { +func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, bool, func(), error) { var err error useClusterFirstPolicy := false cgroupParent := kl.GetPodCgroupParent(pod) opts := &kubecontainer.RunContainerOptions{CgroupParent: cgroupParent} hostname, hostDomainName, err := kl.GeneratePodHostNameAndDomain(pod) if err != nil { - return nil, false, err + return nil, false, nil, err } opts.Hostname = hostname podName := volumehelper.GetUniquePodName(pod) @@ -328,16 +350,17 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai // TODO(random-liu): Move following convert functions into pkg/kubelet/container opts.Devices, err = kl.makeDevices(pod, container) if err != nil { - return nil, false, err + return nil, false, nil, err } - opts.Mounts, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes) + var cleanupAction func() + opts.Mounts, cleanupAction, err = makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter) if err != nil { - return nil, false, err + return nil, false, cleanupAction, err } opts.Envs, err = kl.makeEnvironmentVariables(pod, container, podIP) if err != nil { - return nil, false, err + return nil, false, cleanupAction, err } // Disabling adding TerminationMessagePath on Windows as these files would be mounted as docker volume and @@ -353,7 +376,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai opts.DNS, opts.DNSSearch, useClusterFirstPolicy, err = kl.GetClusterDNS(pod) if err != nil { - return nil, false, err + return nil, false, cleanupAction, err } // only do this check if the experimental behavior is enabled, otherwise allow it to default to false @@ -361,7 +384,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai opts.EnableHostUserNamespace = kl.enableHostUserNamespace(pod) } - return opts, useClusterFirstPolicy, nil + return opts, useClusterFirstPolicy, cleanupAction, nil } var masterServices = sets.NewString("kubernetes") diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods_test.go index ef3cb60cb3b5..ab47825b108c 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/kubelet_pods_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" "k8s.io/kubernetes/pkg/api" @@ -39,6 +40,7 @@ import ( containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" "k8s.io/kubernetes/pkg/kubelet/server/portforward" "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" + "k8s.io/kubernetes/pkg/util/mount" ) func TestMakeMounts(t *testing.T) { @@ -149,13 +151,14 @@ func TestMakeMounts(t *testing.T) { for name, tc := range testCases { t.Run(name, func(t *testing.T) { + fm := &mount.FakeMounter{} pod := v1.Pod{ Spec: v1.PodSpec{ HostNetwork: true, }, } - mounts, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes) + mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm) // validate only the error if we expect an error if tc.expectErr { @@ -338,7 +341,7 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) { options := make([]*kubecontainer.RunContainerOptions, 4) for i, pod := range pods { var err error - options[i], _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") + options[i], _, _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") if err != nil { t.Fatalf("failed to generate container options: %v", err) } @@ -371,7 +374,7 @@ func TestGenerateRunContainerOptions_DNSConfigurationParams(t *testing.T) { kubelet.resolverConfig = "/etc/resolv.conf" for i, pod := range pods { var err error - options[i], _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") + options[i], _, _, err = kubelet.GenerateRunContainerOptions(pod, &v1.Container{}, "") if err != nil { t.Fatalf("failed to generate container options: %v", err) } @@ -2143,3 +2146,59 @@ func TestTruncatePodHostname(t *testing.T) { assert.Equal(t, test.output, output) } } + +func TestDisabledSubpath(t *testing.T) { + fm := &mount.FakeMounter{} + pod := v1.Pod{ + Spec: v1.PodSpec{ + HostNetwork: true, + }, + } + podVolumes := kubecontainer.VolumeMap{ + "disk": kubecontainer.VolumeInfo{Mounter: &stubVolume{path: "/mnt/disk"}}, + } + + cases := map[string]struct { + container v1.Container + expectError bool + }{ + "subpath not specified": { + v1.Container{ + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/mnt/path3", + Name: "disk", + ReadOnly: true, + }, + }, + }, + false, + }, + "subpath specified": { + v1.Container{ + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "/mnt/path3", + SubPath: "/must/not/be/absolute", + Name: "disk", + ReadOnly: true, + }, + }, + }, + true, + }, + } + + utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false") + defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true") + + for name, test := range cases { + _, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm) + if err != nil && !test.expectError { + t.Errorf("test %v failed: %v", name, err) + } + if err == nil && test.expectError { + t.Errorf("test %v failed: expected error", name) + } + } +} diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go index 76fbe3e123fa..81ddd7b1c10b 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -107,7 +107,10 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb restartCount = containerStatus.RestartCount + 1 } - containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef) + containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef) + if cleanupAction != nil { + defer cleanupAction() + } if err != nil { m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err)) return grpc.ErrorDesc(err), ErrCreateContainerConfig @@ -165,20 +168,20 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb } // generateContainerConfig generates container config for kubelet runtime v1. -func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string) (*runtimeapi.ContainerConfig, error) { - opts, _, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) +func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string) (*runtimeapi.ContainerConfig, func(), error) { + opts, _, cleanupAction, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) if err != nil { - return nil, err + return nil, nil, err } uid, username, err := m.getImageUser(container.Image) if err != nil { - return nil, err + return nil, cleanupAction, err } // Verify RunAsNonRoot. Non-root verification only supports numeric user. if err := verifyRunAsNonRoot(pod, container, uid, username); err != nil { - return nil, err + return nil, cleanupAction, err } command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs) @@ -215,7 +218,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai } config.Envs = envs - return config, nil + return config, cleanupAction, nil } // generateLinuxContainerConfig generates linux container config for kubelet runtime v1. diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container_test.go index f00dc4d7a680..59b230db490b 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_container_test.go @@ -170,7 +170,7 @@ func makeExpetectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIn container := &pod.Spec.Containers[containerIndex] podIP := "" restartCount := 0 - opts, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) + opts, _, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP) containerLogsPath := buildContainerLogsPath(container.Name, restartCount) restartCountUint32 := uint32(restartCount) envs := make([]*runtimeapi.KeyValue, len(opts.Envs)) @@ -222,7 +222,7 @@ func TestGenerateContainerConfig(t *testing.T) { } expectedConfig := makeExpetectedConfig(m, pod, 0) - containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image) + containerConfig, _, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image) assert.NoError(t, err) assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.") @@ -252,7 +252,7 @@ func TestGenerateContainerConfig(t *testing.T) { } expectedConfig = makeExpetectedConfig(m, podWithContainerSecurityContext, 0) - containerConfig, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + containerConfig, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) assert.Error(t, err) imageId, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil) @@ -264,6 +264,6 @@ func TestGenerateContainerConfig(t *testing.T) { podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue - _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) + _, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image) assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username") } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go index f7655ea10a3b..b7bd99317ded 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/kuberuntime/kuberuntime_manager_test.go @@ -142,7 +142,7 @@ func makeFakeContainer(t *testing.T, m *kubeGenericRuntimeManager, template cont sandboxConfig, err := m.generatePodSandboxConfig(template.pod, template.sandboxAttempt) assert.NoError(t, err, "generatePodSandboxConfig for container template %+v", template) - containerConfig, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image) + containerConfig, _, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image) assert.NoError(t, err, "generateContainerConfig for container template %+v", template) podSandboxID := apitest.BuildSandboxName(sandboxConfig.Metadata) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/rkt/rkt.go b/vendor/k8s.io/kubernetes/pkg/kubelet/rkt/rkt.go index a366950675a3..bc85e6149a54 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/rkt/rkt.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/rkt/rkt.go @@ -832,7 +832,7 @@ func (r *Runtime) newAppcRuntimeApp(pod *v1.Pod, podIP string, c v1.Container, r } // TODO: determine how this should be handled for rkt - opts, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) + opts, _, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP) if err != nil { return err } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/util/util_unsupported.go b/vendor/k8s.io/kubernetes/pkg/kubelet/util/util_unsupported.go index 616aabfe3b3b..a08abb276156 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/util/util_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/util/util_unsupported.go @@ -31,3 +31,12 @@ func CreateListener(endpoint string) (net.Listener, error) { func GetAddressAndDialer(endpoint string) (string, func(addr string, timeout time.Duration) (net.Conn, error), error) { return "", nil, fmt.Errorf("GetAddressAndDialer is unsupported in this build") } + +// LockAndCheckSubPath empty implementation +func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) { + return []uintptr{}, nil +} + +// UnlockPath empty implementation +func UnlockPath(fileHandles []uintptr) { +} diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler/reconciler.go b/vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler/reconciler.go index b2c2ea3fcdd1..2b69d446c85c 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler/reconciler.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/volumemanager/reconciler/reconciler.go @@ -174,7 +174,7 @@ func (rc *reconciler) reconcile() { // Volume is mounted, unmount it glog.V(12).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", "")) err := rc.operationExecutor.UnmountVolume( - mountedVolume.MountedVolume, rc.actualStateOfWorld) + mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir) if err != nil && !nestedpendingoperations.IsAlreadyExists(err) && !exponentialbackoff.IsExponentialBackoff(err) { diff --git a/vendor/k8s.io/kubernetes/pkg/util/io/BUILD b/vendor/k8s.io/kubernetes/pkg/util/io/BUILD index def8f7bee258..e456b74f19c7 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/io/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/util/io/BUILD @@ -11,6 +11,7 @@ load( go_library( name = "go_default_library", srcs = [ + "consistentread.go", "io.go", "writer.go", ], diff --git a/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go b/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go new file mode 100644 index 000000000000..6e1f17b09855 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/util/io/consistentread.go @@ -0,0 +1,45 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package io + +import ( + "bytes" + "fmt" + "io/ioutil" +) + +// ConsistentRead repeatedly reads a file until it gets the same content twice. +// This is useful when reading files in /proc that are larger than page size +// and kernel may modify them between individual read() syscalls. +func ConsistentRead(filename string, attempts int) ([]byte, error) { + oldContent, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + for i := 0; i < attempts; i++ { + newContent, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + if bytes.Compare(oldContent, newContent) == 0 { + return newContent, nil + } + // Files are different, continue reading + oldContent = newContent + } + return nil, fmt.Errorf("could not get consistent content of %s after %d attempts", filename, attempts) +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/BUILD b/vendor/k8s.io/kubernetes/pkg/util/mount/BUILD index 3e5a2d7203a5..47cfa0160f93 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/BUILD @@ -20,6 +20,7 @@ go_library( tags = ["automanaged"], deps = [ "//pkg/util/exec:go_default_library", + "//pkg/util/io:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", ], @@ -34,7 +35,10 @@ go_test( ], library = ":go_default_library", tags = ["automanaged"], - deps = ["//pkg/util/exec:go_default_library"], + deps = [ + "//pkg/util/exec:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + ], ) filegroup( diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/fake.go b/vendor/k8s.io/kubernetes/pkg/util/mount/fake.go index 2b71fa0a7285..2ad5f0f3af03 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/fake.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/fake.go @@ -17,6 +17,7 @@ limitations under the License. package mount import ( + "os" "path/filepath" "sync" @@ -171,3 +172,14 @@ func (f *FakeMounter) PathIsDevice(pathname string) (bool, error) { func (f *FakeMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { return getDeviceNameFromMount(f, mountPath, pluginDir) } + +func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, nil +} + +func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} +func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/mount.go b/vendor/k8s.io/kubernetes/pkg/util/mount/mount.go index 4bb1c1d453dd..f8b0db047465 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/mount.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/mount.go @@ -20,9 +20,11 @@ package mount import ( "fmt" + "os" "path" "path/filepath" "strings" + "syscall" "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/exec" @@ -68,6 +70,42 @@ type Interface interface { // GetDeviceNameFromMount finds the device name by checking the mount path // to get the global mount path which matches its plugin directory GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) + // SafeMakeDir makes sure that the created directory does not escape given + // base directory mis-using symlinks. The directory is created in the same + // mount namespace as where kubelet is running. Note that the function makes + // sure that it creates the directory somewhere under the base, nothing + // else. E.g. if the directory already exists, it may exists outside of the + // base due to symlinks. + SafeMakeDir(pathname string, base string, perm os.FileMode) error + // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given + // pod volume directory. + CleanSubPaths(podDir string, volumeName string) error + // PrepareSafeSubpath does everything that's necessary to prepare a subPath + // that's 1) inside given volumePath and 2) immutable after this call. + // + // newHostPath - location of prepared subPath. It should be used instead of + // hostName when running the container. + // cleanupAction - action to run when the container is running or it failed to start. + // + // CleanupAction must be called immediately after the container with given + // subpath starts. On the other hand, Interface.CleanSubPaths must be called + // when the pod finishes. + PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) +} + +type Subpath struct { + // index of the VolumeMount for this container + VolumeMountIndex int + // Full path to the subpath directory on the host + Path string + // name of the volume that is a valid directory name. + VolumeName string + // Full path to the volume path + VolumePath string + // Path to the pod's directory, including pod UID + PodDir string + // Name of the container + ContainerName string } // Compile-time check to ensure all Mounter implementations satisfy @@ -204,6 +242,13 @@ func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (str return path.Base(mountPath), nil } +func isNotDirErr(err error) bool { + if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOTDIR { + return true + } + return false +} + // IsNotMountPoint determines if a directory is a mountpoint. // It should return ErrNotExist when the directory does not exist. // This method uses the List() of all mountpoints @@ -213,7 +258,13 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) { // IsLikelyNotMountPoint provides a quick check // to determine whether file IS A mountpoint notMnt, notMntErr := mounter.IsLikelyNotMountPoint(file) - if notMntErr != nil { + if notMntErr != nil && os.IsPermission(notMntErr) { + // We were not allowed to do the simple stat() check, e.g. on NFS with + // root_squash. Fall back to /proc/mounts check below. + notMnt = true + notMntErr = nil + } + if notMntErr != nil && isNotDirErr(notMntErr) { return notMnt, notMntErr } // identified as mountpoint, so return this fact @@ -234,3 +285,22 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) { } return notMnt, nil } + +// pathWithinBase checks if give path is withing given base directory. +func pathWithinBase(fullPath, basePath string) bool { + rel, err := filepath.Rel(basePath, fullPath) + if err != nil { + return false + } + if startsWithBackstep(rel) { + // Needed to escape the base path + return false + } + return true +} + +// startsWithBackstep checks if the given path starts with a backstep segment +func startsWithBackstep(rel string) bool { + // normalize to / and check for ../ + return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../") +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go index ae7709d471b6..5170868790c9 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go @@ -23,8 +23,10 @@ import ( "fmt" "hash/fnv" "io" + "io/ioutil" "os" "os/exec" + "path/filepath" "strconv" "strings" "syscall" @@ -32,6 +34,7 @@ import ( "github.com/golang/glog" "k8s.io/apimachinery/pkg/util/sets" utilexec "k8s.io/kubernetes/pkg/util/exec" + utilio "k8s.io/kubernetes/pkg/util/io" ) const ( @@ -48,6 +51,11 @@ const ( fsckErrorsCorrected = 1 // 'fsck' found errors but exited without correcting them fsckErrorsUncorrected = 4 + + // place for subpath mounts + containerSubPathDirectoryName = "volume-subpaths" + // syscall.Openat flags used to traverse directories not following symlinks + nofollowFlags = syscall.O_RDONLY | syscall.O_NOFOLLOW ) // Mounter provides the default implementation of mount.Interface @@ -255,7 +263,7 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) { if err != nil { return true, err } - rootStat, err := os.Lstat(file + "/..") + rootStat, err := os.Lstat(filepath.Dir(strings.TrimSuffix(file, "/"))) if err != nil { return true, err } @@ -371,6 +379,7 @@ func readProcMountsFrom(file io.Reader, out *[]MountPoint) (uint32, error) { if err == io.EOF { break } + // See `man proc` for authoritative description of format of the file. fields := strings.Fields(line) if len(fields) != expectedNumFieldsPerLine { return 0, fmt.Errorf("wrong number of fields (expected %d, got %d): %s", expectedNumFieldsPerLine, len(fields), line) @@ -504,3 +513,475 @@ func (mounter *SafeFormatAndMount) getDiskFormat(disk string) (string, error) { // and MD RAID are reported as FSTYPE and caught above). return "unknown data, probably partitions", nil } + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + newHostPath, err = doBindSubPath(mounter, subPath, os.Getpid()) + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +// This implementation is shared between Linux and NsEnterMounter +// kubeletPid is PID of kubelet in the PID namespace where bind-mount is done, +// i.e. pid on the *host* if kubelet runs in a container. +func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath string, err error) { + // Check early for symlink. This is just a pre-check to avoid bind-mount + // before the final check. + evalSubPath, err := filepath.EvalSymlinks(subpath.Path) + if err != nil { + return "", fmt.Errorf("evalSymlinks %q failed: %v", subpath.Path, err) + } + glog.V(5).Infof("doBindSubPath %q, full subpath %q for volumepath %q", subpath.Path, evalSubPath, subpath.VolumePath) + + evalSubPath = filepath.Clean(evalSubPath) + if !pathWithinBase(evalSubPath, subpath.VolumePath) { + return "", fmt.Errorf("subpath %q not within volume path %q", evalSubPath, subpath.VolumePath) + } + + // Prepare directory for bind mounts + // containerName is DNS label, i.e. safe as a directory name. + bindDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + err = os.MkdirAll(bindDir, 0750) + if err != nil && !os.IsExist(err) { + return "", fmt.Errorf("error creating directory %s: %s", bindDir, err) + } + bindPathTarget := filepath.Join(bindDir, strconv.Itoa(subpath.VolumeMountIndex)) + + success := false + defer func() { + // Cleanup subpath on error + if !success { + glog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget) + if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil { + glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr) + } + } + }() + + // Check it's not already bind-mounted + notMount, err := IsNotMountPoint(mounter, bindPathTarget) + if err != nil { + if !os.IsNotExist(err) { + return "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err) + } + // Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet. + notMount = true + } + if !notMount { + // It's already mounted + glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget) + success = true + return bindPathTarget, nil + } + + // Create target of the bind mount. A directory for directories, empty file + // for everything else. + t, err := os.Lstat(subpath.Path) + if err != nil { + return "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err) + } + if t.Mode()&os.ModeDir > 0 { + if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) { + return "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err) + } + } else { + // "/bin/touch ". + // A file is enough for all possible targets (symlink, device, pipe, + // socket, ...), bind-mounting them into a file correctly changes type + // of the target file. + if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil { + return "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err) + } + } + + // Safe open subpath and get the fd + fd, err := doSafeOpen(evalSubPath, subpath.VolumePath) + if err != nil { + return "", fmt.Errorf("error opening subpath %v: %v", evalSubPath, err) + } + defer syscall.Close(fd) + + mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd) + + // Do the bind mount + glog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget) + if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil { + return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err) + } + + success = true + glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget) + return bindPathTarget, nil +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +// This implementation is shared between Linux and NsEnterMounter +func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error { + glog.V(4).Infof("Cleaning up subpath mounts for %s", podDir) + // scan /var/lib/kubelet/pods//volume-subpaths//* + subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName) + containerDirs, err := ioutil.ReadDir(subPathDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("error reading %s: %s", subPathDir, err) + } + + for _, containerDir := range containerDirs { + if !containerDir.IsDir() { + glog.V(4).Infof("Container file is not a directory: %s", containerDir.Name()) + continue + } + glog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name()) + + // scan /var/lib/kubelet/pods//volume-subpaths///* + fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name()) + subPaths, err := ioutil.ReadDir(fullContainerDirPath) + if err != nil { + return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err) + } + for _, subPath := range subPaths { + if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil { + return err + } + } + // Whole container has been processed, remove its directory. + if err := os.Remove(fullContainerDirPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err) + } + glog.V(5).Infof("Removed %s", fullContainerDirPath) + } + // Whole pod volume subpaths have been cleaned up, remove its subpath directory. + if err := os.Remove(subPathDir); err != nil { + return fmt.Errorf("error deleting %s: %s", subPathDir, err) + } + glog.V(5).Infof("Removed %s", subPathDir) + + // Remove entire subpath directory if it's the last one + podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName) + if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) { + return fmt.Errorf("error deleting %s: %s", podSubPathDir, err) + } + glog.V(5).Infof("Removed %s", podSubPathDir) + return nil +} + +// doCleanSubPath tears down the single subpath bind mount +func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error { + // process /var/lib/kubelet/pods//volume-subpaths/// + glog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex) + fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex) + notMnt, err := IsNotMountPoint(mounter, fullSubPath) + if err != nil { + return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err) + } + // Unmount it + if !notMnt { + if err = mounter.Unmount(fullSubPath); err != nil { + return fmt.Errorf("error unmounting %s: %s", fullSubPath, err) + } + glog.V(5).Infof("Unmounted %s", fullSubPath) + } + // Remove it *non*-recursively, just in case there were some hiccups. + if err = os.Remove(fullSubPath); err != nil { + return fmt.Errorf("error deleting %s: %s", fullSubPath, err) + } + glog.V(5).Infof("Removed %s", fullSubPath) + return nil +} + +// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty +func cleanSubPath(mounter Interface, subpath Subpath) error { + containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName) + + // Clean subdir bindmount + if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) { + return err + } + + // Recusively remove directories if empty + if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil { + return err + } + + return nil +} + +// removeEmptyDirs works backwards from endDir to baseDir and removes each directory +// if it is empty. It stops once it encounters a directory that has content +func removeEmptyDirs(baseDir, endDir string) error { + if !pathWithinBase(endDir, baseDir) { + return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir) + } + + for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) { + s, err := os.Stat(curDir) + if err != nil { + if os.IsNotExist(err) { + glog.V(5).Infof("curDir %q doesn't exist, skipping", curDir) + continue + } + return fmt.Errorf("error stat %q: %v", curDir, err) + } + if !s.IsDir() { + return fmt.Errorf("path %q not a directory", curDir) + } + + err = os.Remove(curDir) + if os.IsExist(err) { + glog.V(5).Infof("Directory %q not empty, not removing", curDir) + break + } else if err != nil { + return fmt.Errorf("error removing directory %q: %v", curDir, err) + } + glog.V(5).Infof("Removed directory %q", curDir) + } + return nil +} + +func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return doSafeMakeDir(pathname, base, perm) +} + +// This implementation is shared between Linux and NsEnterMounter +func doSafeMakeDir(pathname string, base string, perm os.FileMode) error { + glog.V(4).Infof("Creating directory %q within base %q", pathname, base) + + if !pathWithinBase(pathname, base) { + return fmt.Errorf("path %s is outside of allowed base %s", pathname, base) + } + + // Quick check if the directory already exists + s, err := os.Stat(pathname) + if err == nil { + // Path exists + if s.IsDir() { + // The directory already exists. It can be outside of the parent, + // but there is no race-proof check. + glog.V(4).Infof("Directory %s already exists", pathname) + return nil + } + return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR} + } + + // Find all existing directories + existingPath, toCreate, err := findExistingPrefix(base, pathname) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", pathname, err) + } + // Ensure the existing directory is inside allowed base + fullExistingPath, err := filepath.EvalSymlinks(existingPath) + if err != nil { + return fmt.Errorf("error opening directory %s: %s", existingPath, err) + } + if !pathWithinBase(fullExistingPath, base) { + return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, base) + } + + glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...)) + parentFD, err := doSafeOpen(fullExistingPath, base) + if err != nil { + return fmt.Errorf("cannot open directory %s: %s", existingPath, err) + } + childFD := -1 + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + } + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := fullExistingPath + // create the directories one by one, making sure nobody can change + // created directory into symlink. + for _, dir := range toCreate { + currentPath = filepath.Join(currentPath, dir) + glog.V(4).Infof("Creating %s", dir) + err = syscall.Mkdirat(parentFD, currentPath, uint32(perm)) + if err != nil { + return fmt.Errorf("cannot create directory %s: %s", currentPath, err) + } + // Dive into the created directory + childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0) + if err != nil { + return fmt.Errorf("cannot open %s: %s", currentPath, err) + } + // We can be sure that childFD is safe to use. It could be changed + // by user after Mkdirat() and before Openat(), however: + // - it could not be changed to symlink - we use nofollowFlags + // - it could be changed to a file (or device, pipe, socket, ...) + // but either subsequent Mkdirat() fails or we mount this file + // to user's container. Security is no violated in both cases + // and user either gets error or the file that it can already access. + + if err = syscall.Close(parentFD); err != nil { + glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err) + } + parentFD = childFD + childFD = -1 + } + + // Everything was created. mkdirat(..., perm) above was affected by current + // umask and we must apply the right permissions to the last directory + // (that's the one that will be available to the container as subpath) + // so user can read/write it. This is the behavior of previous code. + // TODO: chmod all created directories, not just the last one. + // parentFD is the last created directory. + if err = syscall.Fchmod(parentFD, uint32(perm)&uint32(os.ModePerm)); err != nil { + return fmt.Errorf("chmod %q failed: %s", currentPath, err) + } + return nil +} + +// findExistingPrefix finds prefix of pathname that exists. In addition, it +// returns list of remaining directories that don't exist yet. +func findExistingPrefix(base, pathname string) (string, []string, error) { + rel, err := filepath.Rel(base, pathname) + if err != nil { + return base, nil, err + } + dirs := strings.Split(rel, string(filepath.Separator)) + + // Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks. + // This should be faster than looping through all dirs and calling os.Stat() + // on each of them, as the symlinks are resolved only once with OpenAt(). + currentPath := base + fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0) + if err != nil { + return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err) + } + defer func() { + if err = syscall.Close(fd); err != nil { + glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + }() + for i, dir := range dirs { + childFD, err := syscall.Openat(fd, dir, syscall.O_RDONLY, 0) + if err != nil { + if os.IsNotExist(err) { + return currentPath, dirs[i:], nil + } + return base, nil, err + } + if err = syscall.Close(fd); err != nil { + glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err) + } + fd = childFD + currentPath = filepath.Join(currentPath, dir) + } + return pathname, []string{}, nil +} + +// This implementation is shared between Linux and NsEnterMounter +// Open path and return its fd. +// Symlinks are disallowed (pathname must already resolve symlinks), +// and the path must be within the base directory. +func doSafeOpen(pathname string, base string) (int, error) { + // Calculate segments to follow + subpath, err := filepath.Rel(base, pathname) + if err != nil { + return -1, err + } + segments := strings.Split(subpath, string(filepath.Separator)) + + // Assumption: base is the only directory that we have under control. + // Base dir is not allowed to be a symlink. + parentFD, err := syscall.Open(base, nofollowFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open directory %s: %s", base, err) + } + defer func() { + if parentFD != -1 { + if err = syscall.Close(parentFD); err != nil { + glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err) + } + } + }() + + childFD := -1 + defer func() { + if childFD != -1 { + if err = syscall.Close(childFD); err != nil { + glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err) + } + } + }() + + currentPath := base + + // Follow the segments one by one using openat() to make + // sure the user cannot change already existing directories into symlinks. + for _, seg := range segments { + currentPath = filepath.Join(currentPath, seg) + if !pathWithinBase(currentPath, base) { + return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base) + } + + glog.V(5).Infof("Opening path %s", currentPath) + childFD, err = syscall.Openat(parentFD, seg, nofollowFlags, 0) + if err != nil { + return -1, fmt.Errorf("cannot open %s: %s", currentPath, err) + } + + // Close parentFD + if err = syscall.Close(parentFD); err != nil { + return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err) + } + // Set child to new parent + parentFD = childFD + childFD = -1 + } + + // We made it to the end, return this fd, don't close it + finalFD := parentFD + parentFD = -1 + + return finalFD, nil +} + +type mountInfo struct { + mountPoint string + // list of "optional parameters", mount propagation is one of them + optional []string +} + +// parseMountInfo parses /proc/xxx/mountinfo. +func parseMountInfo(filename string) ([]mountInfo, error) { + content, err := utilio.ConsistentRead(filename, maxListTries) + if err != nil { + return []mountInfo{}, err + } + contentStr := string(content) + infos := []mountInfo{} + + for _, line := range strings.Split(contentStr, "\n") { + if line == "" { + // the last split() item is empty string following the last \n + continue + } + fields := strings.Fields(line) + if len(fields) < 7 { + return nil, fmt.Errorf("wrong number of fields in (expected %d, got %d): %s", 8, len(fields), line) + } + info := mountInfo{ + mountPoint: fields[4], + optional: []string{}, + } + for i := 6; i < len(fields) && fields[i] != "-"; i++ { + info.optional = append(info.optional, fields[i]) + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux_test.go b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux_test.go index e13cb7a0f988..06ab9c52ad84 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux_test.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux_test.go @@ -19,8 +19,18 @@ limitations under the License. package mount import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" "strings" + "syscall" "testing" + + "strconv" + + "github.com/golang/glog" ) func TestReadProcMountsFrom(t *testing.T) { @@ -181,3 +191,1223 @@ func TestGetDeviceNameFromMount(t *testing.T) { } } } + +func TestPathWithinBase(t *testing.T) { + tests := []struct { + name string + fullPath string + basePath string + expected bool + }{ + { + name: "good subpath", + fullPath: "/a/b/c", + basePath: "/a", + expected: true, + }, + { + name: "good subpath 2", + fullPath: "/a/b/c", + basePath: "/a/b", + expected: true, + }, + { + name: "good subpath end slash", + fullPath: "/a/b/c/", + basePath: "/a/b", + expected: true, + }, + { + name: "good subpath backticks", + fullPath: "/a/b/../c", + basePath: "/a", + expected: true, + }, + { + name: "good subpath equal", + fullPath: "/a/b/c", + basePath: "/a/b/c", + expected: true, + }, + { + name: "good subpath equal 2", + fullPath: "/a/b/c/", + basePath: "/a/b/c", + expected: true, + }, + { + name: "good subpath root", + fullPath: "/a", + basePath: "/", + expected: true, + }, + { + name: "bad subpath parent", + fullPath: "/a/b/c", + basePath: "/a/b/c/d", + expected: false, + }, + { + name: "bad subpath outside", + fullPath: "/b/c", + basePath: "/a/b/c", + expected: false, + }, + { + name: "bad subpath prefix", + fullPath: "/a/b/cd", + basePath: "/a/b/c", + expected: false, + }, + { + name: "bad subpath backticks", + fullPath: "/a/../b", + basePath: "/a", + expected: false, + }, + { + name: "configmap subpath", + fullPath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config/..timestamp/file.txt", + basePath: "/var/lib/kubelet/pods/uuid/volumes/kubernetes.io~configmap/config", + expected: true, + }, + } + for _, test := range tests { + if pathWithinBase(test.fullPath, test.basePath) != test.expected { + t.Errorf("test %q failed: expected %v", test.name, test.expected) + } + + } +} + +func TestSafeMakeDir(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + checkPath string + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "test/directory", + "test/directory", + false, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + "test/directory", + false, + }, + { + "create-base", + func(base string) error { + return nil + }, + "", + "", + false, + }, + { + "escape-base-using-dots", + func(base string) error { + return nil + }, + "..", + "", + true, + }, + { + "escape-base-using-dots-2", + func(base string) error { + return nil + }, + "test/../../..", + "", + true, + }, + { + "follow-symlinks", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test/directory", + "destination/directory", + false, + }, + { + "follow-symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test/directory", + "", + true, + }, + { + "follow-symlink-multiple follow", + func(base string) error { + /* test1/dir points to test2 and test2/dir points to test1 */ + if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { + return err + } + return nil + }, + "test1/dir/dir/dir/dir/dir/dir/dir/foo", + "test2/foo", + false, + }, + { + "danglink-symlink", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + "test/directory", + "", + true, + }, + { + "non-directory", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test/directory", + "", + true, + }, + { + "non-directory-final", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test", + "", + true, + }, + { + "escape-with-relative-symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { + return err + } + return os.Symlink("../exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + "", + false, + }, + { + "escape-with-relative-symlink-not-exists", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + "", + true, + }, + { + "escape-with-symlink", + func(base string) error { + return os.Symlink("/", filepath.Join(base, "test")) + }, + "test/directory", + "", + true, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "safe-make-dir-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + test.prepare(base) + pathToCreate := filepath.Join(base, test.path) + err = doSafeMakeDir(pathToCreate, base, defaultPerm) + if err != nil && !test.expectError { + t.Errorf("test %q: %s", test.name, err) + } + if err != nil { + glog.Infof("got error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("test %q: expected error, got none", test.name) + } + + if test.checkPath != "" { + if _, err := os.Stat(filepath.Join(base, test.checkPath)); err != nil { + t.Errorf("test %q: cannot read path %s", test.name, test.checkPath) + } + } + + os.RemoveAll(base) + } + +} + +func validateDirEmpty(dir string) error { + files, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + + if len(files) != 0 { + return fmt.Errorf("Directory %q is not empty", dir) + } + return nil +} + +func validateDirExists(dir string) error { + _, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + return nil +} + +func validateDirNotExists(dir string) error { + _, err := ioutil.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + return fmt.Errorf("dir %q still exists", dir) +} + +func validateFileExists(file string) error { + if _, err := os.Stat(file); err != nil { + return err + } + return nil +} + +func TestRemoveEmptyDirs(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + // Function that validates directory structure after the test + validate func(base string) error + baseDir string + endDir string + expectError bool + }{ + { + name: "all-empty", + prepare: func(base string) error { + return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) + }, + validate: func(base string) error { + return validateDirEmpty(filepath.Join(base, "a")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "dir-not-empty", + prepare: func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm); err != nil { + return err + } + return os.Mkdir(filepath.Join(base, "a/b/d"), defaultPerm) + }, + validate: func(base string) error { + if err := validateDirNotExists(filepath.Join(base, "a/b/c")); err != nil { + return err + } + return validateDirExists(filepath.Join(base, "a/b")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "path-not-within-base", + prepare: func(base string) error { + return os.MkdirAll(filepath.Join(base, "a/b/c"), defaultPerm) + }, + validate: func(base string) error { + return validateDirExists(filepath.Join(base, "a")) + }, + baseDir: "a", + endDir: "b/c", + expectError: true, + }, + { + name: "path-already-deleted", + prepare: func(base string) error { + return nil + }, + validate: func(base string) error { + return nil + }, + baseDir: "a", + endDir: "a/b/c", + expectError: false, + }, + { + name: "path-not-dir", + prepare: func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "a/b"), defaultPerm); err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(base, "a/b", "c"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + if err := validateDirExists(filepath.Join(base, "a/b")); err != nil { + return err + } + return validateFileExists(filepath.Join(base, "a/b/c")) + }, + baseDir: "a", + endDir: "a/b/c", + expectError: true, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "remove-empty-dirs-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + if err = test.prepare(base); err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + err = removeEmptyDirs(filepath.Join(base, test.baseDir), filepath.Join(base, test.endDir)) + if err != nil && !test.expectError { + t.Errorf("test %q failed: %v", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("test %q failed: expected error, got success", test.name) + } + + if err = test.validate(base); err != nil { + t.Errorf("test %q failed validation: %v", test.name, err) + } + + os.RemoveAll(base) + } +} + +func TestCleanSubPaths(t *testing.T) { + defaultPerm := os.FileMode(0750) + testVol := "vol1" + + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) ([]MountPoint, error) + // Function that validates directory structure after the test + validate func(base string) error + expectError bool + }{ + { + name: "not-exists", + prepare: func(base string) ([]MountPoint, error) { + return nil, nil + }, + validate: func(base string) error { + return nil + }, + expectError: false, + }, + { + name: "subpath-not-mount", + prepare: func(base string) ([]MountPoint, error) { + return nil, os.MkdirAll(filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0"), defaultPerm) + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + expectError: false, + }, + { + name: "subpath-file", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "0"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + expectError: false, + }, + { + name: "subpath-container-not-dir", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "container1"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + return validateDirExists(filepath.Join(base, containerSubPathDirectoryName, testVol)) + }, + expectError: true, + }, + { + name: "subpath-multiple-container-not-dir", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := os.MkdirAll(filepath.Join(path, "container1"), defaultPerm); err != nil { + return nil, err + } + return nil, ioutil.WriteFile(filepath.Join(path, "container2"), []byte{}, defaultPerm) + }, + validate: func(base string) error { + path := filepath.Join(base, containerSubPathDirectoryName, testVol) + if err := validateDirNotExists(filepath.Join(path, "container1")); err != nil { + return err + } + return validateFileExists(filepath.Join(path, "container2")) + }, + expectError: true, + }, + { + name: "subpath-mount", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{{Device: "/dev/sdb", Path: path}} + return mounts, nil + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + }, + { + name: "subpath-mount-multiple", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + path2 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "1") + path3 := filepath.Join(base, containerSubPathDirectoryName, testVol, "container2", "1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path2, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path3, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{ + {Device: "/dev/sdb", Path: path}, + {Device: "/dev/sdb", Path: path3}, + } + return mounts, nil + }, + validate: func(base string) error { + return validateDirNotExists(filepath.Join(base, containerSubPathDirectoryName)) + }, + }, + { + name: "subpath-mount-multiple-vols", + prepare: func(base string) ([]MountPoint, error) { + path := filepath.Join(base, containerSubPathDirectoryName, testVol, "container1", "0") + path2 := filepath.Join(base, containerSubPathDirectoryName, "vol2", "container1", "1") + if err := os.MkdirAll(path, defaultPerm); err != nil { + return nil, err + } + if err := os.MkdirAll(path2, defaultPerm); err != nil { + return nil, err + } + mounts := []MountPoint{ + {Device: "/dev/sdb", Path: path}, + } + return mounts, nil + }, + validate: func(base string) error { + baseSubdir := filepath.Join(base, containerSubPathDirectoryName) + if err := validateDirNotExists(filepath.Join(baseSubdir, testVol)); err != nil { + return err + } + return validateDirExists(baseSubdir) + }, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "clean-subpaths-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + mounts, err := test.prepare(base) + if err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + fm := &FakeMounter{MountPoints: mounts} + + err = doCleanSubPaths(fm, base, testVol) + if err != nil && !test.expectError { + t.Errorf("test %q failed: %v", test.name, err) + } + if err == nil && test.expectError { + t.Errorf("test %q failed: expected error, got success", test.name) + } + if err = test.validate(base); err != nil { + t.Errorf("test %q failed validation: %v", test.name, err) + } + + os.RemoveAll(base) + } +} + +var ( + testVol = "vol1" + testPod = "pod0" + testContainer = "container0" + testSubpath = 1 +) + +func setupFakeMounter(testMounts []string) *FakeMounter { + mounts := []MountPoint{} + for _, mountPoint := range testMounts { + mounts = append(mounts, MountPoint{Device: "/foo", Path: mountPoint}) + } + return &FakeMounter{MountPoints: mounts} +} + +func getTestPaths(base string) (string, string) { + return filepath.Join(base, testVol), + filepath.Join(base, testPod, containerSubPathDirectoryName, testVol, testContainer, strconv.Itoa(testSubpath)) +} + +func TestBindSubPath(t *testing.T) { + defaultPerm := os.FileMode(0750) + + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) ([]string, string, string, error) + expectError bool + }{ + { + name: "subpath-dir", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-dir-symlink", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + if err := os.MkdirAll(subpath, defaultPerm); err != nil { + return nil, "", "", err + } + subpathLink := filepath.Join(volpath, "dirLink") + return nil, volpath, subpath, os.Symlink(subpath, subpathLink) + }, + expectError: false, + }, + { + name: "subpath-file", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, ioutil.WriteFile(subpath, []byte{}, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-not-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "file0") + return nil, volpath, subpath, nil + }, + expectError: true, + }, + { + name: "subpath-outside", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, os.Symlink(base, subpath) + }, + expectError: true, + }, + { + name: "subpath-symlink-child-outside", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { + return nil, "", "", err + } + return nil, volpath, subpath, os.Symlink(base, subpath) + }, + expectError: true, + }, + { + name: "subpath-child-outside-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + child := filepath.Join(base, "child0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + // touch file outside + if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for subpath dir + return nil, volpath, subpath, os.Symlink(base, subpathDir) + }, + expectError: true, + }, + { + name: "subpath-child-outside-not-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + subpath := filepath.Join(subpathDir, "child0") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + // create symlink for subpath dir + return nil, volpath, subpath, os.Symlink(base, subpathDir) + }, + expectError: true, + }, + { + name: "subpath-child-outside-exists-middle-dir-symlink", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpathDir := filepath.Join(volpath, "dir0") + symlinkDir := filepath.Join(subpathDir, "linkDir0") + child := filepath.Join(base, "child0") + subpath := filepath.Join(symlinkDir, "child0") + if err := os.MkdirAll(subpathDir, defaultPerm); err != nil { + return nil, "", "", err + } + // touch file outside + if err := ioutil.WriteFile(child, []byte{}, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for middle dir + return nil, volpath, subpath, os.Symlink(base, symlinkDir) + }, + expectError: true, + }, + { + name: "subpath-backstepping", + prepare: func(base string) ([]string, string, string, error) { + volpath, _ := getTestPaths(base) + subpath := filepath.Join(volpath, "dir0") + symlinkBase := filepath.Join(volpath, "..") + if err := os.MkdirAll(volpath, defaultPerm); err != nil { + return nil, "", "", err + } + + // create symlink for subpath + return nil, volpath, subpath, os.Symlink(symlinkBase, subpath) + }, + expectError: true, + }, + { + name: "subpath-mountdir-already-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { + return nil, "", "", err + } + + subpath := filepath.Join(volpath, "dir0") + return nil, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + { + name: "subpath-mount-already-exists", + prepare: func(base string) ([]string, string, string, error) { + volpath, subpathMount := getTestPaths(base) + mounts := []string{subpathMount} + if err := os.MkdirAll(subpathMount, defaultPerm); err != nil { + return nil, "", "", err + } + + subpath := filepath.Join(volpath, "dir0") + return mounts, volpath, subpath, os.MkdirAll(subpath, defaultPerm) + }, + expectError: false, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "bind-subpath-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + mounts, volPath, subPath, err := test.prepare(base) + if err != nil { + os.RemoveAll(base) + t.Fatalf("failed to prepare test %q: %v", test.name, err.Error()) + } + + fm := setupFakeMounter(mounts) + + subpath := Subpath{ + VolumeMountIndex: testSubpath, + Path: subPath, + VolumeName: testVol, + VolumePath: volPath, + PodDir: filepath.Join(base, "pod0"), + ContainerName: testContainer, + } + + _, subpathMount := getTestPaths(base) + bindPathTarget, err := doBindSubPath(fm, subpath, 1) + if test.expectError { + if err == nil { + t.Errorf("test %q failed: expected error, got success", test.name) + } + if bindPathTarget != "" { + t.Errorf("test %q failed: expected empty bindPathTarget, got %v", test.name, bindPathTarget) + } + if err = validateDirNotExists(subpathMount); err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + } + if !test.expectError { + if err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + if bindPathTarget != subpathMount { + t.Errorf("test %q failed: expected bindPathTarget %v, got %v", test.name, subpathMount, bindPathTarget) + } + if err = validateFileExists(subpathMount); err != nil { + t.Errorf("test %q failed: %v", test.name, err) + } + } + + os.RemoveAll(base) + } +} + +func TestParseMountInfo(t *testing.T) { + info := + `62 0 253:0 / / rw,relatime shared:1 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +78 62 0:41 / /tmp rw,nosuid,nodev shared:30 - tmpfs tmpfs rw,seclabel +80 62 0:42 / /var/lib/nfs/rpc_pipefs rw,relatime shared:31 - rpc_pipefs sunrpc rw +82 62 0:43 / /var/lib/foo rw,relatime shared:32 - tmpfs tmpfs rw +83 63 0:44 / /var/lib/bar rw,relatime - tmpfs tmpfs rw +227 62 253:0 /var/lib/docker/devicemapper /var/lib/docker/devicemapper rw,relatime - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +224 62 253:0 /var/lib/docker/devicemapper/test/shared /var/lib/docker/devicemapper/test/shared rw,relatime master:1 shared:44 - ext4 /dev/mapper/ssd-root rw,seclabel,data=ordered +76 17 8:1 / /mnt/stateful_partition rw,nosuid,nodev,noexec,relatime - ext4 /dev/sda1 rw,commit=30,data=ordered +80 17 8:1 /var /var rw,nosuid,nodev,noexec,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +189 80 8:1 /var/lib/kubelet /var/lib/kubelet rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +818 77 8:40 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +819 78 8:48 / /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +900 100 8:48 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +901 101 8:1 /dir1 /var/lib/kubelet/pods/c25464af-e52e-11e7-ab4d-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +902 102 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol1/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/0 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +903 103 8:1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volumes/kubernetes.io~empty-dir/vol2/dir1 /var/lib/kubelet/pods/d4076f24-e53a-11e7-ba15-42010a800002/volume-subpaths/vol1/subpath1/1 rw,relatime shared:30 - ext4 /dev/sda1 rw,commit=30,data=ordered +178 25 253:0 /etc/bar /var/lib/kubelet/pods/12345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:1 - ext4 /dev/sdb2 rw,errors=remount-ro,data=ordered +698 186 0:41 /tmp1/dir1 /var/lib/kubelet/pods/41135147-e697-11e7-9342-42010a800002/volume-subpaths/vol1/subpath1/0 rw shared:26 - tmpfs tmpfs rw +918 77 8:50 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +919 78 8:58 / /var/lib/kubelet/pods/2345/volumes/kubernetes.io~gce-pd/vol1 rw,relatime shared:290 - ext4 /dev/sdd rw,data=ordered +920 100 8:50 /dir1 /var/lib/kubelet/pods/2345/volume-subpaths/vol1/subpath1/0 rw,relatime shared:290 - ext4 /dev/sdc rw,data=ordered +150 23 1:58 / /media/nfs_vol rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +151 24 1:58 / /media/nfs_bindmount rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +134 23 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs1 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +187 23 0:58 / /var/lib/kubelet/pods/1fc5ea21-eff4-11e7-ac80-0e858b8eaf40/volumes/kubernetes.io~nfs/nfs2 rw,relatime shared:96 - nfs4 172.18.4.223:/srv/nfs2 rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +188 24 0:58 / /var/lib/kubelet/pods/43219158-e5e1-11e7-a392-0e858b8eaf40/volume-subpaths/nfs1/subpath1/0 rw,relatime shared:89 - nfs4 172.18.4.223:/srv/nfs/foo rw,vers=4.0,rsize=524288,wsize=524288,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.18.4.223,local_lock=none,addr=172.18.4.223 +347 60 0:71 / /var/lib/kubelet/pods/13195d46-f9fa-11e7-bbf1-5254007a695a/volumes/kubernetes.io~nfs/vol2 rw,relatime shared:170 - nfs 172.17.0.3:/exports/2 rw,vers=3,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=172.17.0.3,mountvers=3,mountport=20048,mountproto=udp,local_lock=none,addr=172.17.0.3 +` + tempDir, filename, err := writeFile(info) + if err != nil { + t.Fatalf("cannot create temporary file: %v", err) + } + defer os.RemoveAll(tempDir) + + tests := []struct { + name string + mountPoint string + expectedInfo mountInfo + }{ + { + "simple bind mount", + "/var/lib/kubelet", + mountInfo{ + mountPoint: "/var/lib/kubelet", + optional: []string{"shared:30"}, + }, + }, + } + + infos, err := parseMountInfo(filename) + if err != nil { + t.Fatalf("Cannot parse %s: %s", filename, err) + } + + for _, test := range tests { + found := false + for _, info := range infos { + if info.mountPoint == test.mountPoint { + found = true + if !reflect.DeepEqual(info, test.expectedInfo) { + t.Errorf("Test case %q:\n expected: %+v\n got: %+v", test.name, test.expectedInfo, info) + } + break + } + } + if !found { + t.Errorf("Test case %q: mountPoint %s not found", test.name, test.mountPoint) + } + } +} + +func TestSafeOpen(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "test/directory", + true, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + false, + }, + { + "escape-base-using-dots", + func(base string) error { + return nil + }, + "..", + true, + }, + { + "escape-base-using-dots-2", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test"), 0750) + }, + "test/../../..", + true, + }, + { + "symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "symlink-nested", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir1/dir2"), defaultPerm); err != nil { + return err + } + return os.Symlink("dir1", filepath.Join(base, "dir1/dir2/test")) + }, + "test", + true, + }, + { + "symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "symlink-not-exists", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + "test", + true, + }, + { + "non-directory", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test/directory", + true, + }, + { + "non-directory-final", + func(base string) error { + return ioutil.WriteFile(filepath.Join(base, "test"), []byte{}, defaultPerm) + }, + "test", + false, + }, + { + "escape-with-relative-symlink", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "exists"), defaultPerm); err != nil { + return err + } + return os.Symlink("../exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + true, + }, + { + "escape-with-relative-symlink-not-exists", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "dir"), defaultPerm); err != nil { + return err + } + return os.Symlink("../not-exists", filepath.Join(base, "dir/test")) + }, + "dir/test", + true, + }, + { + "escape-with-symlink", + func(base string) error { + return os.Symlink("/", filepath.Join(base, "test")) + }, + "test", + true, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "safe-open-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + test.prepare(base) + pathToCreate := filepath.Join(base, test.path) + fd, err := doSafeOpen(pathToCreate, base) + if err != nil && !test.expectError { + t.Errorf("test %q: %s", test.name, err) + } + if err != nil { + glog.Infof("got error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("test %q: expected error, got none", test.name) + } + + syscall.Close(fd) + os.RemoveAll(base) + } +} + +func TestFindExistingPrefix(t *testing.T) { + defaultPerm := os.FileMode(0750) + tests := []struct { + name string + // Function that prepares directory structure for the test under given + // base. + prepare func(base string) error + path string + expectedPath string + expectedDirs []string + expectError bool + }{ + { + "directory-does-not-exist", + func(base string) error { + return nil + }, + "directory", + "", + []string{"directory"}, + false, + }, + { + "directory-exists", + func(base string) error { + return os.MkdirAll(filepath.Join(base, "test/directory"), 0750) + }, + "test/directory", + "test/directory", + []string{}, + false, + }, + { + "follow-symlinks", + func(base string) error { + if err := os.MkdirAll(filepath.Join(base, "destination/directory"), defaultPerm); err != nil { + return err + } + return os.Symlink("destination", filepath.Join(base, "test")) + }, + "test/directory", + "test/directory", + []string{}, + false, + }, + { + "follow-symlink-loop", + func(base string) error { + return os.Symlink("test", filepath.Join(base, "test")) + }, + "test/directory", + "", + nil, + true, + }, + { + "follow-symlink-multiple follow", + func(base string) error { + /* test1/dir points to test2 and test2/dir points to test1 */ + if err := os.MkdirAll(filepath.Join(base, "test1"), defaultPerm); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(base, "test2"), defaultPerm); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test2"), filepath.Join(base, "test1/dir")); err != nil { + return err + } + if err := os.Symlink(filepath.Join(base, "test1"), filepath.Join(base, "test2/dir")); err != nil { + return err + } + return nil + }, + "test1/dir/dir/foo/bar", + "test1/dir/dir", + []string{"foo", "bar"}, + false, + }, + { + "danglink-symlink", + func(base string) error { + return os.Symlink("non-existing", filepath.Join(base, "test")) + }, + // OS returns IsNotExist error both for dangling symlink and for + // non-existing directory. + "test/directory", + "", + []string{"test", "directory"}, + false, + }, + } + + for _, test := range tests { + glog.V(4).Infof("test %q", test.name) + base, err := ioutil.TempDir("", "find-prefix-"+test.name+"-") + if err != nil { + t.Fatalf(err.Error()) + } + test.prepare(base) + path := filepath.Join(base, test.path) + existingPath, dirs, err := findExistingPrefix(base, path) + if err != nil && !test.expectError { + t.Errorf("test %q: %s", test.name, err) + } + if err != nil { + glog.Infof("got error: %s", err) + } + if err == nil && test.expectError { + t.Errorf("test %q: expected error, got none", test.name) + } + + fullExpectedPath := filepath.Join(base, test.expectedPath) + if existingPath != fullExpectedPath { + t.Errorf("test %q: expected path %q, got %q", test.name, fullExpectedPath, existingPath) + } + if !reflect.DeepEqual(dirs, test.expectedDirs) { + t.Errorf("test %q: expected dirs %v, got %v", test.name, test.expectedDirs, dirs) + } + os.RemoveAll(base) + } +} + +func writeFile(content string) (string, string, error) { + tempDir, err := ioutil.TempDir("", "mounter_shared_test") + if err != nil { + return "", "", err + } + filename := filepath.Join(tempDir, "mountinfo") + err = ioutil.WriteFile(filename, []byte(content), 0600) + if err != nil { + os.RemoveAll(tempDir) + return "", "", err + } + return tempDir, filename, nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go index 6c4000a0ee83..c27ccec5c788 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go @@ -18,6 +18,10 @@ limitations under the License. package mount +import ( + "os" +) + type Mounter struct { mounterPath string } @@ -74,3 +78,15 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) { return true, nil } + +func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, nil +} + +func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} + +func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go b/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go index d237483b4295..8f0d7767ecae 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go @@ -22,10 +22,23 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "strconv" "strings" "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/exec" + utilio "k8s.io/kubernetes/pkg/util/io" +) + +const ( + // hostProcSelfStatusPath is the default path to /proc/self/status on the host + hostProcSelfStatusPath = "/rootfs/proc/self/status" +) + +var ( + // pidRegExp matches "Pid: " in /proc/self/status + pidRegExp = regexp.MustCompile(`\nPid:\t([0-9]*)\n`) ) // NsenterMounter is part of experimental support for running the kubelet @@ -290,3 +303,41 @@ func (n *NsenterMounter) absHostPath(command string) string { } return path } + +func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { + return doCleanSubPaths(mounter, podDir, volumeName) +} + +// getPidOnHost returns kubelet's pid in the host pid namespace +func (mounter *NsenterMounter) getPidOnHost(procStatusPath string) (int, error) { + // Get the PID from /rootfs/proc/self/status + statusBytes, err := utilio.ConsistentRead(procStatusPath, maxListTries) + if err != nil { + return 0, fmt.Errorf("error reading %s: %s", procStatusPath, err) + } + matches := pidRegExp.FindSubmatch(statusBytes) + if len(matches) < 2 { + return 0, fmt.Errorf("cannot parse %s: no Pid:", procStatusPath) + } + return strconv.Atoi(string(matches[1])) +} + +func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + hostPid, err := mounter.getPidOnHost(hostProcSelfStatusPath) + if err != nil { + return "", nil, err + } + glog.V(4).Infof("Kubelet's PID on the host is %d", hostPid) + + // Bind-mount the subpath to avoid using symlinks in subpaths. + newHostPath, err = doBindSubPath(mounter, subPath, hostPid) + + // There is no action when the container starts. Bind-mount will be cleaned + // when container stops by CleanSubPaths. + cleanupAction = nil + return newHostPath, cleanupAction, err +} + +func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return doSafeMakeDir(pathname, base, perm) +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go b/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go index e955e1b781b4..6039ce4523c5 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go @@ -18,6 +18,10 @@ limitations under the License. package mount +import ( + "os" +) + type NsenterMounter struct{} func NewNsenterMounter() *NsenterMounter { @@ -61,3 +65,15 @@ func (*NsenterMounter) PathIsDevice(pathname string) (bool, error) { func (*NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { return "", nil } + +func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error { + return nil +} + +func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) { + return subPath.Path, nil, nil +} + +func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error { + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/util/removeall/removeall_test.go b/vendor/k8s.io/kubernetes/pkg/util/removeall/removeall_test.go index a5b19fe41a20..2dbf906be571 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/removeall/removeall_test.go +++ b/vendor/k8s.io/kubernetes/pkg/util/removeall/removeall_test.go @@ -161,3 +161,15 @@ func TestRemoveAllOneFilesystem(t *testing.T) { } } } + +func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) { + return "", nil, nil +} + +func (mounter *fakeMounter) CleanSubPaths(_, _ string) error { + return nil +} + +func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error { + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/configmap/configmap.go b/vendor/k8s.io/kubernetes/pkg/volume/configmap/configmap.go index 8f087cb6317e..f67a499d98c6 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/configmap/configmap.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/configmap/configmap.go @@ -194,6 +194,9 @@ func (b *configMapVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } + if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { + return err + } optional := b.source.Optional != nil && *b.source.Optional configMap, err := b.getConfigMap(b.pod.Namespace, b.source.Name) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/downwardapi/downwardapi.go b/vendor/k8s.io/kubernetes/pkg/volume/downwardapi/downwardapi.go index aac6eb730cff..c6edf12703b8 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/downwardapi/downwardapi.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/downwardapi/downwardapi.go @@ -184,6 +184,9 @@ func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { glog.Errorf("Unable to setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err } + if err := volumeutil.MakeNestedMountpoints(b.volName, dir, *b.pod); err != nil { + return err + } data, err := CollectData(b.source.Items, b.pod, b.plugin.host, b.source.DefaultMode) if err != nil { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/nfs/nfs.go b/vendor/k8s.io/kubernetes/pkg/volume/nfs/nfs.go index db8ab0e5ca67..4917b98691ba 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/nfs/nfs.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/nfs/nfs.go @@ -234,7 +234,7 @@ func (b *nfsMounter) SetUp(fsGroup *int64) error { } func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { - notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) + notMnt, err := b.mounter.IsNotMountPoint(dir) glog.V(4).Infof("NFS mount set up: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { return err @@ -253,7 +253,7 @@ func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { mountOptions := volume.JoinMountOptions(b.mountOptions, options) err = b.mounter.Mount(source, dir, "nfs", mountOptions) if err != nil { - notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + notMnt, mntErr := b.mounter.IsNotMountPoint(dir) if mntErr != nil { glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err @@ -263,7 +263,7 @@ func (b *nfsMounter) SetUpAt(dir string, fsGroup *int64) error { glog.Errorf("Failed to unmount: %v", mntErr) return err } - notMnt, mntErr := b.mounter.IsLikelyNotMountPoint(dir) + notMnt, mntErr := b.mounter.IsNotMountPoint(dir) if mntErr != nil { glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr) return err @@ -291,7 +291,10 @@ func (c *nfsUnmounter) TearDown() error { } func (c *nfsUnmounter) TearDownAt(dir string) error { - return util.UnmountPath(dir, c.mounter) + // Use extensiveMountPointCheck to consult /proc/mounts. We can't use faster + // IsLikelyNotMountPoint (lstat()), since there may be root_squash on the + // NFS server and kubelet may not be able to do lstat/stat() there. + return util.UnmountMountPoint(dir, c.mounter, true /* extensiveMountPointCheck */) } func getVolumeSource(spec *volume.Spec) (*v1.NFSVolumeSource, bool, error) { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/projected/projected.go b/vendor/k8s.io/kubernetes/pkg/volume/projected/projected.go index a4d3e57e2ba3..f3ac72cfa03e 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/projected/projected.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/projected/projected.go @@ -191,6 +191,9 @@ func (s *projectedVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } + if err := volumeutil.MakeNestedMountpoints(s.volName, dir, *s.pod); err != nil { + return err + } data, err := s.collectData() if err != nil { diff --git a/vendor/k8s.io/kubernetes/pkg/volume/secret/secret.go b/vendor/k8s.io/kubernetes/pkg/volume/secret/secret.go index f16d950560d6..3cc7384dc289 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/secret/secret.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/secret/secret.go @@ -193,6 +193,9 @@ func (b *secretVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { if err := wrapped.SetUpAt(dir, fsGroup); err != nil { return err } + if err := volumeutil.MakeNestedMountpoints(b.volName, dir, b.pod); err != nil { + return err + } optional := b.source.Optional != nil && *b.source.Optional secret, err := b.getSecret(b.pod.Namespace, b.source.SecretName) diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/BUILD b/vendor/k8s.io/kubernetes/pkg/volume/util/BUILD index ddaade57f51b..7d8086052a18 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/BUILD @@ -18,6 +18,7 @@ go_library( "fs.go", "io_util.go", "metrics.go", + "nested_volumes.go", "util.go", ], tags = ["automanaged"], @@ -41,6 +42,7 @@ go_test( srcs = [ "atomic_writer_test.go", "device_util_linux_test.go", + "nested_volumes_test.go", "util_test.go", ], library = ":go_default_library", @@ -49,6 +51,7 @@ go_test( "//pkg/api/v1:go_default_library", "//pkg/api/v1/helper:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/client-go/util/testing:go_default_library", ], diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes.go b/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes.go new file mode 100644 index 000000000000..d9a2513fc088 --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes.go @@ -0,0 +1,99 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "k8s.io/kubernetes/pkg/api/v1" + "os" + "path" + "path/filepath" + "sort" + "strings" +) + +// getNestedMountpoints returns a list of mountpoint directories that should be created +// for the volume indicated by name. +// note: the returned list is relative to baseDir +func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) { + var retval []string + checkContainer := func(container *v1.Container) error { + var allMountPoints []string // all mount points in this container + var myMountPoints []string // mount points that match name + for _, vol := range container.VolumeMounts { + cleaned := filepath.Clean(vol.MountPath) + allMountPoints = append(allMountPoints, cleaned) + if vol.Name == name { + myMountPoints = append(myMountPoints, cleaned) + } + } + sort.Strings(allMountPoints) + parentPrefix := ".." + string(os.PathSeparator) + // Examine each place where this volume is mounted + for _, myMountPoint := range myMountPoints { + if strings.HasPrefix(myMountPoint, parentPrefix) { + // Don't let a container trick us into creating directories outside of its rootfs + return fmt.Errorf("Invalid container mount point %v", myMountPoint) + } + myMPSlash := myMountPoint + string(os.PathSeparator) + // The previously found nested mountpoint (or "" if none found yet) + prevNestedMP := "" + // examine each mount point to see if it's nested beneath this volume + // (but skip any that are double-nested beneath this volume) + // For example, if this volume is mounted as /dir and other volumes are mounted + // as /dir/nested and /dir/nested/other, only create /dir/nested. + for _, mp := range allMountPoints { + if !strings.HasPrefix(mp, myMPSlash) { + continue // skip -- not nested beneath myMountPoint + } + if prevNestedMP != "" && strings.HasPrefix(mp, prevNestedMP) { + continue // skip -- double nested beneath myMountPoint + } + // since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one + prevNestedMP = mp + string(os.PathSeparator) + retval = append(retval, mp[len(myMPSlash):]) + } + } + return nil + } + for _, container := range pod.Spec.InitContainers { + if err := checkContainer(&container); err != nil { + return nil, err + } + } + for _, container := range pod.Spec.Containers { + if err := checkContainer(&container); err != nil { + return nil, err + } + } + return retval, nil +} + +// MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name +func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error { + dirs, err := getNestedMountpoints(name, baseDir, pod) + if err != nil { + return err + } + for _, dir := range dirs { + err := os.MkdirAll(path.Join(baseDir, dir), 0755) + if err != nil { + return fmt.Errorf("Unable to create nested volume mountpoints: %v", err) + } + } + return nil +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes_test.go b/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes_test.go new file mode 100644 index 000000000000..ab709dc4651d --- /dev/null +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes_test.go @@ -0,0 +1,233 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "io/ioutil" + "os" + "path" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/api/v1" +) + +type testCases struct { + name string + err bool + expected sets.String + volname string + pod v1.Pod +} + +func TestGetNestedMountpoints(t *testing.T) { + var ( + testNamespace = "test_namespace" + testPodUID = types.UID("test_pod_uid") + ) + + tc := []testCases{ + { + name: "Simple Pod", + err: false, + expected: sets.NewString(), + volname: "vol1", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/dir", Name: "vol1"}, + }, + }, + }, + }, + }, + }, + { + name: "Simple Nested Pod", + err: false, + expected: sets.NewString("nested"), + volname: "vol1", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/dir", Name: "vol1"}, + {MountPath: "/dir/nested", Name: "vol2"}, + }, + }, + }, + }, + }, + }, + { + name: "Unsorted Nested Pod", + err: false, + expected: sets.NewString("nested", "nested2"), + volname: "vol1", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/dir/nested/double", Name: "vol3"}, + {MountPath: "/ignore", Name: "vol4"}, + {MountPath: "/dir/nested", Name: "vol2"}, + {MountPath: "/ignore2", Name: "vol5"}, + {MountPath: "/dir", Name: "vol1"}, + {MountPath: "/dir/nested2", Name: "vol3"}, + }, + }, + }, + }, + }, + }, + { + name: "Multiple vol1 mounts Pod", + err: false, + expected: sets.NewString("nested", "nested2"), + volname: "vol1", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/dir", Name: "vol1"}, + {MountPath: "/dir/nested", Name: "vol2"}, + {MountPath: "/ignore", Name: "vol4"}, + {MountPath: "/other", Name: "vol1"}, + {MountPath: "/other/nested2", Name: "vol3"}, + }, + }, + }, + }, + }, + }, + { + name: "Big Pod", + err: false, + volname: "vol1", + expected: sets.NewString("sub1/sub2/sub3", "sub1/sub2/sub4", "sub1/sub2/sub6", "sub"), + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/mnt", Name: "vol1"}, + {MountPath: "/ignore", Name: "vol2"}, + {MountPath: "/mnt/sub1/sub2/sub3", Name: "vol3"}, + {MountPath: "/mnt/sub1/sub2/sub4", Name: "vol4"}, + {MountPath: "/mnt/sub1/sub2/sub4/skip", Name: "vol5"}, + {MountPath: "/mnt/sub1/sub2/sub4/skip2", Name: "vol5a"}, + {MountPath: "/mnt/sub1/sub2/sub6", Name: "vol6"}, + {MountPath: "/mnt7", Name: "vol7"}, + }, + }, + }, + InitContainers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "/mnt/dir", Name: "vol1"}, + {MountPath: "/mnt/dir_ignore", Name: "vol8"}, + {MountPath: "/ignore", Name: "vol9"}, + {MountPath: "/mnt/dir/sub", Name: "vol11"}, + }, + }, + }, + }, + }, + }, + { + name: "Naughty Pod", + err: true, + expected: nil, + volname: "vol1", + pod: v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + UID: testPodUID, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + VolumeMounts: []v1.VolumeMount{ + {MountPath: "foo/../../dir", Name: "vol1"}, + {MountPath: "foo/../../dir/skip", Name: "vol10"}, + }, + }, + }, + }, + }, + }, + } + for _, test := range tc { + dir, err := ioutil.TempDir("", "TestMakeNestedMountpoints.") + if err != nil { + t.Errorf("Unexpected error trying to create temp directory: %v", err) + return + } + defer os.RemoveAll(dir) + + rootdir := path.Join(dir, "vol") + err = os.Mkdir(rootdir, 0755) + if err != nil { + t.Errorf("Unexpected error trying to create temp root directory: %v", err) + return + } + + dirs, err := getNestedMountpoints(test.volname, rootdir, test.pod) + if test.err { + if err == nil { + t.Errorf("%v: expected error, got nil", test.name) + } + continue + } else { + if err != nil { + t.Errorf("%v: expected no error, got %v", test.name, err) + continue + } + } + actual := sets.NewString(dirs...) + if !test.expected.Equal(actual) { + t.Errorf("%v: unexpected nested directories created:\nexpected: %v\n got: %v", test.name, test.expected, actual) + } + } +} diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor.go b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor.go index da95adbba975..a404da0dec58 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor.go @@ -97,7 +97,7 @@ type OperationExecutor interface { // UnmountVolume unmounts the volume from the pod specified in // volumeToUnmount and updates the actual state of the world to reflect that. - UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error + UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) error // UnmountDevice unmounts the volumes global mount path from the device (for // attachable volumes only, freeing it for detach. It then updates the @@ -687,10 +687,11 @@ func (oe *operationExecutor) MountVolume( func (oe *operationExecutor) UnmountVolume( volumeToUnmount MountedVolume, - actualStateOfWorld ActualStateOfWorldMounterUpdater) error { + actualStateOfWorld ActualStateOfWorldMounterUpdater, + podsDir string) error { unmountFunc, plugin, err := - oe.operationGenerator.GenerateUnmountVolumeFunc(volumeToUnmount, actualStateOfWorld) + oe.operationGenerator.GenerateUnmountVolumeFunc(volumeToUnmount, actualStateOfWorld, podsDir) if err != nil { return err } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor_test.go b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor_test.go index 941b7bf88c45..d00bf255e709 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor_test.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_executor_test.go @@ -120,7 +120,7 @@ func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testi PodUID: pod.UID, } } - oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */) + oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */, "" /*podsDir*/) } // Assert @@ -245,7 +245,7 @@ func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout return nil }, "", nil } -func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { +func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (func() error, string, error) { return func() error { startOperationAndBlock(fopg.ch, fopg.quit) return nil diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go index b97ae9630f00..ab149330c79b 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go @@ -18,6 +18,7 @@ package operationexecutor import ( "fmt" + "path" "time" "github.com/golang/glog" @@ -76,7 +77,7 @@ type OperationGenerator interface { GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (func() error, string, error) // Generates the UnmountVolume function needed to perform the unmount of a volume plugin - GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) + GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (func() error, string, error) // Generates the AttachVolume function needed to perform attach of a volume plugin GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (func() error, string, error) @@ -499,7 +500,8 @@ func (og *operationGenerator) GenerateMountVolumeFunc( func (og *operationGenerator) GenerateUnmountVolumeFunc( volumeToUnmount MountedVolume, - actualStateOfWorld ActualStateOfWorldMounterUpdater) (func() error, string, error) { + actualStateOfWorld ActualStateOfWorldMounterUpdater, + podsDir string) (func() error, string, error) { // Get mountable plugin volumePlugin, err := og.volumePluginMgr.FindPluginByName(volumeToUnmount.PluginName) @@ -514,6 +516,14 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc( } return func() error { + mounter := og.volumePluginMgr.Host.GetMounter() + + // Remove all bind-mounts for subPaths + podDir := path.Join(podsDir, string(volumeToUnmount.PodUID)) + if err := mounter.CleanSubPaths(podDir, volumeToUnmount.OuterVolumeSpecName); err != nil { + return volumeToUnmount.GenerateErrorDetailed("error cleaning subPath mounts", err) + } + // Execute unmount unmountErr := volumeUnmounter.TearDown() if unmountErr != nil { diff --git a/vendor/k8s.io/kubernetes/test/e2e/common/downwardapi_volume.go b/vendor/k8s.io/kubernetes/test/e2e/common/downwardapi_volume.go index 2e85ccd64d7b..ef0afc615058 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/common/downwardapi_volume.go +++ b/vendor/k8s.io/kubernetes/test/e2e/common/downwardapi_volume.go @@ -41,7 +41,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide podname only [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podname") + pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), @@ -51,20 +51,20 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should set DefaultMode on files [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) defaultMode := int32(0400) - pod := downwardAPIVolumePodForModeTest(podName, "/etc/podname", nil, &defaultMode) + pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode) f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--------", + "mode of file \"/etc/podinfo/podname\": -r--------", }) }) It("should set mode on item file [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) mode := int32(0400) - pod := downwardAPIVolumePodForModeTest(podName, "/etc/podname", &mode, nil) + pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--------", + "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -72,7 +72,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { podName := "metadata-volume-" + string(uuid.NewUUID()) uid := int64(1001) gid := int64(1234) - pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podname") + pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") pod.Spec.SecurityContext = &v1.PodSecurityContext{ RunAsUser: &uid, FSGroup: &gid, @@ -87,13 +87,13 @@ var _ = framework.KubeDescribe("Downward API volume", func() { uid := int64(1001) gid := int64(1234) mode := int32(0440) /* setting fsGroup sets mode to at least 440 */ - pod := downwardAPIVolumePodForModeTest(podName, "/etc/podname", &mode, nil) + pod := downwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) pod.Spec.SecurityContext = &v1.PodSecurityContext{ RunAsUser: &uid, FSGroup: &gid, } f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--r-----", + "mode of file \"/etc/podinfo/podname\": -r--r-----", }) }) @@ -103,7 +103,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { labels["key2"] = "value2" podName := "labelsupdate" + string(uuid.NewUUID()) - pod := downwardAPIVolumePodForUpdateTest(podName, labels, map[string]string{}, "/etc/labels") + pod := downwardAPIVolumePodForUpdateTest(podName, labels, map[string]string{}, "/etc/podinfo/labels") containerName := "client-container" By("Creating the pod") podClient.CreateSync(pod) @@ -128,7 +128,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { annotations := map[string]string{} annotations["builder"] = "bar" podName := "annotationupdate" + string(uuid.NewUUID()) - pod := downwardAPIVolumePodForUpdateTest(podName, map[string]string{}, annotations, "/etc/annotations") + pod := downwardAPIVolumePodForUpdateTest(podName, map[string]string{}, annotations, "/etc/podinfo/annotations") containerName := "client-container" By("Creating the pod") @@ -155,7 +155,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide container's cpu limit [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_limit") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("2\n"), @@ -164,7 +164,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide container's memory limit [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_limit") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("67108864\n"), @@ -173,7 +173,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide container's cpu request [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_request") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("1\n"), @@ -182,7 +182,7 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide container's memory request [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_request") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("33554432\n"), @@ -191,14 +191,14 @@ var _ = framework.KubeDescribe("Downward API volume", func() { It("should provide node allocatable (cpu) as default cpu limit if the limit is not set [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/cpu_limit") + pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit") f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) }) It("should provide node allocatable (memory) as default memory limit if the limit is not set [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/memory_limit") + pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit") f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) }) @@ -216,7 +216,7 @@ func downwardAPIVolumePodForModeTest(name, filePath string, itemMode, defaultMod VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", }, }, }, @@ -242,7 +242,7 @@ func downwardAPIVolumePodForSimpleTest(name string, filePath string) *v1.Pod { VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", ReadOnly: false, }, }, @@ -283,7 +283,7 @@ func downwardAPIVolumeBaseContainers(name, filePath string) []v1.Container { VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", ReadOnly: false, }, }, @@ -301,7 +301,7 @@ func downwardAPIVolumeDefaultBaseContainer(name, filePath string) []v1.Container VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", }, }, }, @@ -320,7 +320,7 @@ func downwardAPIVolumePodForUpdateTest(name string, labels, annotations map[stri VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", ReadOnly: false, }, }, diff --git a/vendor/k8s.io/kubernetes/test/e2e/common/projected.go b/vendor/k8s.io/kubernetes/test/e2e/common/projected.go index 3af9acb548db..b4f257aac2fa 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/common/projected.go +++ b/vendor/k8s.io/kubernetes/test/e2e/common/projected.go @@ -804,7 +804,7 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide podname only [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podname") + pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("%s\n", podName), @@ -814,20 +814,20 @@ var _ = framework.KubeDescribe("Projected", func() { It("should set DefaultMode on files [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) defaultMode := int32(0400) - pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podname", nil, &defaultMode) + pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", nil, &defaultMode) f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--------", + "mode of file \"/etc/podinfo/podname\": -r--------", }) }) It("should set mode on item file [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) mode := int32(0400) - pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podname", &mode, nil) + pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--------", + "mode of file \"/etc/podinfo/podname\": -r--------", }) }) @@ -835,7 +835,7 @@ var _ = framework.KubeDescribe("Projected", func() { podName := "metadata-volume-" + string(uuid.NewUUID()) uid := int64(1001) gid := int64(1234) - pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podname") + pod := downwardAPIVolumePodForSimpleTest(podName, "/etc/podinfo/podname") pod.Spec.SecurityContext = &v1.PodSecurityContext{ RunAsUser: &uid, FSGroup: &gid, @@ -850,13 +850,13 @@ var _ = framework.KubeDescribe("Projected", func() { uid := int64(1001) gid := int64(1234) mode := int32(0440) /* setting fsGroup sets mode to at least 440 */ - pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podname", &mode, nil) + pod := projectedDownwardAPIVolumePodForModeTest(podName, "/etc/podinfo/podname", &mode, nil) pod.Spec.SecurityContext = &v1.PodSecurityContext{ RunAsUser: &uid, FSGroup: &gid, } f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ - "mode of file \"/etc/podname\": -r--r-----", + "mode of file \"/etc/podinfo/podname\": -r--r-----", }) }) @@ -866,7 +866,7 @@ var _ = framework.KubeDescribe("Projected", func() { labels["key2"] = "value2" podName := "labelsupdate" + string(uuid.NewUUID()) - pod := projectedDownwardAPIVolumePodForUpdateTest(podName, labels, map[string]string{}, "/etc/labels") + pod := projectedDownwardAPIVolumePodForUpdateTest(podName, labels, map[string]string{}, "/etc/podinfo/labels") containerName := "client-container" By("Creating the pod") podClient.CreateSync(pod) @@ -891,7 +891,7 @@ var _ = framework.KubeDescribe("Projected", func() { annotations := map[string]string{} annotations["builder"] = "bar" podName := "annotationupdate" + string(uuid.NewUUID()) - pod := projectedDownwardAPIVolumePodForUpdateTest(podName, map[string]string{}, annotations, "/etc/annotations") + pod := projectedDownwardAPIVolumePodForUpdateTest(podName, map[string]string{}, annotations, "/etc/podinfo/annotations") containerName := "client-container" By("Creating the pod") @@ -918,7 +918,7 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide container's cpu limit [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_limit") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_limit") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("2\n"), @@ -927,7 +927,7 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide container's memory limit [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_limit") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_limit") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("67108864\n"), @@ -936,7 +936,7 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide container's cpu request [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/cpu_request") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/cpu_request") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("1\n"), @@ -945,7 +945,7 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide container's memory request [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForContainerResources(podName, "/etc/memory_request") + pod := downwardAPIVolumeForContainerResources(podName, "/etc/podinfo/memory_request") f.TestContainerOutput("downward API volume plugin", pod, 0, []string{ fmt.Sprintf("33554432\n"), @@ -954,14 +954,14 @@ var _ = framework.KubeDescribe("Projected", func() { It("should provide node allocatable (cpu) as default cpu limit if the limit is not set [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/cpu_limit") + pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/cpu_limit") f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) }) It("should provide node allocatable (memory) as default memory limit if the limit is not set [Conformance] [Volume]", func() { podName := "downwardapi-volume-" + string(uuid.NewUUID()) - pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/memory_limit") + pod := downwardAPIVolumeForDefaultContainerResources(podName, "/etc/podinfo/memory_limit") f.TestContainerOutputRegexp("downward API volume plugin", pod, 0, []string{"[1-9]"}) }) @@ -1373,7 +1373,7 @@ func projectedDownwardAPIVolumePodForModeTest(name, filePath string, itemMode, d VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", }, }, }, @@ -1399,7 +1399,7 @@ func projectedDownwardAPIVolumePodForUpdateTest(name string, labels, annotations VolumeMounts: []v1.VolumeMount{ { Name: "podinfo", - MountPath: "/etc", + MountPath: "/etc/podinfo", ReadOnly: false, }, },