Read directory and unlink local file integration test (#1386)

* integration tests to verify readDir and unlink operations on local file

rebase changes

* rebase changes

* Empty-Commit

tools/integration_tests/local_file/read_dir_test.go

@@ -0,0 +1,149 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for readDir call containing local files.
+package local_file_test
+
+import (
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestReadDir(t *testing.T) {
+	// Structure
+	// mntDir/
+	// mntDir/explicit/		    				--- directory
+	// mntDir/explicit/explicitFile1  --- file
+	// mntDir/foo1 										--- empty local file
+	// mntDir/foo2  									--- non empty local file
+	// mntDir/foo3										--- gcs synced file
+
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create explicit dir with 1 local file.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath,
+		path.Join(ExplicitDirName, ExplicitFileName1), t)
+	// Create empty local file.
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	// Create non-empty local file.
+	_, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t)
+	WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh3, testDirName, FileName2, t)
+	// Create GCS synced file.
+	CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName3, GCSFileContent, t)
+
+	// Attempt to list mnt and explicit directory.
+	entriesMnt := operations.ReadDirectory(testDirPath, t)
+	entriesDir := operations.ReadDirectory(path.Join(testDirPath, ExplicitDirName), t)
+
+	// Verify entriesMnt received successfully.
+	operations.VerifyCountOfDirectoryEntries(4, len(entriesMnt), t)
+	operations.VerifyDirectoryEntry(entriesMnt[0], ExplicitDirName, t)
+	operations.VerifyFileEntry(entriesMnt[1], FileName1, 0, t)
+	operations.VerifyFileEntry(entriesMnt[2], FileName2, SizeOfFileContents, t)
+	operations.VerifyFileEntry(entriesMnt[3], FileName3, GCSFileSize, t)
+	// Verify entriesDir received successfully.
+	operations.VerifyCountOfDirectoryEntries(1, len(entriesDir), t)
+	operations.VerifyFileEntry(entriesDir[0], ExplicitFileName1, 0, t)
+	// Close the local files.
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName,
+		path.Join(ExplicitDirName, ExplicitFileName1), "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName,
+		FileName1, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName,
+		FileName2, FileContents, t)
+	ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, FileName3,
+		GCSFileContent, t)
+}
+
+func TestRecursiveListingWithLocalFiles(t *testing.T) {
+	// Structure
+	// mntDir/
+	// mntDir/foo1 										--- file
+	// mntDir/explicit/		    				--- directory
+	// mntDir/explicit/explicitFile1  --- file
+
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local file in mnt/ dir.
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	// Create explicit dir with 1 local file.
+	operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath,
+		path.Join(ExplicitDirName, ExplicitFileName1), t)
+
+	// Recursively list mntDir/ directory.
+	err := filepath.WalkDir(testDirPath, func(walkPath string, dir fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// The object type is not directory.
+		if !dir.IsDir() {
+			return nil
+		}
+
+		objs := operations.ReadDirectory(walkPath, t)
+
+		// Check if mntDir has correct objects.
+		if walkPath == testDirPath {
+			// numberOfObjects = 2
+			operations.VerifyCountOfDirectoryEntries(2, len(objs), t)
+			operations.VerifyDirectoryEntry(objs[0], ExplicitDirName, t)
+			operations.VerifyFileEntry(objs[1], FileName1, 0, t)
+		}
+
+		// Check if mntDir/explicit/ has correct objects.
+		if walkPath == path.Join(setup.MntDir(), ExplicitDirName) {
+			// numberOfObjects = 1
+			operations.VerifyCountOfDirectoryEntries(1, len(objs), t)
+			operations.VerifyFileEntry(objs[0], ExplicitFileName1, 0, t)
+		}
+
+		return nil
+	})
+
+	// Validate and close the files.
+	if err != nil {
+		t.Fatalf("filepath.WalkDir() err: %v", err)
+	}
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName,
+		FileName1, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName,
+		path.Join(ExplicitDirName, ExplicitFileName1), "", t)
+}
+
+func TestReadDirWithSameNameLocalAndGCSFile(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local file.
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	// Create same name gcs file.
+	time.Sleep(2 * time.Second)
+	CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t)
+
+	// Attempt to list testDir.
+	_, err := os.ReadDir(testDirPath)
+	if err == nil || !strings.Contains(err.Error(), "input/output error") {
+		t.Fatalf("Expected error: %s, Got error: %v", "input/output error", err)
+	}
+
+	// Close the local file.
+	operations.CloseFileShouldNotThrowError(fh1, t)
+}

tools/integration_tests/local_file/unlinked_file_test.go

@@ -0,0 +1,101 @@
+// Copyright 2023 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Provides integration tests for operation on unlinked local files.
+package local_file_test
+
+import (
+	"path"
+	"testing"
+
+	. "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations"
+	"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup"
+)
+
+func TestStatOnUnlinkedLocalFile(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create a local file.
+	filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	// Unlink the local file.
+	operations.RemoveFile(filePath)
+
+	// Stat the local file and validate error.
+	operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t)
+
+	// Close the file and validate that file is not created on GCS.
+	operations.CloseFileShouldNotThrowError(fh, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t)
+}
+
+func TestReadDirContainingUnlinkedLocalFiles(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local files.
+	_, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	_, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t)
+	filepath3, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName3, t)
+	// Unlink local file 3.
+	operations.RemoveFile(filepath3)
+
+	// Attempt to list testDir.
+	entries := operations.ReadDirectory(testDirPath, t)
+
+	// Verify unlinked entries are not listed.
+	operations.VerifyCountOfDirectoryEntries(2, len(entries), t)
+	operations.VerifyFileEntry(entries[0], FileName1, 0, t)
+	operations.VerifyFileEntry(entries[1], FileName2, 0, t)
+	// Close the local files and validate they are written to GCS.
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName,
+		FileName1, "", t)
+	CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName,
+		FileName2, "", t)
+	// Verify unlinked file is not written to GCS.
+	operations.CloseFileShouldNotThrowError(fh3, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName3, t)
+}
+
+func TestWriteOnUnlinkedLocalFileSucceeds(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local file.
+	filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+	// Verify unlink operation succeeds.
+	operations.RemoveFile(filepath)
+	operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t)
+
+	// Write to unlinked local file.
+	operations.WriteWithoutClose(fh, FileContents, t)
+
+	// Validate flush file does not throw error.
+	operations.CloseFileShouldNotThrowError(fh, t)
+	// Validate unlinked file is not written to GCS.
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t)
+}
+
+func TestSyncOnUnlinkedLocalFile(t *testing.T) {
+	testDirPath = setup.SetupTestDirectory(testDirName)
+	// Create local file.
+	filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t)
+
+	// Attempt to unlink local file.
+	operations.RemoveFile(filepath)
+
+	// Verify unlink operation succeeds.
+	operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t)
+	// Validate sync operation does not write to GCS after unlink.
+	operations.SyncFile(fh, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t)
+	// Close the local file and validate it is not present on GCS.
+	operations.CloseFileShouldNotThrowError(fh, t)
+	ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t)
+}

tools/integration_tests/util/client/gcs_helper.go

@@ -28,6 +28,7 @@ import (
 const (
 	FileName1          = "foo1"
 	FileName2          = "foo2"
+	FileName3          = "foo3"
 	ExplicitDirName    = "explicit"
 	ExplicitFileName1  = "explicitFile1"
 	ImplicitDirName    = "implicit"

tools/integration_tests/util/operations/file_operations.go

@@ -548,3 +548,12 @@ func CloseFileShouldNotThrowError(file *os.File, t *testing.T) {
 		t.Fatalf("file.Close() for file %s: %v", file.Name(), err)
 	}
 }
+
+func SyncFile(fh *os.File, t *testing.T) {
+	err := fh.Sync()
+
+	// Verify fh.Sync operation succeeds.
+	if err != nil {
+		t.Fatalf("%s.Sync(): %v", fh.Name(), err)
+	}
+}