diff --git a/tools/integration_tests/local_file/read_dir_test.go b/tools/integration_tests/local_file/read_dir_test.go new file mode 100644 index 0000000000..b94c76c818 --- /dev/null +++ b/tools/integration_tests/local_file/read_dir_test.go @@ -0,0 +1,149 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for readDir call containing local files. +package local_file_test + +import ( + "io/fs" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestReadDir(t *testing.T) { + // Structure + // mntDir/ + // mntDir/explicit/ --- directory + // mntDir/explicit/explicitFile1 --- file + // mntDir/foo1 --- empty local file + // mntDir/foo2 --- non empty local file + // mntDir/foo3 --- gcs synced file + + testDirPath = setup.SetupTestDirectory(testDirName) + // Create explicit dir with 1 local file. + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, + path.Join(ExplicitDirName, ExplicitFileName1), t) + // Create empty local file. + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + // Create non-empty local file. + _, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t) + WritingToLocalFileShouldNotWriteToGCS(ctx, storageClient, fh3, testDirName, FileName2, t) + // Create GCS synced file. + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName3, GCSFileContent, t) + + // Attempt to list mnt and explicit directory. + entriesMnt := operations.ReadDirectory(testDirPath, t) + entriesDir := operations.ReadDirectory(path.Join(testDirPath, ExplicitDirName), t) + + // Verify entriesMnt received successfully. + operations.VerifyCountOfDirectoryEntries(4, len(entriesMnt), t) + operations.VerifyDirectoryEntry(entriesMnt[0], ExplicitDirName, t) + operations.VerifyFileEntry(entriesMnt[1], FileName1, 0, t) + operations.VerifyFileEntry(entriesMnt[2], FileName2, SizeOfFileContents, t) + operations.VerifyFileEntry(entriesMnt[3], FileName3, GCSFileSize, t) + // Verify entriesDir received successfully. + operations.VerifyCountOfDirectoryEntries(1, len(entriesDir), t) + operations.VerifyFileEntry(entriesDir[0], ExplicitFileName1, 0, t) + // Close the local files. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, + path.Join(ExplicitDirName, ExplicitFileName1), "", t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, + FileName1, "", t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh3, testDirName, + FileName2, FileContents, t) + ValidateObjectContentsFromGCS(ctx, storageClient, testDirName, FileName3, + GCSFileContent, t) +} + +func TestRecursiveListingWithLocalFiles(t *testing.T) { + // Structure + // mntDir/ + // mntDir/foo1 --- file + // mntDir/explicit/ --- directory + // mntDir/explicit/explicitFile1 --- file + + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local file in mnt/ dir. + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + // Create explicit dir with 1 local file. + operations.CreateDirectory(path.Join(testDirPath, ExplicitDirName), t) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, + path.Join(ExplicitDirName, ExplicitFileName1), t) + + // Recursively list mntDir/ directory. + err := filepath.WalkDir(testDirPath, func(walkPath string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + // The object type is not directory. + if !dir.IsDir() { + return nil + } + + objs := operations.ReadDirectory(walkPath, t) + + // Check if mntDir has correct objects. + if walkPath == testDirPath { + // numberOfObjects = 2 + operations.VerifyCountOfDirectoryEntries(2, len(objs), t) + operations.VerifyDirectoryEntry(objs[0], ExplicitDirName, t) + operations.VerifyFileEntry(objs[1], FileName1, 0, t) + } + + // Check if mntDir/explicit/ has correct objects. + if walkPath == path.Join(setup.MntDir(), ExplicitDirName) { + // numberOfObjects = 1 + operations.VerifyCountOfDirectoryEntries(1, len(objs), t) + operations.VerifyFileEntry(objs[0], ExplicitFileName1, 0, t) + } + + return nil + }) + + // Validate and close the files. + if err != nil { + t.Fatalf("filepath.WalkDir() err: %v", err) + } + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, + FileName1, "", t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, + path.Join(ExplicitDirName, ExplicitFileName1), "", t) +} + +func TestReadDirWithSameNameLocalAndGCSFile(t *testing.T) { + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local file. + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + // Create same name gcs file. + time.Sleep(2 * time.Second) + CreateObjectInGCSTestDir(ctx, storageClient, testDirName, FileName1, GCSFileContent, t) + + // Attempt to list testDir. + _, err := os.ReadDir(testDirPath) + if err == nil || !strings.Contains(err.Error(), "input/output error") { + t.Fatalf("Expected error: %s, Got error: %v", "input/output error", err) + } + + // Close the local file. + operations.CloseFileShouldNotThrowError(fh1, t) +} diff --git a/tools/integration_tests/local_file/unlinked_file_test.go b/tools/integration_tests/local_file/unlinked_file_test.go new file mode 100644 index 0000000000..e8e41984f4 --- /dev/null +++ b/tools/integration_tests/local_file/unlinked_file_test.go @@ -0,0 +1,101 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Provides integration tests for operation on unlinked local files. +package local_file_test + +import ( + "path" + "testing" + + . "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/operations" + "github.com/googlecloudplatform/gcsfuse/tools/integration_tests/util/setup" +) + +func TestStatOnUnlinkedLocalFile(t *testing.T) { + testDirPath = setup.SetupTestDirectory(testDirName) + // Create a local file. + filePath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + // Unlink the local file. + operations.RemoveFile(filePath) + + // Stat the local file and validate error. + operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + + // Close the file and validate that file is not created on GCS. + operations.CloseFileShouldNotThrowError(fh, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) +} + +func TestReadDirContainingUnlinkedLocalFiles(t *testing.T) { + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local files. + _, fh1 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + _, fh2 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName2, t) + filepath3, fh3 := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName3, t) + // Unlink local file 3. + operations.RemoveFile(filepath3) + + // Attempt to list testDir. + entries := operations.ReadDirectory(testDirPath, t) + + // Verify unlinked entries are not listed. + operations.VerifyCountOfDirectoryEntries(2, len(entries), t) + operations.VerifyFileEntry(entries[0], FileName1, 0, t) + operations.VerifyFileEntry(entries[1], FileName2, 0, t) + // Close the local files and validate they are written to GCS. + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh1, testDirName, + FileName1, "", t) + CloseFileAndValidateContentFromGCS(ctx, storageClient, fh2, testDirName, + FileName2, "", t) + // Verify unlinked file is not written to GCS. + operations.CloseFileShouldNotThrowError(fh3, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName3, t) +} + +func TestWriteOnUnlinkedLocalFileSucceeds(t *testing.T) { + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local file. + filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + // Verify unlink operation succeeds. + operations.RemoveFile(filepath) + operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + + // Write to unlinked local file. + operations.WriteWithoutClose(fh, FileContents, t) + + // Validate flush file does not throw error. + operations.CloseFileShouldNotThrowError(fh, t) + // Validate unlinked file is not written to GCS. + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) +} + +func TestSyncOnUnlinkedLocalFile(t *testing.T) { + testDirPath = setup.SetupTestDirectory(testDirName) + // Create local file. + filepath, fh := CreateLocalFileInTestDir(ctx, storageClient, testDirPath, FileName1, t) + + // Attempt to unlink local file. + operations.RemoveFile(filepath) + + // Verify unlink operation succeeds. + operations.ValidateNoFileOrDirError(path.Join(testDirPath, FileName1), t) + // Validate sync operation does not write to GCS after unlink. + operations.SyncFile(fh, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) + // Close the local file and validate it is not present on GCS. + operations.CloseFileShouldNotThrowError(fh, t) + ValidateObjectNotFoundErrOnGCS(ctx, storageClient, testDirName, FileName1, t) +} diff --git a/tools/integration_tests/util/client/gcs_helper.go b/tools/integration_tests/util/client/gcs_helper.go index 64cc38ead9..281feb4bef 100644 --- a/tools/integration_tests/util/client/gcs_helper.go +++ b/tools/integration_tests/util/client/gcs_helper.go @@ -28,6 +28,7 @@ import ( const ( FileName1 = "foo1" FileName2 = "foo2" + FileName3 = "foo3" ExplicitDirName = "explicit" ExplicitFileName1 = "explicitFile1" ImplicitDirName = "implicit" diff --git a/tools/integration_tests/util/operations/file_operations.go b/tools/integration_tests/util/operations/file_operations.go index d57f164e33..77b9bff06b 100644 --- a/tools/integration_tests/util/operations/file_operations.go +++ b/tools/integration_tests/util/operations/file_operations.go @@ -548,3 +548,12 @@ func CloseFileShouldNotThrowError(file *os.File, t *testing.T) { t.Fatalf("file.Close() for file %s: %v", file.Name(), err) } } + +func SyncFile(fh *os.File, t *testing.T) { + err := fh.Sync() + + // Verify fh.Sync operation succeeds. + if err != nil { + t.Fatalf("%s.Sync(): %v", fh.Name(), err) + } +}