Skip to content

Commit

Permalink
Integration tests setup (#1013)
Browse files Browse the repository at this point in the history 
* Integration tests

* Integration tests lint fix

* Integration tests lint fix

* keeping only setup

* making flags as 2D variable

* adding getter setter for variable

* formating the code

* changing variable name

* formating code

* fixing comments

* fixing lint test

* fixing comments

* fixing lint test

* fixing lint test

* fixing lint test

* fixing comments
  • Loading branch information
@Tulsishah
Tulsishah committed Mar 21, 2023
1 parent 6dc99f6 commit fdd1b8a
Show file tree
Hide file tree
Showing 5 changed files with 286 additions and 211 deletions.
21 changes: 12 additions & 9 deletions tools/integration_tests/implicitdir/file_operations_test.go
View file
Expand Up @@ -21,14 +21,17 @@ import (
"path"
"testing"
"time"

"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/setup"
)

func TestRenameFile(t *testing.T) {
fileName := createTempFile()
err := clearKernelCache()
fileName := setup.CreateTempFile()
err := setup.ClearKernelCache()
if err != nil {
t.Errorf("Clear Kernel Cache: %v", err)
}

content, err := os.ReadFile(fileName)
if err != nil {
t.Errorf("Read: %v", err)
Expand All @@ -49,20 +52,20 @@ func TestRenameFile(t *testing.T) {
t.Errorf("Renamed file %s not found", newFileName)
}
// Check if the data in the file is the same after renaming.
compareFileContents(t, newFileName, string(content))
setup.CompareFileContents(t, newFileName, string(content))
}

func TestFileAttributes(t *testing.T) {
preCreateTime := time.Now()
fileName := createTempFile()
fileName := setup.CreateTempFile()
postCreateTime := time.Now()

fStat, err := os.Stat(fileName)

if err != nil {
t.Errorf("os.Stat error: %s, %v", fileName, err)
}
statFileName := path.Join(tmpDir, fStat.Name())
statFileName := path.Join(setup.TmpDir(), fStat.Name())
if fileName != statFileName {
t.Errorf("File name not matched in os.Stat, found: %s, expected: %s", statFileName, fileName)
}
Expand All @@ -76,8 +79,8 @@ func TestFileAttributes(t *testing.T) {
}

func TestCopyFile(t *testing.T) {
fileName := createTempFile()
err := clearKernelCache()
fileName := setup.CreateTempFile()
err := setup.ClearKernelCache()
if err != nil {
t.Errorf("Clear Kernel Cache: %v", err)
}
Expand Down Expand Up @@ -108,6 +111,6 @@ func TestCopyFile(t *testing.T) {

// Check if the data in the copied file matches the original file,
// and the data in original file is unchanged.
compareFileContents(t, newFileName, string(content))
compareFileContents(t, fileName, string(content))
setup.CompareFileContents(t, newFileName, string(content))
setup.CompareFileContents(t, fileName, string(content))
}
204 changes: 13 additions & 191 deletions tools/integration_tests/implicitdir/implicitdir_test.go
View file
Expand Up @@ -16,221 +16,43 @@
package implicitdir_test

import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"testing"

"github.com/googlecloudplatform/gcsfuse/tools/util"
"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/setup"
)

var testBucket = flag.String("testbucket", "", "The GCS bucket used for the test.")
var mountedDirectory = flag.String("mountedDirectory", "", "The GCSFuse mounted directory used for the test.")

var (
binFile string
logFile string
mntDir string
testDir string
tmpDir string
)

func setUpTestDir() error {
var err error
testDir, err = ioutil.TempDir("", "gcsfuse_readwrite_test_")
if err != nil {
return fmt.Errorf("TempDir: %w\n", err)
}

err = util.BuildGcsfuse(testDir)
if err != nil {
return fmt.Errorf("BuildGcsfuse(%q): %w\n", testDir, err)
}

binFile = path.Join(testDir, "bin/gcsfuse")
logFile = path.Join(testDir, "gcsfuse.log")
mntDir = path.Join(testDir, "mnt")

err = os.Mkdir(mntDir, 0755)
if err != nil {
return fmt.Errorf("Mkdir(%q): %v\n", mntDir, err)
}
return nil
}

func mountGcsfuse(flag string) error {
mountCmd := exec.Command(
binFile,
"--debug_gcs",
"--debug_fs",
"--debug_fuse",
"--log-file="+logFile,
"--log-format=text",
flag,
*testBucket,
mntDir,
)

// Adding mount command in logFile
file, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println("Could not open logfile")
}
defer file.Close()

_, err = file.WriteString(mountCmd.String() + "\n")
if err != nil {
fmt.Println("Could not write cmd to logFile")
}

output, err := mountCmd.CombinedOutput()
if err != nil {
log.Println(mountCmd.String())
return fmt.Errorf("cannot mount gcsfuse: %w\n", err)
}
if lines := bytes.Count(output, []byte{'\n'}); lines > 1 {
return fmt.Errorf("mount output: %q\n", output)
}
return nil
}

func unMount() error {
fusermount, err := exec.LookPath("fusermount")
if err != nil {
return fmt.Errorf("cannot find fusermount: %w", err)
}
cmd := exec.Command(fusermount, "-uz", mntDir)
if _, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("fusermount error: %w", err)
}
return nil
}

func clearKernelCache() error {
if _, err := os.Stat("/proc/sys/vm/drop_caches"); err != nil {
log.Printf("Kernel cache file not found: %v", err)
// No need to stop the test execution if cache file is not found. Further
// reads will be served from kernel cache.
return nil
}

// sudo permission is required to clear kernel page cache.
cmd := exec.Command("sudo", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches")
if err := cmd.Run(); err != nil {
return fmt.Errorf("clear kernel cache failed with error: %w", err)
}
return nil
}

func compareFileContents(t *testing.T, fileName string, fileContent string) {
// After write, data will be cached by kernel. So subsequent read will be
// served using cached data by kernel instead of calling gcsfuse.
// Clearing kernel cache to ensure that gcsfuse is invoked during read operation.
err := clearKernelCache()
if err != nil {
t.Errorf("Clear Kernel Cache: %v", err)
}

content, err := os.ReadFile(fileName)
if err != nil {
t.Errorf("Read: %v", err)
}

if got := string(content); got != fileContent {
t.Errorf("File content doesn't match. Expected: %q, Actual: %q", got, fileContent)
}
}

func logAndExit(s string) {
log.Print(s)
os.Exit(1)
}

func createTempFile() string {
// A temporary file is created and some lines are added
// to it for testing purposes.
fileName := path.Join(tmpDir, "tmpFile")
err := os.WriteFile(fileName, []byte("line 1\nline 2\n"), 0666)
if err != nil {
logAndExit(fmt.Sprintf("Temporary file at %v", err))
}
return fileName
}

func executeTest(m *testing.M) (successCode int) {
// Creating a temporary directory to store files
// to be used for testing.
var err error
tmpDir, err = os.MkdirTemp(mntDir, "tmpDir")
if err != nil {
logAndExit(fmt.Sprintf("Mkdir at %q: %v", mntDir, err))
}

successCode = m.Run()

os.RemoveAll(mntDir)

return successCode
}

func executeTestForFlags(flags []string, m *testing.M) (successCode int) {
var err error
for i := 0; i < len(flags); i++ {
if err = mountGcsfuse(flags[i]); err != nil {
logAndExit(fmt.Sprintf("mountGcsfuse: %v\n", err))
}

successCode = executeTest(m)

err = unMount()
if err != nil {
logAndExit(fmt.Sprintf("Error in unmounting bucket: %v", err))
}

// Print flag on which test fails
if successCode != 0 {
log.Print("Test Fails on " + flags[i])
return
}

}
return
}

func TestMain(m *testing.M) {
flag.Parse()

if *testBucket == "" && *mountedDirectory == "" {
if setup.TestBucket() == "" && setup.MountedDirectory() == "" {
log.Printf("--testbucket or --mountedDirectory must be specified")
os.Exit(0)
} else if *testBucket != "" && *mountedDirectory != "" {
} else if setup.TestBucket() != "" && setup.MountedDirectory() != "" {
log.Printf("Both --testbucket and --mountedDirectory can't be specified at the same time.")
os.Exit(0)
}

if *mountedDirectory != "" {
mntDir = *mountedDirectory
successCode := executeTest(m)
if setup.MountedDirectory() != "" {
setup.SetMntDir(setup.MountedDirectory())
successCode := setup.ExecuteTest(m)
os.Exit(successCode)
}

if err := setUpTestDir(); err != nil {
if err := setup.SetUpTestDir(); err != nil {
log.Printf("setUpTestDir: %v\n", err)
os.Exit(1)
}

flags := []string{"--enable-storage-client-library=true",
"--enable-storage-client-library=false",
"--implicit-dirs=true",
"--implicit-dirs=false"}
flags := [][]string{{"--enable-storage-client-library=true", "--implicit-dirs=true"},
{"--enable-storage-client-library=false"},
{"--implicit-dirs=true"},
{"--implicit-dirs=false"}}

successCode := executeTestForFlags(flags, m)
successCode := setup.ExecuteTestForFlags(flags, m)

log.Printf("Test log: %s\n", logFile)
log.Printf("Test log: %s\n", setup.LogFile())
os.Exit(successCode)
}
12 changes: 9 additions & 3 deletions tools/integration_tests/implicitdir/read_test.go
View file
Expand Up @@ -19,12 +19,14 @@ import (
"io/ioutil"
"os"
"testing"

"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/setup"
)

func TestReadAfterWrite(t *testing.T) {
tmpDir, err := ioutil.TempDir(mntDir, "tmpDir")
tmpDir, err := ioutil.TempDir(setup.MntDir(), "tmpDir")
if err != nil {
t.Errorf("Mkdir at %q: %v", mntDir, err)
t.Errorf("Mkdir at %q: %v", setup.MntDir(), err)
return
}

Expand All @@ -46,7 +48,11 @@ func TestReadAfterWrite(t *testing.T) {
// After write, data will be cached by kernel. So subsequent read will be
// served using cached data by kernel instead of calling gcsfuse.
// Clearing kernel cache to ensure that gcsfuse is invoked during read operation.
clearKernelCache()
err = setup.ClearKernelCache()
if err != nil {
t.Errorf("Clear Kernel Cache: %v", err)
}

tmpFile, err = os.Open(fileName)
if err != nil {
t.Errorf("Open %q: %v", fileName, err)
Expand Down
18 changes: 10 additions & 8 deletions tools/integration_tests/implicitdir/write_test.go
View file
Expand Up @@ -18,10 +18,12 @@ package implicitdir_test
import (
"os"
"testing"

"github.com/googlecloudplatform/gcsfuse/tools/integration_tests/setup"
)

func TestWriteAtEndOfFile(t *testing.T) {
fileName := createTempFile()
fileName := setup.CreateTempFile()
f, err := os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
t.Errorf("Open file for append: %v", err)
Expand All @@ -32,11 +34,11 @@ func TestWriteAtEndOfFile(t *testing.T) {
}
f.Close()

compareFileContents(t, fileName, "line 1\nline 2\nline 3\n")
setup.CompareFileContents(t, fileName, "line 1\nline 2\nline 3\n")
}

func TestWriteAtStartOfFile(t *testing.T) {
fileName := createTempFile()
fileName := setup.CreateTempFile()
f, err := os.OpenFile(fileName, os.O_WRONLY, 0600)
if err != nil {
t.Errorf("Open file for write at start: %v", err)
Expand All @@ -47,11 +49,11 @@ func TestWriteAtStartOfFile(t *testing.T) {
}
f.Close()

compareFileContents(t, fileName, "line 4\nline 2\n")
setup.CompareFileContents(t, fileName, "line 4\nline 2\n")
}

func TestWriteAtRandom(t *testing.T) {
fileName := createTempFile()
fileName := setup.CreateTempFile()

f, err := os.OpenFile(fileName, os.O_WRONLY, 0600)
if err != nil {
Expand All @@ -65,16 +67,16 @@ func TestWriteAtRandom(t *testing.T) {
}
f.Close()

compareFileContents(t, fileName, "line 1\nline 5\n")
setup.CompareFileContents(t, fileName, "line 1\nline 5\n")
}

func TestCreateFile(t *testing.T) {
fileName := createTempFile()
fileName := setup.CreateTempFile()

// Stat the file to check if it exists.
if _, err := os.Stat(fileName); err != nil {
t.Errorf("File not found, %v", err)
}

compareFileContents(t, fileName, "line 1\nline 2\n")
setup.CompareFileContents(t, fileName, "line 1\nline 2\n")
}

0 comments on commit fdd1b8a

Please sign in to comment.