Changes to allow parallel dirops and lookups

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.12.3
 	github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
-	github.com/jacobsa/fuse v0.0.0-20231003132804-d0f3daf365c3
+	github.com/jacobsa/fuse v0.0.0-20240419171848-edf18a690d4b
 	github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd
 	github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff
 	github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11
@@ -21,7 +21,7 @@ require (
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
 	github.com/urfave/cli v1.22.14
 	go.opencensus.io v0.24.0
-	golang.org/x/net v0.22.0
+	golang.org/x/net v0.23.0
 	golang.org/x/oauth2 v0.18.0
 	golang.org/x/sync v0.6.0
 	golang.org/x/sys v0.18.0

go.sum

@@ -716,6 +716,8 @@ github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f h1:X+tnaqoCcBgAw
 github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f/go.mod h1:Ip4fOwzCrnDVuluHBd7FXIMb7SHOKfkt9/UDrYSZvqI=
 github.com/jacobsa/fuse v0.0.0-20231003132804-d0f3daf365c3 h1:NhlhxaD3w7/+ztfJzZHWJ9FeNPl3kDYQPXQ/arbXAbA=
 github.com/jacobsa/fuse v0.0.0-20231003132804-d0f3daf365c3/go.mod h1:XUKuYy1M4vamyxQjW8/WZBTxyZ0NnUiq+kkA+WWOfeI=
+github.com/jacobsa/fuse v0.0.0-20240419171848-edf18a690d4b h1:WelCHTd/WEv4E2XCARZwmQpKlVDxInTPLZROxVgRktI=
+github.com/jacobsa/fuse v0.0.0-20240419171848-edf18a690d4b/go.mod h1:JYi9iIxdYNgxmMgLwtSHO/hmVnP2kfX1oc+mtx+XWLA=
 github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd h1:9GCSedGjMcLZCrusBZuo4tyKLpKUPenUUqi34AkuFmA=
 github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M=
 github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff h1:2xRHTvkpJ5zJmglXLRqHiZQNjUoOkhUyhTAhEQvPAWw=
@@ -1347,6 +1349,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
 golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/config/mount_config.go

@@ -83,7 +83,8 @@ type EnableHNS bool
 type CacheDir string
 
 type FileSystemConfig struct {
-	IgnoreInterrupts bool `yaml:"ignore-interrupts"`
+	IgnoreInterrupts      bool `yaml:"ignore-interrupts"`
+	DisableParallelDirops bool `yaml:"disable-parallel-dirops"`
 }
 
 type FileCacheConfig struct {
@@ -169,6 +170,5 @@ func NewMountConfig() *MountConfig {
 		ConnPoolSize: DefaultGrpcConnPoolSize,
 	}
 	mountConfig.EnableHNS = DefaultEnableHNS
-
 	return mountConfig
 }

internal/config/testdata/file_system_config/invalid_disable_parallel_dirops.yaml

@@ -0,0 +1,2 @@
+file-system:
+  disable-parallel-dirops: -1

internal/config/testdata/file_system_config/unset_disable_parallel_dirops.yaml

@@ -0,0 +1,2 @@
+write:
+  create-empty-file: true

internal/config/testdata/valid_config.yaml

@@ -23,3 +23,4 @@ grpc:
 enable-hns: true
 file-system:
   ignore-interrupts: true
+  disable-parallel-dirops: true

internal/config/yaml_parser_test.go

@@ -46,6 +46,7 @@ func validateDefaultConfig(mountConfig *MountConfig) {
 	ExpectEq(1, mountConfig.GrpcClientConfig.ConnPoolSize)
 	ExpectEq(false, mountConfig.EnableHNS)
 	ExpectFalse(mountConfig.FileSystemConfig.IgnoreInterrupts)
+	ExpectEq(false, mountConfig.FileSystemConfig.DisableParallelDirops)
 }
 
 func (t *YamlParserTest) TestReadConfigFile_EmptyFileName() {
@@ -126,6 +127,7 @@ func (t *YamlParserTest) TestReadConfigFile_ValidConfig() {
 
 	// file-system config
 	ExpectTrue(mountConfig.FileSystemConfig.IgnoreInterrupts)
+	ExpectTrue(mountConfig.FileSystemConfig.DisableParallelDirops)
 }
 
 func (t *YamlParserTest) TestReadConfigFile_InvalidLogConfig() {
@@ -247,3 +249,18 @@ func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_UnsetIgnoreInterrup
 	AssertNe(nil, mountConfig)
 	AssertEq(false, mountConfig.FileSystemConfig.IgnoreInterrupts)
 }
+
+func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_InvalidDisableParallelDirops() {
+	_, err := ParseConfigFile("testdata/file_system_config/invalid_disable_parallel_dirops.yaml")
+
+	AssertNe(nil, err)
+	AssertTrue(strings.Contains(err.Error(), "error parsing config file: yaml: unmarshal errors:\n  line 2: cannot unmarshal !!int `-1` into bool"))
+}
+
+func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_UnsetDisableParallelDirops() {
+	mountConfig, err := ParseConfigFile("testdata/file_system_config/unset_disable_parallel_dirops.yaml")
+
+	AssertEq(nil, err)
+	AssertNe(nil, mountConfig)
+	AssertEq(false, mountConfig.FileSystemConfig.DisableParallelDirops)
+}

internal/fs/fs.go

@@ -923,8 +923,13 @@ func (fs *fileSystem) lookUpOrCreateChildInode(
 	// Set up a function that will find a lookup result for the child with the
 	// given name. Expects no locks to be held.
 	getLookupResult := func() (*inode.Core, error) {
-		parent.Lock()
-		defer parent.Unlock()
+		if fs.mountConfig.FileSystemConfig.DisableParallelDirops {
+			parent.Lock()
+			defer parent.Unlock()
+		} else {
+			parent.LockForChildLookup()
+			defer parent.UnlockForChildLookup()
+		}
 		return parent.LookUpChild(ctx, childName)
 	}
 

internal/fs/inode/base_dir.go

@@ -92,6 +92,14 @@ func (d *baseDirInode) Unlock() {
 	d.mu.Unlock()
 }
 
+func (d *baseDirInode) LockForChildLookup() {
+	d.mu.Lock()
+}
+
+func (d *baseDirInode) UnlockForChildLookup() {
+	d.mu.Unlock()
+}
+
 func (d *baseDirInode) ID() fuseops.InodeID {
 	return d.id
 }

internal/fs/inode/dir.go

@@ -132,6 +132,9 @@ type DirInode interface {
 	// LocalFileEntries lists the local files present in the directory.
 	// Local means that the file is not yet present on GCS.
 	LocalFileEntries(localFileInodes map[Name]Inode) (localEntries []fuseutil.Dirent)
+
+	LockForChildLookup()
+	UnlockForChildLookup()
 }
 
 // An inode that represents a directory from a GCS bucket.
@@ -168,9 +171,9 @@ type dirInode struct {
 	// Mutable state
 	/////////////////////////
 
-	// A mutex that must be held when calling certain methods. See documentation
-	// for each method.
-	mu locker.Locker
+	// A RW mutex that must be held when calling certain methods. See
+	// documentation for each method.
+	mu locker.RWLocker
 
 	// GUARDED_BY(mu)
 	lc lookupCount
@@ -235,7 +238,7 @@ func NewDirInode(
 	typed.lc.Init(id)
 
 	// Set up invariant checking.
-	typed.mu = locker.New(name.GcsObjectName(), typed.checkInvariants)
+	typed.mu = locker.NewRW(name.GcsObjectName(), typed.checkInvariants)
 
 	d = typed
 	return
@@ -386,6 +389,14 @@ func (d *dirInode) Unlock() {
 	d.mu.Unlock()
 }
 
+func (d *dirInode) LockForChildLookup() {
+	d.mu.RLock()
+}
+
+func (d *dirInode) UnlockForChildLookup() {
+	d.mu.RUnlock()
+}
+
 func (d *dirInode) ID() fuseops.InodeID {
 	return d.id
 }

main_test.go

@@ -176,7 +176,7 @@ func (t *MainTest) TestStringifyShouldReturnAllFlagsPassedInMountConfigAsMarshal
 	actual, err := util.Stringify(mountConfig)
 	AssertEq(nil, err)
 
-	expected := "{\"CreateEmptyFile\":false,\"Severity\":\"TRACE\",\"Format\":\"\",\"FilePath\":\"\\\"path\\\"to\\\"file\\\"\",\"LogRotateConfig\":{\"MaxFileSizeMB\":2,\"BackupFileCount\":2,\"Compress\":true},\"MaxSizeMB\":0,\"CacheFileForRangeRead\":false,\"CacheDir\":\"\",\"TtlInSeconds\":0,\"TypeCacheMaxSizeMB\":0,\"StatCacheMaxSizeMB\":0,\"EnableEmptyManagedFolders\":false,\"ConnPoolSize\":0,\"EnableHNS\":true,\"IgnoreInterrupts\":false}"
+	expected := "{\"CreateEmptyFile\":false,\"Severity\":\"TRACE\",\"Format\":\"\",\"FilePath\":\"\\\"path\\\"to\\\"file\\\"\",\"LogRotateConfig\":{\"MaxFileSizeMB\":2,\"BackupFileCount\":2,\"Compress\":true},\"MaxSizeMB\":0,\"CacheFileForRangeRead\":false,\"CacheDir\":\"\",\"TtlInSeconds\":0,\"TypeCacheMaxSizeMB\":0,\"StatCacheMaxSizeMB\":0,\"EnableEmptyManagedFolders\":false,\"ConnPoolSize\":0,\"EnableHNS\":true,\"IgnoreInterrupts\":false,\"DisableParallelDirops\":false}"
 	AssertEq(expected, actual)
 }
 
@@ -188,7 +188,7 @@ func (t *MainTest) TestEnableHNSFlagFalse() {
 	actual, err := util.Stringify(mountConfig)
 	AssertEq(nil, err)
 
-	expected := "{\"CreateEmptyFile\":false,\"Severity\":\"\",\"Format\":\"\",\"FilePath\":\"\",\"LogRotateConfig\":{\"MaxFileSizeMB\":0,\"BackupFileCount\":0,\"Compress\":false},\"MaxSizeMB\":0,\"CacheFileForRangeRead\":false,\"CacheDir\":\"\",\"TtlInSeconds\":0,\"TypeCacheMaxSizeMB\":0,\"StatCacheMaxSizeMB\":0,\"EnableEmptyManagedFolders\":false,\"ConnPoolSize\":0,\"EnableHNS\":false,\"IgnoreInterrupts\":false}"
+	expected := "{\"CreateEmptyFile\":false,\"Severity\":\"\",\"Format\":\"\",\"FilePath\":\"\",\"LogRotateConfig\":{\"MaxFileSizeMB\":0,\"BackupFileCount\":0,\"Compress\":false},\"MaxSizeMB\":0,\"CacheFileForRangeRead\":false,\"CacheDir\":\"\",\"TtlInSeconds\":0,\"TypeCacheMaxSizeMB\":0,\"StatCacheMaxSizeMB\":0,\"EnableEmptyManagedFolders\":false,\"ConnPoolSize\":0,\"EnableHNS\":false,\"IgnoreInterrupts\":false,\"DisableParallelDirops\":false}"
 	AssertEq(expected, actual)
 }
 

mount.go

@@ -141,10 +141,11 @@ be interacting with the file system.`)
 	// Mount the file system.
 	logger.Infof("Mounting file system %q...", fsName)
 	mountCfg := &fuse.MountConfig{
-		FSName:     fsName,
-		Subtype:    "gcsfuse",
-		VolumeName: "gcsfuse",
-		Options:    flags.MountOptions,
+		FSName:               fsName,
+		Subtype:              "gcsfuse",
+		VolumeName:           "gcsfuse",
+		Options:              flags.MountOptions,
+		EnableParallelDirOps: !(mountConfig.FileSystemConfig.DisableParallelDirops),
 	}
 
 	mountCfg.ErrorLogger = logger.NewLegacyLogger(logger.LevelError, "fuse: ")