Intro anonymous-access flag (#1827)

* adding disable auth flag * adding unit tests * rebasing on top of master and fix lint tests * rebasing on top of master and fix lint tests * fix lint tests * adding unit tests for storage handle * adding test for client.go * lint fix * adding log * skip auth changes * comment update * unit tests changes * adding new unit tests * rebase on master * unit tests fix * fix linux tests * Update name * adding unit tests * review comments * adding skip-auth condition in gRPC client * Adding unit test * Updating flag config * small fix * lint fix * removing transoport * adding user agent option * rebasing * changing flag name * removing unnecessary changes * changing flag name * small fix * lint fix * lint fix * Adding comment * removed user-agent * review comments * returning error in case of grpc client * formating * lint fix
GoogleCloudPlatform · Apr 30, 2024 · 7742c68 · 7742c68
1 parent 5625d97
commit 7742c68
Show file tree

Hide file tree

Showing 11 changed files with 166 additions and 77 deletions.
diff --git a/flags.go b/flags.go
@@ -155,8 +155,7 @@ func newApp() (app *cli.App) {
 				Usage: "Specifies an alternative custom endpoint for fetching data. Should only be used for testing. " +
 					"The custom endpoint must support the equivalent resources and operations as the GCS " +
 					"JSON endpoint, https://storage.googleapis.com/storage/v1. If a custom endpoint is not specified, " +
-					"GCSFuse uses the global GCS JSON API endpoint, https://storage.googleapis.com/storage/v1. " +
-					"If a custom endpoint is specified, authentication is disabled on the endpoint.",
+					"GCSFuse uses the global GCS JSON API endpoint, https://storage.googleapis.com/storage/v1.",
 			},
 
 			cli.StringFlag{

diff --git a/internal/config/mount_config.go b/internal/config/mount_config.go
@@ -48,6 +48,7 @@ const (
 	DefaultFileCacheMaxSizeMB               int64 = -1
 	DefaultEnableEmptyManagedFoldersListing       = false
 	DefaultGrpcConnPoolSize                       = 1
+	DefaultAnonymousAccess                        = false
 	DefaultEnableHNS                              = false
 )
 
@@ -78,6 +79,12 @@ type GrpcClientConfig struct {
 	ConnPoolSize int `yaml:"conn-pool-size,omitempty"`
 }
 
+type AuthConfig struct {
+	// Authentication is enabled by default. The skip flag disables authentication. For users of the --custom-endpoint flag,
+	// please pass anonymous-access flag explicitly if you do not want authentication enabled for your workflow.
+	AnonymousAccess bool `yaml:"anonymous-access"`
+}
+
 // Enable the storage control client flow on HNS buckets to utilize new APIs.
 type EnableHNS bool
 type CacheDir string
@@ -119,6 +126,7 @@ type MountConfig struct {
 	MetadataCacheConfig `yaml:"metadata-cache"`
 	ListConfig          `yaml:"list"`
 	GrpcClientConfig    `yaml:"grpc"`
+	AuthConfig          `yaml:"auth-config"`
 	EnableHNS           `yaml:"enable-hns"`
 	FileSystemConfig    `yaml:"file-system"`
 }
@@ -168,6 +176,9 @@ func NewMountConfig() *MountConfig {
 	mountConfig.GrpcClientConfig = GrpcClientConfig{
 		ConnPoolSize: DefaultGrpcConnPoolSize,
 	}
+	mountConfig.AuthConfig = AuthConfig{
+		AnonymousAccess: DefaultAnonymousAccess,
+	}
 	mountConfig.EnableHNS = DefaultEnableHNS
 
 	return mountConfig

diff --git a/internal/config/testdata/valid_config.yaml b/internal/config/testdata/valid_config.yaml
@@ -18,6 +18,8 @@ metadata-cache:
   stat-cache-max-size-mb: 3
 list:
   enable-empty-managed-folders: true
+auth-config:
+  anonymous-access: true
 grpc:
   conn-pool-size: 4
 enable-hns: true

diff --git a/internal/config/yaml_parser_test.go b/internal/config/yaml_parser_test.go
@@ -44,6 +44,7 @@ func validateDefaultConfig(mountConfig *MountConfig) {
 	ExpectEq(-1, mountConfig.FileCacheConfig.MaxSizeMB)
 	ExpectEq(false, mountConfig.FileCacheConfig.CacheFileForRangeRead)
 	ExpectEq(1, mountConfig.GrpcClientConfig.ConnPoolSize)
+	ExpectEq(false, mountConfig.AuthConfig.AnonymousAccess)
 	ExpectEq(false, mountConfig.EnableHNS)
 	ExpectFalse(mountConfig.FileSystemConfig.IgnoreInterrupts)
 }
@@ -121,6 +122,9 @@ func (t *YamlParserTest) TestReadConfigFile_ValidConfig() {
 	// list config
 	ExpectEq(true, mountConfig.ListConfig.EnableEmptyManagedFolders)
 
+	// auth config
+	ExpectEq(true, mountConfig.AuthConfig.AnonymousAccess)
+
 	// enable-hns
 	ExpectEq(true, mountConfig.EnableHNS)
 

diff --git a/internal/storage/storage_handle.go b/internal/storage/storage_handle.go
@@ -61,18 +61,27 @@ func createGRPCClientHandle(ctx context.Context, clientConfig *storageutil.Stora
 
 	// Add Custom endpoint option.
 	if clientConfig.CustomEndpoint != nil {
-		clientOpts = append(clientOpts, option.WithEndpoint(storageutil.StripScheme(clientConfig.CustomEndpoint.String())))
-		// Explicitly disable auth in case of custom-endpoint, aligned with the http-client.
-		// TODO: to revisit here when supporting TPC for grpc client.
-		clientOpts = append(clientOpts, option.WithoutAuthentication())
-		clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())))
-	} else {
-		tokenSrc, tokenCreationErr := storageutil.CreateTokenSource(clientConfig)
-		if tokenCreationErr != nil {
-			err = fmt.Errorf("while fetching tokenSource: %w", tokenCreationErr)
+		if clientConfig.AnonymousAccess {
+			clientOpts = append(clientOpts, option.WithEndpoint(storageutil.StripScheme(clientConfig.CustomEndpoint.String())))
+			// Explicitly disable auth in case of custom-endpoint, aligned with the http-client.
+			// TODO: to revisit here when supporting TPC for grpc client.
+			clientOpts = append(clientOpts, option.WithoutAuthentication())
+			clientOpts = append(clientOpts, option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())))
+		} else {
+			err = fmt.Errorf("GRPC client doesn't support auth for custom-endpoint. Please set anonymous-access: true via config-file.")
 			return
 		}
-		clientOpts = append(clientOpts, option.WithTokenSource(tokenSrc))
+	} else {
+		if clientConfig.AnonymousAccess {
+			clientOpts = append(clientOpts, option.WithoutAuthentication())
+		} else {
+			tokenSrc, tokenCreationErr := storageutil.CreateTokenSource(clientConfig)
+			if tokenCreationErr != nil {
+				err = fmt.Errorf("while fetching tokenSource: %w", tokenCreationErr)
+				return
+			}
+			clientOpts = append(clientOpts, option.WithTokenSource(tokenSrc))
+		}
 	}
 
 	clientOpts = append(clientOpts, option.WithGRPCConnectionPool(clientConfig.GrpcConnPoolSize))
@@ -105,6 +114,10 @@ func createHTTPClientHandle(ctx context.Context, clientConfig *storageutil.Stora
 		return nil, fmt.Errorf("client-protocol requested is not HTTP1 or HTTP2: %s", clientConfig.ClientProtocol)
 	}
 
+	if clientConfig.AnonymousAccess {
+		clientOpts = append(clientOpts, option.WithoutAuthentication())
+	}
+
 	// Create client with JSON read flow, if EnableJasonRead flag is set.
 	if clientConfig.ExperimentalEnableJsonRead {
 		clientOpts = append(clientOpts, storage.WithJSONReads())

diff --git a/internal/storage/storage_handle_test.go b/internal/storage/storage_handle_test.go
@@ -53,6 +53,7 @@ func (t *StorageHandleTest) TearDown() {
 
 func (t *StorageHandleTest) invokeAndVerifyStorageHandle(sc storageutil.StorageClientConfig) {
 	handleCreated, err := NewStorageHandle(context.Background(), sc)
+
 	AssertEq(nil, err)
 	AssertNe(nil, handleCreated)
 }
@@ -94,10 +95,21 @@ func (t *StorageHandleTest) TestNewStorageHandleHttp2Disabled() {
 }
 
 func (t *StorageHandleTest) TestNewStorageHandleHttp2Enabled() {
+	sc := storageutil.GetDefaultStorageClientConfig() // by default http1 enabled
+
+	t.invokeAndVerifyStorageHandle(sc)
+}
+
+func (t *StorageHandleTest) TestNewStorageHandleHttp2EnabledAndAuthEnabled() {
 	sc := storageutil.GetDefaultStorageClientConfig()
 	sc.ClientProtocol = mountpkg.HTTP2
+	sc.AnonymousAccess = false
 
-	t.invokeAndVerifyStorageHandle(sc)
+	handleCreated, err := NewStorageHandle(context.Background(), sc)
+
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	AssertEq(nil, handleCreated)
 }
 
 func (t *StorageHandleTest) TestNewStorageHandleWithZeroMaxConnsPerHost() {
@@ -123,10 +135,25 @@ func (t *StorageHandleTest) TestNewStorageHandleWithCustomEndpoint() {
 	t.invokeAndVerifyStorageHandle(sc)
 }
 
+func (t *StorageHandleTest) TestNewStorageHandleWithCustomEndpointAndAuthEnabled() {
+	url, err := url.Parse(storageutil.CustomEndpoint)
+	AssertEq(nil, err)
+	sc := storageutil.GetDefaultStorageClientConfig()
+	sc.CustomEndpoint = url
+	sc.AnonymousAccess = false
+
+	handleCreated, err := NewStorageHandle(context.Background(), sc)
+
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	AssertEq(nil, handleCreated)
+}
+
 // This will fail while fetching the token-source, since key-file doesn't exist.
-func (t *StorageHandleTest) TestNewStorageHandleWhenCustomEndpointIsNil() {
+func (t *StorageHandleTest) TestNewStorageHandleWhenCustomEndpointIsNilAndAuthEnabled() {
 	sc := storageutil.GetDefaultStorageClientConfig()
 	sc.CustomEndpoint = nil
+	sc.AnonymousAccess = false
 
 	handleCreated, err := NewStorageHandle(context.Background(), sc)
 
@@ -188,10 +215,7 @@ func (t *StorageHandleTest) TestCreateGRPCClientHandle() {
 func (t *StorageHandleTest) TestCreateHTTPClientHandle() {
 	sc := storageutil.GetDefaultStorageClientConfig()
 
-	handleCreated, err := createHTTPClientHandle(context.Background(), &sc)
-
-	AssertEq(nil, err)
-	AssertNe(nil, handleCreated)
+	t.invokeAndVerifyStorageHandle(sc)
 }
 
 func (t *StorageHandleTest) TestNewStorageHandleWithGRPCClientProtocol() {
@@ -222,3 +246,31 @@ func (t *StorageHandleTest) TestCreateHTTPClientHandle_WithGRPCClientProtocol()
 	AssertEq(nil, handleCreated)
 	AssertTrue(strings.Contains(err.Error(), fmt.Sprintf("client-protocol requested is not HTTP1 or HTTP2: %s", mountpkg.GRPC)))
 }
+
+func (t *StorageHandleTest) TestNewStorageHandleWithGRPCClientWithCustomEndpointNilAndAuthEnabled() {
+	sc := storageutil.GetDefaultStorageClientConfig()
+	sc.CustomEndpoint = nil
+	sc.AnonymousAccess = false
+	sc.ClientProtocol = mountpkg.GRPC
+
+	handleCreated, err := NewStorageHandle(context.Background(), sc)
+
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	AssertEq(nil, handleCreated)
+}
+
+func (t *StorageHandleTest) TestNewStorageHandleWithGRPCClientWithCustomEndpointAndAuthEnabled() {
+	url, err := url.Parse(storageutil.CustomEndpoint)
+	AssertEq(nil, err)
+	sc := storageutil.GetDefaultStorageClientConfig()
+	sc.CustomEndpoint = url
+	sc.AnonymousAccess = false
+	sc.ClientProtocol = mountpkg.GRPC
+
+	handleCreated, err := NewStorageHandle(context.Background(), sc)
+
+	AssertNe(nil, err)
+	AssertTrue(strings.Contains(err.Error(), "GRPC client doesn't support auth for custom-endpoint. Please set anonymous-access: true via config-file."))
+	AssertEq(nil, handleCreated)
+}
diff --git a/internal/storage/storageutil/client.go b/internal/storage/storageutil/client.go
@@ -48,6 +48,7 @@ type StorageClientConfig struct {
 	MaxIdleConnsPerHost        int
 	HttpClientTimeout          time.Duration
 	ExperimentalEnableJsonRead bool
+	AnonymousAccess            bool
 
 	/** Grpc client parameters. */
 	GrpcConnPoolSize int
@@ -76,39 +77,43 @@ func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *htt
 		}
 	}
 
-	tokenSrc, err := CreateTokenSource(storageClientConfig)
-	if err != nil {
-		err = fmt.Errorf("while fetching tokenSource: %w", err)
-		return
-	}
-
-	// Custom http client for Go Client.
-	httpClient = &http.Client{
-		Transport: &oauth2.Transport{
-			Base:   transport,
-			Source: tokenSrc,
-		},
-		Timeout: storageClientConfig.HttpClientTimeout,
-	}
+	if storageClientConfig.AnonymousAccess {
+		// UserAgent will not be added if authentication is disabled. Bypassing authentication prevents the creation of an HTTP transport because it requires a token source.
+		// Setting a dummy token would conflict with the "WithoutAuthentication" option.
+		// While the "WithUserAgent" option could set a custom User-Agent, it's incompatible with the "WithHTTPClient" option, preventing the direct injection of a user agent
+		// when authentication is skipped.
+		httpClient = &http.Client{
+			Timeout: storageClientConfig.HttpClientTimeout,
+		}
+	} else {
+		var tokenSrc oauth2.TokenSource
+		tokenSrc, err = CreateTokenSource(storageClientConfig)
+		if err != nil {
+			err = fmt.Errorf("while fetching tokenSource: %w", err)
+			return
+		}
 
-	// Setting UserAgent through RoundTripper middleware
-	httpClient.Transport = &userAgentRoundTripper{
-		wrapped:   httpClient.Transport,
-		UserAgent: storageClientConfig.UserAgent,
+		// Custom http client for Go Client.
+		httpClient = &http.Client{
+			Transport: &oauth2.Transport{
+				Base:   transport,
+				Source: tokenSrc,
+			},
+			Timeout: storageClientConfig.HttpClientTimeout,
+		}
+		// Setting UserAgent through RoundTripper middleware
+		httpClient.Transport = &userAgentRoundTripper{
+			wrapped:   httpClient.Transport,
+			UserAgent: storageClientConfig.UserAgent,
+		}
 	}
-
 	return httpClient, err
 }
 
-// It creates dummy token-source in case of non-nil custom url. If the custom-endpoint
-// is nil, it creates the token-source from the provided key-file or using ADC search
-// order (https://cloud.google.com/docs/authentication/application-default-credentials#order).
+// It creates the token-source from the provided
+// key-file or using ADC search order (https://cloud.google.com/docs/authentication/application-default-credentials#order).
 func CreateTokenSource(storageClientConfig *StorageClientConfig) (tokenSrc oauth2.TokenSource, err error) {
-	if storageClientConfig.CustomEndpoint == nil {
-		return auth.GetTokenSource(context.Background(), storageClientConfig.KeyFile, storageClientConfig.TokenUrl, storageClientConfig.ReuseTokenFromUrl)
-	} else {
-		return oauth2.StaticTokenSource(&oauth2.Token{}), nil
-	}
+	return auth.GetTokenSource(context.Background(), storageClientConfig.KeyFile, storageClientConfig.TokenUrl, storageClientConfig.ReuseTokenFromUrl)
 }
 
 // StripScheme strips the scheme part of given url.

diff --git a/internal/storage/storageutil/client_test.go b/internal/storage/storageutil/client_test.go
@@ -16,7 +16,6 @@ package storageutil
 
 import (
 	"net/http"
-	"net/url"
 	"testing"
 
 	"github.com/jacobsa/oglematchers"
@@ -47,54 +46,57 @@ func (t *clientTest) validateProxyInTransport(httpClient *http.Client) {
 
 // Tests
 
-func (t *clientTest) TestCreateTokenSrcWithCustomEndpoint() {
-	url, err := url.Parse(CustomEndpoint)
-	AssertEq(nil, err)
-	sc := GetDefaultStorageClientConfig()
-	sc.CustomEndpoint = url
+func (t *clientTest) TestCreateHttpClientWithHttp1() {
+	sc := GetDefaultStorageClientConfig() // By default http1 enabled
 
-	tokenSrc, err := CreateTokenSource(&sc)
+	httpClient, err := CreateHttpClient(&sc)
 
 	ExpectEq(nil, err)
-	ExpectNe(nil, &tokenSrc)
+	ExpectNe(nil, httpClient)
+	ExpectEq(sc.HttpClientTimeout, httpClient.Timeout)
 }
 
-func (t *clientTest) TestCreateTokenSrcWhenCustomEndpointIsNil() {
+func (t *clientTest) TestCreateHttpClientWithHttp2() {
 	sc := GetDefaultStorageClientConfig()
-	sc.CustomEndpoint = nil
 
-	// It will try to create the actual auth token and fail since key-file doesn't exist.
-	tokenSrc, err := CreateTokenSource(&sc)
+	httpClient, err := CreateHttpClient(&sc)
 
-	ExpectNe(nil, err)
-	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
-	ExpectEq(nil, tokenSrc)
+	ExpectEq(nil, err)
+	ExpectNe(nil, httpClient)
+	ExpectEq(sc.HttpClientTimeout, httpClient.Timeout)
 }
 
-func (t *clientTest) TestCreateHttpClientWithHttp1() {
+func (t *clientTest) TestCreateHttpClientWithHttp1AndAuthEnabled() {
 	sc := GetDefaultStorageClientConfig() // By default http1 enabled
+	sc.AnonymousAccess = false
 
 	// Act: this method add tokenSource and clientOptions.
 	httpClient, err := CreateHttpClient(&sc)
 
-	ExpectEq(nil, err)
-	ExpectNe(nil, httpClient)
-	ExpectNe(nil, httpClient.Transport)
-	t.validateProxyInTransport(httpClient)
-	ExpectEq(sc.HttpClientTimeout, httpClient.Timeout)
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	AssertEq(nil, httpClient)
 }
 
-func (t *clientTest) TestCreateHttpClientWithHttp2() {
+func (t *clientTest) TestCreateHttpClientWithHttp2AndAuthEnabled() {
 	sc := GetDefaultStorageClientConfig()
-
+	sc.AnonymousAccess = false
 	// Act: this method add tokenSource and clientOptions.
 	httpClient, err := CreateHttpClient(&sc)
 
-	ExpectEq(nil, err)
-	ExpectNe(nil, httpClient)
-	ExpectNe(nil, httpClient.Transport)
-	t.validateProxyInTransport(httpClient)
-	ExpectEq(sc.HttpClientTimeout, httpClient.Timeout)
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	AssertEq(nil, httpClient)
+}
+
+func (t *clientTest) TestCreateTokenSrc() {
+	sc := GetDefaultStorageClientConfig()
+
+	tokenSrc, err := CreateTokenSource(&sc)
+
+	AssertNe(nil, err)
+	ExpectThat(err, oglematchers.Error(oglematchers.HasSubstr("no such file or directory")))
+	ExpectNe(nil, &tokenSrc)
 }
 
 func (t *clientTest) TestStripScheme() {