scrubber: simplify S3 SDK setup

[problame]

The s3 client setup code in scrubber

neon/s3_scrubber/src/lib.rs

Lines 279 to 326 in 3a2f107

	pub fn init_s3_client(account_id: Option<String>, bucket_region: Region) -> Client {
	let credentials_provider = {
	// uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"
	let chain = CredentialsProviderChain::first_try(
	"env",
	EnvironmentVariableCredentialsProvider::new(),
	)
	// uses "AWS_PROFILE" / `aws sso login --profile <profile>`
	.or_else(
	"profile-sso",
	ProfileFileCredentialsProvider::builder().build(),
	);
	// Use SSO if we were given an account ID
	match account_id {
	Some(sso_account) => chain.or_else(
	"sso",
	SsoCredentialsProvider::builder()
	.account_id(sso_account)
	.role_name("PowerUserAccess")
	.start_url("https://neondb.awsapps.com/start")
	.region(bucket_region.clone())
	.build(),
	),
	None => chain,
	}
	.or_else(
	// Finally try IMDS
	"imds",
	ImdsCredentialsProvider::builder().build(),
	)
	};
	let sleep_impl: Arc<dyn AsyncSleep> = Arc::new(TokioSleep::new());
	let mut builder = Config::builder()
	.behavior_version(BehaviorVersion::v2023_11_09())
	.region(bucket_region)
	.retry_config(RetryConfig::adaptive().with_max_attempts(3))
	.sleep_impl(SharedAsyncSleep::from(sleep_impl))
	.credentials_provider(credentials_provider);
	if let Ok(endpoint) = env::var("AWS_ENDPOINT_URL") {
	builder = builder.endpoint_url(endpoint)
	}
	Client::from_conf(builder.build())
	}

looks quite copy-pasted from remote_storage

neon/libs/remote_storage/src/s3_bucket.rs

Lines 84 to 138 in c18d334

	let region = Some(Region::new(aws_config.bucket_region.clone()));
	let provider_conf = ProviderConfig::without_region().with_region(region.clone());
	let credentials_provider = {
	// uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"
	CredentialsProviderChain::first_try(
	"env",
	EnvironmentVariableCredentialsProvider::new(),
	)
	// uses "AWS_PROFILE" / `aws sso login --profile <profile>`
	.or_else(
	"profile-sso",
	ProfileFileCredentialsProvider::builder()
	.configure(&provider_conf)
	.build(),
	)
	// uses "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME"
	// needed to access remote extensions bucket
	.or_else(
	"token",
	WebIdentityTokenCredentialsProvider::builder()
	.configure(&provider_conf)
	.build(),
	)
	// uses imds v2
	.or_else("imds", ImdsCredentialsProvider::builder().build())
	};
	// AWS SDK requires us to specify how the RetryConfig should sleep when it wants to back off
	let sleep_impl: Arc<dyn AsyncSleep> = Arc::new(TokioSleep::new());
	// We do our own retries (see [`backoff::retry`]). However, for the AWS SDK to enable rate limiting in response to throttling
	// responses (e.g. 429 on too many ListObjectsv2 requests), we must provide a retry config. We set it to use at most one
	// attempt, and enable 'Adaptive' mode, which causes rate limiting to be enabled.
	let mut retry_config = RetryConfigBuilder::new();
	retry_config
	.set_max_attempts(Some(1))
	.set_mode(Some(RetryMode::Adaptive));
	let mut config_builder = Builder::default()
	.behavior_version(BehaviorVersion::v2023_11_09())
	.region(region)
	.identity_cache(IdentityCache::lazy().build())
	.credentials_provider(SharedCredentialsProvider::new(credentials_provider))
	.retry_config(retry_config.build())
	.sleep_impl(SharedAsyncSleep::from(sleep_impl));
	if let Some(custom_endpoint) = aws_config.endpoint.clone() {
	config_builder = config_builder
	.endpoint_url(custom_endpoint)
	.force_path_style(true);
	}
	let client = Client::from_conf(config_builder.build());

The differences appear to be:

• scrubber explicitly handles AWS_ENDPOINT_URL, pageserver doesn't
  • best practice would probably be to just use aws_config::from_env() and let the AWS SDK handle this (AWS_ENDPOINT_URL is a standard env var in AWS SDK land)
  • the remote_storage code will default to this as of in remote_storage: AWS_PROFILE with endpoint overrides in ~/.aws/config (updates AWS SDKs) #7664

Note that this isn't advocating for sharing a piece of code with remote_storage.
I think it makes sense for the scrubber to be very deliberate about how it sets up its S3 client to have its own S3 client.

Then again, if we want to support another cloud storage, maybe it would be better for the scrubber to use an abstraction layer like remote_storage 🤔

[jcsp]

This might make sense to clean up as we add Azure support to the scrubber in #7547 -- that might involve just using remote_storage everywhere.

[arpad-m]

Btw since #7189 we now have this also in the proxy:

neon/proxy/src/bin/proxy.rs

Lines 288 to 308 in c6d5ff9

	let aws_credentials_provider = {
	// uses "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"
	CredentialsProviderChain::first_try("env", EnvironmentVariableCredentialsProvider::new())
	// uses "AWS_PROFILE" / `aws sso login --profile <profile>`
	.or_else(
	"profile-sso",
	ProfileFileCredentialsProvider::builder()
	.configure(&provider_conf)
	.build(),
	)
	// uses "AWS_WEB_IDENTITY_TOKEN_FILE", "AWS_ROLE_ARN", "AWS_ROLE_SESSION_NAME"
	// needed to access remote extensions bucket
	.or_else(
	"token",
	WebIdentityTokenCredentialsProvider::builder()
	.configure(&provider_conf)
	.build(),
	)
	// uses imds v2
	.or_else("imds", ImdsCredentialsProvider::builder().build())
	};