Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

implement FDBAWSCredentialsProviderChain(release-7.1) #10880

Draft
wants to merge 3 commits into
base: release-7.1
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 15 additions & 2 deletions cmake/awssdk.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ if(APPLE OR USE_LIBCXX)
set(AWSSDK_COMPILER_FLAGS "-stdlib=libc++ -nostdlib++")
endif()


set(AWSSDK_MODULES "identity-management")

include(ExternalProject)
ExternalProject_Add(awssdk_project
GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git
Expand All @@ -15,7 +18,7 @@ ExternalProject_Add(awssdk_project
GIT_CONFIG advice.detachedHead=false
CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF # SDK builds shared libs by default, we want static libs
-DENABLE_TESTING=OFF
-DBUILD_ONLY=core # git repo contains SDK for every AWS product, we only want the core auth libraries
-DBUILD_ONLY=${AWSSDK_MODULES} # git repo contains SDK for every AWS product, we only want the core auth libraries
-DSIMPLE_INSTALL=ON
-DCMAKE_INSTALL_PREFIX=install # need to specify an install prefix so it doesn't install in /usr/lib - FIXME: use absolute path
-DBYO_CRYPTO=ON # we have our own crypto libraries that conflict if we let aws sdk build and link its own
Expand All @@ -39,10 +42,20 @@ ExternalProject_Add(awssdk_project
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-compression.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-cal.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-c-common.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-sts.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-identity-management.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/curl/lib/libcurl.a"
"${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/external-install/zlib/lib/libz.a"
)

add_library(awssdk_cpp_identity STATIC IMPORTED)
add_dependencies(awssdk_cpp_identity awssdk_project)
set_target_properties(awssdk_cpp_identity PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-identity-management.a")

add_library(awssdk_cpp_sts STATIC IMPORTED)
add_dependencies(awssdk_cpp_sts awssdk_project)
set_target_properties(awssdk_cpp_sts PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-sts.a")

add_library(awssdk_core STATIC IMPORTED)
add_dependencies(awssdk_core awssdk_project)
set_target_properties(awssdk_core PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/lib64/libaws-cpp-sdk-core.a")
Expand Down Expand Up @@ -107,4 +120,4 @@ set_property(TARGET zlib PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}
# link them all together in one interface target
add_library(awssdk_target INTERFACE)
target_include_directories(awssdk_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/awssdk-build/install/include)
target_link_libraries(awssdk_target INTERFACE awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl zlib)
target_link_libraries(awssdk_target INTERFACE awssdk_cpp_identity awssdk_cpp_sts awssdk_core awssdk_crt awssdk_c_s3 awssdk_c_auth awssdk_c_eventstream awssdk_c_http awssdk_c_mqtt awssdk_c_sdkutils awssdk_c_io awssdk_checksums awssdk_c_compression awssdk_c_cal awssdk_c_common curl zlib)
4 changes: 3 additions & 1 deletion fdbclient/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,9 @@ if(WITH_AWS_BACKUP)

set(FDBCLIENT_SRCS
${FDBCLIENT_SRCS}
FDBAWSCredentialsProvider.h)
FDBAWSCredentialsProvider.h
FDBAWSCredentialsProviderChain.h
FDBAWSCredentialsProviderChain.cpp)
Comment on lines +213 to +215
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of here, they should be added to where FDBCLIENT_SRCS is defined.


include(awssdk)
endif()
Expand Down
6 changes: 5 additions & 1 deletion fdbclient/FDBAWSCredentialsProvider.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@
#include "aws/core/Aws.h"
#include "aws/core/auth/AWSCredentialsProviderChain.h"

#include "fdbclient/FDBAWSCredentialsProvider.h"
#include "fdbclient/FDBAWSCredentialsProviderChain.h"
#include "flow/Tracing.h"

// Singleton
namespace FDBAWSCredentialsProvider {
bool doneInit = false;
Expand All @@ -38,7 +42,7 @@ static Aws::Auth::AWSCredentials getAwsCredentials() {
Aws::InitAPI(options);
TraceEvent("AWSSDKInitSuccessful");
}
Aws::Auth::DefaultAWSCredentialsProviderChain credProvider;
FDBAWSCredentialsProviderChain credProvider;
Aws::Auth::AWSCredentials creds = credProvider.GetAWSCredentials();
return creds;
}
Expand Down
91 changes: 91 additions & 0 deletions fdbclient/FDBAWSCredentialsProviderChain.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* FDBAWSCredentialsProviderChain.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "fdbclient/FDBAWSCredentialsProviderChain.h"

#ifdef BUILD_AWS_BACKUP

#include "aws/core/auth/STSCredentialsProvider.h"
#include "aws/core/auth/SSOCredentialsProvider.h"
#include "aws/identity-management/auth/STSProfileCredentialsProvider.h"
#include "aws/core/platform/Environment.h"
#include "aws/core/utils/memory/AWSMemory.h"
#include "aws/core/utils/StringUtils.h"
#include "aws/core/utils/logging/LogMacros.h"

static const char AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
static const char AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI";
static const char AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN";
static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED";
static const char DefaultCredentialsProviderChainTag[] = "DefaultAWSCredentialsProviderChain";

// this method is copied from
// https://github.com/aws/aws-sdk-cpp/blob/98a87de4e2e532cc88e527a27fbdf110d9375e6b/src/aws-cpp-sdk-core/source/auth/AWSCredentialsProviderChain.cpp#L36

FDBAWSCredentialsProviderChain::FDBAWSCredentialsProviderChain() : Aws::Auth::AWSCredentialsProviderChain() {
AddProvider(Aws::MakeShared<Aws::Auth::EnvironmentAWSCredentialsProvider>(DefaultCredentialsProviderChainTag));
AddProvider(
Aws::MakeShared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>(DefaultCredentialsProviderChainTag));
AddProvider(Aws::MakeShared<Aws::Auth::ProcessCredentialsProvider>(DefaultCredentialsProviderChainTag));
// added line below to support assume role and profile
AddProvider(Aws::MakeShared<Aws::Auth::STSProfileCredentialsProvider>(DefaultCredentialsProviderChainTag));
AddProvider(
Aws::MakeShared<Aws::Auth::STSAssumeRoleWebIdentityCredentialsProvider>(DefaultCredentialsProviderChainTag));
AddProvider(Aws::MakeShared<Aws::Auth::SSOCredentialsProvider>(DefaultCredentialsProviderChainTag));

// ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set
const auto relativeUri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI);
AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
"The environment variable value " << AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI << " is "
<< relativeUri);

const auto absoluteUri = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI);
AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
"The environment variable value " << AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI << " is "
<< absoluteUri);

const auto ec2MetadataDisabled = Aws::Environment::GetEnv(AWS_EC2_METADATA_DISABLED);
AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
"The environment variable value " << AWS_EC2_METADATA_DISABLED << " is "
<< ec2MetadataDisabled);

if (!relativeUri.empty()) {
AddProvider(Aws::MakeShared<Aws::Auth::TaskRoleCredentialsProvider>(DefaultCredentialsProviderChainTag,
relativeUri.c_str()));
AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
"Added ECS metadata service credentials provider with relative path: ["
<< relativeUri << "] to the provider chain.");
} else if (!absoluteUri.empty()) {
const auto token = Aws::Environment::GetEnv(AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN);
AddProvider(Aws::MakeShared<Aws::Auth::TaskRoleCredentialsProvider>(
DefaultCredentialsProviderChainTag, absoluteUri.c_str(), token.c_str()));

// DO NOT log the value of the authorization token for security purposes.
AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
"Added ECS credentials provider with URI: ["
<< absoluteUri << "] to the provider chain with a"
<< (token.empty() ? "n empty " : " non-empty ") << "authorization token.");
} else if (Aws::Utils::StringUtils::ToLower(ec2MetadataDisabled.c_str()) != "true") {
AddProvider(Aws::MakeShared<Aws::Auth::InstanceProfileCredentialsProvider>(DefaultCredentialsProviderChainTag));
AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
"Added EC2 metadata service credentials provider to the provider chain.");
}
}
#endif
31 changes: 31 additions & 0 deletions fdbclient/FDBAWSCredentialsProviderChain.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* FDBAWSCredentialsProviderChain.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifdef BUILD_AWS_BACKUP

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add something like #ifndef xxx to make sure this header file is only included once.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks, will update the PR once I discuss testing with @johscheuer

#include "aws/core/Aws.h"
#include "aws/core/auth/AWSCredentialsProviderChain.h"

class FDBAWSCredentialsProviderChain : public Aws::Auth::AWSCredentialsProviderChain {
public:
FDBAWSCredentialsProviderChain();
};

#endif