Merge remote-tracking branch 'upstream/master' into upgrade-gcp-pubsu…

…b-lib

.github/scripts/automerge.py

@@ -14,9 +14,9 @@
 repo = g.get_repo(os.getenv("GITHUB_REPOSITORY"))
 maintainers = [m.strip() for m in os.getenv("MAINTAINERS").split(',')]
 
-def fetch_pulls(mergeable_state):
+def fetch_pulls(mergeable_state, label = 'automerge'):
     return [pr for pr in repo.get_pulls(state='open', sort='created') \
-        if pr.mergeable_state == mergeable_state and 'automerge' in [l.name for l in pr.labels]]
+        if (not pr.draft) and pr.mergeable_state == mergeable_state and (not label or label in [l.name for l in pr.labels])]
 
 def is_approved(pr):
     approvers = [r.user.login for r in pr.get_reviews() if r.state == 'APPROVED' and r.user.login in maintainers]
@@ -40,17 +40,15 @@ def is_approved(pr):
 if len(pulls) > 0 and not merged:
     print("No PR was automerged.")
 
-# Now, update all PRs that are behind.
-pulls = fetch_pulls('behind')
+# Now, update all PRs that are behind, regardless of automerge label.
+pulls = fetch_pulls('behind', '')
 print(f"Detected {len(pulls)} open pull requests in {repo.name} to be updated.")
 for pr in pulls:
-    if is_approved(pr):
-        # Update all PRs since there is no guarantee they will all pass.
-        print(f"Updating PR {pr.html_url}")
-        try:
-            pr.update_branch()
-        except:
-            print(f"Failed to update PR {pr.html_url}")
+    print(f"Updating PR {pr.html_url}")
+    try:
+        pr.update_branch()
+    except:
+        print(f"Failed to update PR {pr.html_url}")
 
 pulls = fetch_pulls('dirty')
 print(f"Detected {len(pulls)} open pull requests in {repo.name} to be automerged but are in dirty state.")

.github/workflows/conformance.yml

@@ -188,9 +188,9 @@ jobs:
         wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip
         unzip -qq ngrok-stable-linux-amd64.zip
         ./ngrok authtoken ${{ env.AzureEventGridNgrokToken }}
-        ./ngrok http -log=stdout -host-header=localhost 9000 > /tmp/ngrok.log &
+        ./ngrok http -log=stdout --log-level debug -host-header=localhost 9000 > /tmp/ngrok.log &
         sleep 10
-        export NGROK_ENDPOINT=`cat /tmp/ngrok.log |  grep -Eo 'https://.*'`
+        export NGROK_ENDPOINT=`cat /tmp/ngrok.log |  grep -Eom1 'https://.*' | sed 's/\s.*//'`
         echo "Ngrok's endpoint: ${NGROK_ENDPOINT}"
         echo "AzureEventGridSubscriberEndpoint=${NGROK_ENDPOINT}/api/events" >> $GITHUB_ENV
         cat /tmp/ngrok.log
@@ -340,7 +340,14 @@ jobs:
     - name: Stop ngrok
       if: contains(matrix.component, 'azure.eventgrid')
       continue-on-error: true
-      run: pkill ngrok; cat /tmp/ngrok.log
+      run: |
+        set +e
+        echo "GET ngrok tunnels:"
+        curl http://localhost:4040/api/tunnels
+        echo "GET ngrok http requests:"
+        curl http://localhost:4040/api/requests/http
+        pkill ngrok
+        cat /tmp/ngrok.log
 
     - name: Cleanup Azure SQL test DB instance
       if: contains(matrix.component, 'azure.sql')

Makefile

@@ -63,6 +63,30 @@ lint:
 	# Due to https://github.com/golangci/golangci-lint/issues/580, we need to add --fix for windows
 	$(GOLANGCI_LINT) run --timeout=20m
 
+################################################################################
+# Target: modtidy-all                                                          #
+################################################################################
+MODFILES := $(shell find . -name go.mod)
+
+define modtidy-target
+.PHONY: modtidy-$(1)
+modtidy-$(1):
+	cd $(shell dirname $(1)); go mod tidy; cd -
+endef
+
+# Generate modtidy target action for each go.mod file
+$(foreach MODFILE,$(MODFILES),$(eval $(call modtidy-target,$(MODFILE))))
+
+# Enumerate all generated modtidy targets
+# Note that the order of execution matters: root and tests/certification go.mod
+# are dependencies in each certification test. This order is preserved by the
+# tree walk when finding the go.mod files.
+TIDY_MODFILES:=$(foreach ITEM,$(MODFILES),modtidy-$(ITEM))
+
+# Define modtidy-all action trigger to run make on all generated modtidy targets
+.PHONY: modtidy-all
+modtidy-all: $(TIDY_MODFILES)
+
 ################################################################################
 # Target: modtidy                                                              #
 ################################################################################

authentication/azure/auth.go

@@ -10,7 +10,10 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"io/ioutil"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/adal"
 	"github.com/Azure/go-autorest/autorest/azure"
@@ -69,10 +72,58 @@ func (s EnvironmentSettings) GetAzureEnvironment() (*azure.Environment, error) {
 	return &env, err
 }
 
+// GetTokenCredential returns an azcore.TokenCredential retrieved from, in order:
+// 1. Client credentials
+// 2. Client certificate
+// 3. MSI
+// This is used by the newer ("track 2") Azure SDKs.
+func (s EnvironmentSettings) GetTokenCredential() (azcore.TokenCredential, error) {
+	// Create a chain
+	var creds []azcore.TokenCredential
+	errMsg := ""
+
+	// 1. Client credentials
+	if c, e := s.GetClientCredentials(); e == nil {
+		cred, err := c.GetTokenCredential()
+		if err == nil {
+			creds = append(creds, cred)
+		} else {
+			errMsg += err.Error() + "\n"
+		}
+	}
+
+	// 2. Client certificate
+	if c, e := s.GetClientCert(); e == nil {
+		cred, err := c.GetTokenCredential()
+		if err == nil {
+			creds = append(creds, cred)
+		} else {
+			errMsg += err.Error() + "\n"
+		}
+	}
+
+	// 3. MSI
+	{
+		c := s.GetMSI()
+		cred, err := c.GetTokenCredential()
+		if err == nil {
+			creds = append(creds, cred)
+		} else {
+			errMsg += err.Error() + "\n"
+		}
+	}
+
+	if len(creds) == 0 {
+		return nil, fmt.Errorf("no suitable token provider for Azure AD; errors: %v", errMsg)
+	}
+	return azidentity.NewChainedTokenCredential(creds, nil)
+}
+
 // GetAuthorizer creates an Authorizer retrieved from, in order:
 // 1. Client credentials
 // 2. Client certificate
-// 3. MSI.
+// 3. MSI
+// This is used by the older Azure SDKs.
 func (s EnvironmentSettings) GetAuthorizer() (autorest.Authorizer, error) {
 	spt, err := s.GetServicePrincipalToken()
 	if err != nil {
@@ -85,7 +136,8 @@ func (s EnvironmentSettings) GetAuthorizer() (autorest.Authorizer, error) {
 // GetServicePrincipalToken returns a Service Principal Token retrieved from, in order:
 // 1. Client credentials
 // 2. Client certificate
-// 3. MSI.
+// 3. MSI
+// This is used by the older Azure SDKs.
 func (s EnvironmentSettings) GetServicePrincipalToken() (*adal.ServicePrincipalToken, error) {
 	// 1. Client credentials
 	if c, e := s.GetClientCredentials(); e == nil {
@@ -182,6 +234,13 @@ func (c CredentialsConfig) ServicePrincipalToken() (*adal.ServicePrincipalToken,
 	return adal.NewServicePrincipalToken(*oauthConfig, c.ClientID, c.ClientSecret, c.Resource)
 }
 
+// GetTokenCredential returns the azcore.TokenCredential object from the credentials.
+func (c CredentialsConfig) GetTokenCredential() (token azcore.TokenCredential, err error) {
+	return azidentity.NewClientSecretCredential(c.TenantID, c.ClientID, c.ClientSecret, &azidentity.ClientSecretCredentialOptions{
+		AuthorityHost: azidentity.AuthorityHost(c.AADEndpoint),
+	})
+}
+
 // CertConfig provides the options to get a bearer authorizer from a client certificate.
 type CertConfig struct {
 	*auth.ClientCertificateConfig
@@ -231,6 +290,39 @@ func (c CertConfig) ServicePrincipalTokenByCertBytes() (*adal.ServicePrincipalTo
 	return adal.NewServicePrincipalTokenFromCertificate(*oauthConfig, c.ClientID, certificate, rsaPrivateKey, c.Resource)
 }
 
+// GetTokenCredential returns the azcore.TokenCredential object from client certificate.
+func (c CertConfig) GetTokenCredential() (token azcore.TokenCredential, err error) {
+	ccc := c.ClientCertificateConfig
+
+	// Certificate data - may be empty here
+	data := c.CertificateData
+
+	// If we have a certificate path, load it
+	if c.ClientCertificateConfig.CertificatePath != "" {
+		var errB error
+		data, errB = ioutil.ReadFile(ccc.CertificatePath)
+		if errB != nil {
+			return nil, fmt.Errorf("failed to read the certificate file (%s): %v", ccc.CertificatePath, errB)
+		}
+	}
+	if len(data) == 0 {
+		return nil, fmt.Errorf("certificate is not given")
+	}
+
+	// Decode the PKCS#12 certificate
+	cert, key, err := c.decodePkcs12(data, c.CertificatePassword)
+	if err != nil || cert == nil {
+		return nil, fmt.Errorf("failed to decode pkcs12 certificate while creating spt: %v", err)
+	}
+
+	// Create the azcore.TokenCredential object
+	certs := []*x509.Certificate{cert}
+	opts := &azidentity.ClientCertificateCredentialOptions{
+		AuthorityHost: azidentity.AuthorityHost(c.AADEndpoint),
+	}
+	return azidentity.NewClientCertificateCredential(c.TenantID, c.ClientID, certs, key, opts)
+}
+
 func (c CertConfig) decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
 	privateKey, certificate, err := pkcs12.Decode(pkcs, password)
 	if err != nil {
@@ -259,20 +351,20 @@ func NewMSIConfig(resource string) MSIConfig {
 }
 
 // ServicePrincipalToken gets the ServicePrincipalToken object from MSI.
-func (mc MSIConfig) ServicePrincipalToken() (*adal.ServicePrincipalToken, error) {
+func (c MSIConfig) ServicePrincipalToken() (*adal.ServicePrincipalToken, error) {
 	msiEndpoint, err := adal.GetMSIEndpoint()
 	if err != nil {
 		return nil, err
 	}
 
 	var spToken *adal.ServicePrincipalToken
-	if mc.ClientID == "" {
-		spToken, err = adal.NewServicePrincipalTokenFromMSI(msiEndpoint, mc.Resource)
+	if c.ClientID == "" {
+		spToken, err = adal.NewServicePrincipalTokenFromMSI(msiEndpoint, c.Resource)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get oauth token from MSI: %v", err)
 		}
 	} else {
-		spToken, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, mc.Resource, mc.ClientID)
+		spToken, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, c.Resource, c.ClientID)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get oauth token from MSI for user assigned identity: %v", err)
 		}
@@ -281,6 +373,15 @@ func (mc MSIConfig) ServicePrincipalToken() (*adal.ServicePrincipalToken, error)
 	return spToken, nil
 }
 
+// GetTokenCredential returns the azcore.TokenCredential object from MSI.
+func (c MSIConfig) GetTokenCredential() (token azcore.TokenCredential, err error) {
+	opts := &azidentity.ManagedIdentityCredentialOptions{}
+	if c.ClientID != "" {
+		opts.ID = azidentity.ClientID(c.ClientID)
+	}
+	return azidentity.NewManagedIdentityCredential(opts)
+}
+
 // GetAzureEnvironment returns the Azure environment for a given name, supporting aliases too.
 func (s EnvironmentSettings) GetEnvironment(key string) (string, bool) {
 	var (

authentication/azure/auth_test.go

@@ -146,6 +146,8 @@ func TestGetMSI(t *testing.T) {
 }
 
 func TestFallbackToMSI(t *testing.T) {
+	os.Setenv("MSI_ENDPOINT", "test")
+	defer os.Unsetenv("MSI_ENDPOINT")
 	settings, err := NewEnvironmentSettings(
 		"keyvault",
 		map[string]string{
@@ -162,6 +164,8 @@ func TestFallbackToMSI(t *testing.T) {
 }
 
 func TestAuthorizorWithMSI(t *testing.T) {
+	os.Setenv("MSI_ENDPOINT", "test")
+	defer os.Unsetenv("MSI_ENDPOINT")
 	settings, err := NewEnvironmentSettings(
 		"keyvault",
 		map[string]string{
@@ -180,6 +184,8 @@ func TestAuthorizorWithMSI(t *testing.T) {
 }
 
 func TestAuthorizorWithMSIAndUserAssignedID(t *testing.T) {
+	os.Setenv("MSI_ENDPOINT", "test")
+	defer os.Unsetenv("MSI_ENDPOINT")
 	settings, err := NewEnvironmentSettings(
 		"keyvault",
 		map[string]string{

bindings/azure/eventgrid/eventgrid.go

@@ -252,7 +252,7 @@ func (a *AzureEventGrid) createSubscription() error {
 		return err
 	}
 
-	res := result.Future.Response()
+	res := result.FutureAPI.Response()
 
 	if res.StatusCode != fasthttp.StatusCreated {
 		bodyBytes, err := ioutil.ReadAll(res.Body)

docs/developing-component.md

@@ -87,7 +87,7 @@ cp ./dist/darwin_amd64/debug/daprd ~/.dapr/bin
 4. Update component-contrib go mod and ensure that component-contrib is updated to the latest version
 ```bash
 go get -u github.com/dapr/components-contrib@master
-go mod tidy
+make modtidy-all
 ```
 5. Import your component to Dapr [main.go](https://github.com/dapr/dapr/blob/b3e1fe848de3ea7b297c712d188136919d314887/cmd/daprd/main.go#L20-L115)
 6. Register your component in Dapr [main.go](https://github.com/dapr/dapr/blob/b3e1fe848de3ea7b297c712d188136919d314887/cmd/daprd/main.go#L256-L385)

go.mod

@@ -9,14 +9,17 @@ require (
 	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-amqp-common-go/v3 v3.1.0
 	github.com/Azure/azure-event-hubs-go/v3 v3.3.10
-	github.com/Azure/azure-sdk-for-go v48.2.0+incompatible
+	github.com/Azure/azure-sdk-for-go v57.2.0+incompatible
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0
+	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.3.0
 	github.com/Azure/azure-service-bus-go v0.10.10
 	github.com/Azure/azure-storage-blob-go v0.10.0
 	github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd
 	github.com/Azure/go-amqp v0.13.1
-	github.com/Azure/go-autorest/autorest v0.11.12
-	github.com/Azure/go-autorest/autorest/adal v0.9.5
-	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
+	github.com/Azure/go-autorest/autorest v0.11.21
+	github.com/Azure/go-autorest/autorest/adal v0.9.16
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/Shopify/sarama v1.23.1
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
@@ -49,7 +52,6 @@ require (
 	github.com/dghubble/go-twitter v0.0.0-20190719072343-39e5462e111f
 	github.com/dghubble/oauth1 v0.6.0
 	github.com/didip/tollbooth v4.0.2+incompatible
-	github.com/dnaeon/go-vcr v1.1.0 // indirect
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/eclipse/paho.mqtt.golang v1.3.5
 	github.com/fasthttp-contrib/sessions v0.0.0-20160905201309-74f6ac73d5d5
@@ -65,7 +67,6 @@ require (
 	github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556
 	github.com/golang-jwt/jwt v3.2.1+incompatible
 	github.com/golang/mock v1.6.0
-	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.2.0
 	github.com/googleapis/gnostic v0.5.1 // indirect
 	github.com/gorilla/mux v1.8.0