Skip to content

Commit 65256d5

Browse files
kishantrkishantr
authored
[SCP-4543] : CLI changes to support custom connector plugins on GCP (#3070)
1 parent e0189be commit 65256d5

File tree

10 files changed

+95
-19
lines changed

10 files changed

+95
-19
lines changed

‎internal/connect/command_custom_plugin_create.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ func (c *customPluginCommand) createCustomPlugin(cmd *cobra.Command, args []stri
107107
if err := utils.UploadFileToAzureBlob(resp.GetUploadUrl(), pluginFileName, strings.ToLower(resp.GetContentFormat())); err != nil {
108108
return err
109109
}
110+
} else if cloud == "GCP" {
111+
if err := utils.UploadFileToGoogleCloudStorage(resp.GetUploadUrl(), pluginFileName, strings.ToLower(resp.GetContentFormat())); err != nil {
112+
return err
113+
}
110114
} else {
111115
if err := utils.UploadFile(resp.GetUploadUrl(), pluginFileName, resp.GetUploadFormData()); err != nil {
112116
return err

‎pkg/ccloudv2/utils.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ const (
2424
)
2525

2626
var (
27-
ByocSupportClouds = []string{"aws", "azure"}
27+
ByocSupportClouds = []string{"aws", "azure", "gcp"}
2828
)
2929

3030
var (

‎pkg/utils/utils.go

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ func UploadFile(url, filePath string, formFields map[string]any) error {
187187
return nil
188188
}
189189

190-
func UploadFileToAzureBlob(url, filePath, contentFormat string) error {
190+
func uploadFileInternal(url, filePath string, headers map[string]string) error {
191191
fileInfo, err := os.Stat(filePath)
192192
if err != nil {
193193
return err
@@ -208,12 +208,9 @@ func UploadFileToAzureBlob(url, filePath, contentFormat string) error {
208208
if err != nil {
209209
return err
210210
}
211-
request.Header.Set("x-ms-blob-type", "BlockBlob")
212-
switch contentFormat {
213-
case "zip":
214-
request.Header.Set("Content-Type", "application/zip")
215-
case "jar":
216-
request.Header.Set("Content-Type", "application/java-archive")
211+
212+
for key, value := range headers {
213+
request.Header.Set(key, value)
217214
}
218215
request.ContentLength = fileInfo.Size()
219216
response, err := client.Do(request)
@@ -232,3 +229,26 @@ func UploadFileToAzureBlob(url, filePath, contentFormat string) error {
232229

233230
return nil
234231
}
232+
233+
func UploadFileToAzureBlob(url, filePath, contentFormat string) error {
234+
headers := map[string]string{
235+
"x-ms-blob-type": "BlockBlob",
236+
}
237+
setContentType(headers, contentFormat)
238+
return uploadFileInternal(url, filePath, headers)
239+
}
240+
241+
func UploadFileToGoogleCloudStorage(url, filePath, contentFormat string) error {
242+
headers := map[string]string{}
243+
setContentType(headers, contentFormat)
244+
return uploadFileInternal(url, filePath, headers)
245+
}
246+
247+
func setContentType(headers map[string]string, contentFormat string) {
248+
switch contentFormat {
249+
case "zip":
250+
headers["Content-Type"] = "application/zip"
251+
case "jar":
252+
headers["Content-Type"] = "application/java-archive"
253+
}
254+
}

‎test/connect_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,13 +207,16 @@ func (s *CLITestSuite) TestConnectCustomPlugin() {
207207
tests := []CLITest{
208208
{args: `connect custom-plugin create my-custom-plugin --plugin-file "test/fixtures/input/connect/confluentinc-kafka-connect-datagen-0.6.1.zip" --connector-type source --connector-class io.confluent.kafka.connect.datagen.DatagenConnector --cloud aws`, fixture: "connect/custom-plugin/create.golden"},
209209
{args: `connect custom-plugin create my-custom-plugin --plugin-file "test/fixtures/input/connect/confluentinc-kafka-connect-datagen-0.6.1.zip" --connector-type source --connector-class io.confluent.kafka.connect.datagen.DatagenConnector`, fixture: "connect/custom-plugin/create.golden"},
210+
{args: `connect custom-plugin create my-custom-plugin --plugin-file "test/fixtures/input/connect/confluentinc-kafka-connect-datagen-0.6.1.zip" --connector-type source --connector-class io.confluent.kafka.connect.datagen.DatagenConnector --cloud gcp`, fixture: "connect/custom-plugin/create-gcp.golden"},
211+
{args: `connect custom-plugin create my-custom-plugin --plugin-file "test/fixtures/input/connect/confluentinc-kafka-connect-datagen-0.6.1.zip" --connector-type source --connector-class io.confluent.kafka.connect.datagen.DatagenConnector --cloud azure`, fixture: "connect/custom-plugin/create-azure.golden"},
210212
{args: `connect custom-plugin create my-custom-plugin --plugin-file "test/fixtures/input/connect/confluentinc-kafka-connect-datagen-0.6.1.pdf" --connector-type source --connector-class io.confluent.kafka.connect.datagen.DatagenConnector --cloud aws`, fixture: "connect/custom-plugin/create-invalid-extension.golden", exitCode: 1},
211213
{args: "connect custom-plugin list", fixture: "connect/custom-plugin/list.golden"},
212214
{args: "connect custom-plugin list --cloud aws", fixture: "connect/custom-plugin/list.golden"},
213215
{args: "connect custom-plugin list -o json", fixture: "connect/custom-plugin/list-json.golden"},
214216
{args: "connect custom-plugin list -o yaml", fixture: "connect/custom-plugin/list-yaml.golden"},
215217
{args: "connect custom-plugin describe ccp-123456", fixture: "connect/custom-plugin/describe.golden"},
216218
{args: "connect custom-plugin describe ccp-789012", fixture: "connect/custom-plugin/describe-with-sensitive-properties.golden"},
219+
{args: "connect custom-plugin describe ccp-401432", fixture: "connect/custom-plugin/describe-with-sensitive-properties-gcp.golden"},
217220
{args: "connect custom-plugin describe ccp-123456 -o json", fixture: "connect/custom-plugin/describe-json.golden"},
218221
{args: "connect custom-plugin describe ccp-123456 -o yaml", fixture: "connect/custom-plugin/describe-yaml.golden"},
219222
{args: "connect custom-plugin delete ccp-123456 --force", fixture: "connect/custom-plugin/delete.golden"},

‎test/fixtures/output/connect/custom-plugin/create-azure.golden

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
+-------+------------------------+
2+
| ID | ccp-123456 |
3+
| Name | my-custom-plugin-azure |
4+
| Cloud | AZURE |
5+
+-------+------------------------+

‎test/fixtures/output/connect/custom-plugin/create-gcp.golden

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
+-------+----------------------+
2+
| ID | ccp-123456 |
3+
| Name | my-custom-plugin-gcp |
4+
| Cloud | GCP |
5+
+-------+----------------------+

‎test/fixtures/output/connect/custom-plugin/create-help.golden

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ Flags:
1515
--description string Description of custom plugin.
1616
--documentation-link string Document link of custom plugin.
1717
--sensitive-properties strings A comma-separated list of sensitive property names.
18-
--cloud string Specify the cloud provider as "aws" or "azure". (default "aws")
18+
--cloud string Specify the cloud provider as "aws", "azure", or "gcp". (default "aws")
1919
--context string CLI context name.
2020
-o, --output string Specify the output format as "human", "json", or "yaml". (default "human")
2121

‎test/fixtures/output/connect/custom-plugin/describe-with-sensitive-properties-gcp.golden

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
+----------------------+---------------------------------+
2+
| ID | ccp-401432 |
3+
| Name | CliPluginTest |
4+
| Description | Source datagen plugin |
5+
| Cloud | GCP |
6+
| Connector Class | io.confluent.kafka.connect.test |
7+
| Connector Type | source |
8+
| Sensitive Properties | gcp.key, gcp.secret |
9+
+----------------------+---------------------------------+

‎test/fixtures/output/connect/custom-plugin/list-help.golden

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ List custom connector plugins in the org
99
$ confluent connect custom-plugin list --cloud aws
1010

1111
Flags:
12-
--cloud string Specify the cloud provider as "aws" or "azure".
12+
--cloud string Specify the cloud provider as "aws", "azure", or "gcp".
1313
--context string CLI context name.
1414
-o, --output string Specify the output format as "human", "json", or "yaml". (default "human")
1515

‎test/test-server/connect_handler.go

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -466,16 +466,36 @@ func handleCustomConnectorPlugins(t *testing.T) http.HandlerFunc {
466466
var decodeRespone connectcustompluginv1.ConnectV1CustomConnectorPlugin
467467
require.NoError(t, json.NewDecoder(r.Body).Decode(&decodeRespone))
468468
var plugin connectcustompluginv1.ConnectV1CustomConnectorPlugin
469-
switch strings.ToLower(decodeRespone.GetRuntimeLanguage()) {
470-
case "java", "":
471-
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
472-
Id: connectcustompluginv1.PtrString("ccp-123456"),
473-
DisplayName: connectcustompluginv1.PtrString("my-custom-plugin"),
474-
Cloud: connectcustompluginv1.PtrString("AWS"),
475-
ConnectorClass: connectcustompluginv1.PtrString("ver-123456"),
476-
ContentFormat: connectcustompluginv1.PtrString("JAR"),
469+
runtimeLanguage := strings.ToLower(decodeRespone.GetRuntimeLanguage())
470+
cloud := strings.ToLower(decodeRespone.GetCloud())
471+
472+
if runtimeLanguage == "java" || runtimeLanguage == "" {
473+
if cloud == "gcp" {
474+
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
475+
Id: connectcustompluginv1.PtrString("ccp-123456"),
476+
DisplayName: connectcustompluginv1.PtrString("my-custom-plugin-gcp"),
477+
Cloud: connectcustompluginv1.PtrString("GCP"),
478+
ConnectorClass: connectcustompluginv1.PtrString("ver-123456"),
479+
ContentFormat: connectcustompluginv1.PtrString("JAR"),
480+
}
481+
} else if cloud == "azure" {
482+
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
483+
Id: connectcustompluginv1.PtrString("ccp-123456"),
484+
DisplayName: connectcustompluginv1.PtrString("my-custom-plugin-azure"),
485+
Cloud: connectcustompluginv1.PtrString("AZURE"),
486+
ConnectorClass: connectcustompluginv1.PtrString("ver-123456"),
487+
ContentFormat: connectcustompluginv1.PtrString("JAR"),
488+
}
489+
} else {
490+
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
491+
Id: connectcustompluginv1.PtrString("ccp-123456"),
492+
DisplayName: connectcustompluginv1.PtrString("my-custom-plugin"),
493+
Cloud: connectcustompluginv1.PtrString("AWS"),
494+
ConnectorClass: connectcustompluginv1.PtrString("ver-123456"),
495+
ContentFormat: connectcustompluginv1.PtrString("JAR"),
496+
}
477497
}
478-
case "python":
498+
} else if runtimeLanguage == "python" {
479499
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
480500
Id: connectcustompluginv1.PtrString("ccp-789012"),
481501
DisplayName: connectcustompluginv1.PtrString("my-custom-python-plugin"),
@@ -538,6 +558,16 @@ func handleCustomConnectorPluginsId(t *testing.T) http.HandlerFunc {
538558
Cloud: connectcustompluginv1.PtrString("AWS"),
539559
SensitiveConfigProperties: &sensitiveProperties,
540560
}
561+
} else if id == "ccp-401432" {
562+
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
563+
Id: connectcustompluginv1.PtrString("ccp-401432"),
564+
DisplayName: connectcustompluginv1.PtrString("CliPluginTest"),
565+
Description: connectcustompluginv1.PtrString("Source datagen plugin"),
566+
ConnectorType: connectcustompluginv1.PtrString("source"),
567+
ConnectorClass: connectcustompluginv1.PtrString("io.confluent.kafka.connect.test"),
568+
Cloud: connectcustompluginv1.PtrString("GCP"),
569+
SensitiveConfigProperties: &[]string{"gcp.key", "gcp.secret"},
570+
}
541571
} else {
542572
plugin = connectcustompluginv1.ConnectV1CustomConnectorPlugin{
543573
Id: connectcustompluginv1.PtrString("ccp-789013"),

0 commit comments

Comments
 (0)