chore(gitops): handle non-utf8 encoding character (#11671)

backend/api/gitops/azure.go

@@ -67,7 +67,7 @@ func getAzurePullRequestInfo(ctx context.Context, vcsProvider *store.VCSProvider
 		if err != nil {
 			return nil, errors.Errorf("failed read file content, merge request %q, file %q, error %v", pushEvent.Resource.Links.Web, file.path, err)
 		}
-		file.content = content
+		file.content = convertFileContentToUTF8String(content)
 	}
 
 	return prInfo, nil

backend/api/gitops/bitbucket.go

@@ -41,7 +41,7 @@ func getBitBucketPullRequestInfo(ctx context.Context, vcsProvider *store.VCSProv
 		if err != nil {
 			return nil, errors.Errorf("failed read file content, merge request %q, file %q, error %v", pushEvent.PullRequest.Links.HTML.Href, file.path, err)
 		}
-		file.content = content
+		file.content = convertFileContentToUTF8String(content)
 	}
 	return prInfo, nil
 }

backend/api/gitops/common.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/bytebase/bytebase/backend/common/log"
 	"github.com/bytebase/bytebase/backend/plugin/vcs"
+	"github.com/bytebase/bytebase/backend/utils"
 	v1pb "github.com/bytebase/bytebase/proto/generated-go/v1"
 )
 
@@ -102,3 +103,12 @@ func getPullRequestID(url string) string {
 func getPullRequestComment(externalURL, issue string) string {
 	return fmt.Sprintf("Bytebase Bot: this pull request has triggered a Bytebase rollout 🚀. Check out the status at %s/%s.", externalURL, issue)
 }
+
+func convertFileContentToUTF8String(content string) string {
+	convertedContent, err := utils.ConvertBytesToUTF8String([]byte(content))
+	if err != nil {
+		// After failed to convert to UTF-8, we will try to convert to valid UTF-8.
+		convertedContent = strings.ToValidUTF8(content, "")
+	}
+	return convertedContent
+}

backend/api/gitops/github.go

@@ -48,7 +48,7 @@ func getGitHubPullRequestInfo(ctx context.Context, vcsProvider *store.VCSProvide
 		if err != nil {
 			return nil, errors.Errorf("failed read file content, merge request %q, file %q, error %v", pushEvent.PullRequest.HTMLURL, file.path, err)
 		}
-		file.content = content
+		file.content = convertFileContentToUTF8String(content)
 	}
 	return prInfo, nil
 }

backend/api/gitops/gitlab.go

@@ -52,7 +52,7 @@ func getGitLabPullRequestInfo(ctx context.Context, vcsProvider *store.VCSProvide
 		if err != nil {
 			return nil, errors.Errorf("failed read file content, merge request %q, file %q, error %v", pushEvent.ObjectAttributes.URL, file.path, err)
 		}
-		file.content = content
+		file.content = convertFileContentToUTF8String(content)
 	}
 	return prInfo, nil
 }

backend/utils/utils.go

@@ -6,15 +6,22 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log/slog"
 	"reflect"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 
 	"github.com/pkg/errors"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	textunicode "golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/encoding/protojson"
@@ -827,3 +834,27 @@ func uniq[T comparable](array []T) []T {
 
 	return res
 }
+
+// ConvertBytesToUTF8String tries to decode a byte slice into a UTF-8 string using common encodings.
+func ConvertBytesToUTF8String(data []byte) (string, error) {
+	encodings := []encoding.Encoding{
+		textunicode.UTF8,
+		simplifiedchinese.GBK,
+		textunicode.UTF16(textunicode.LittleEndian, textunicode.UseBOM),
+		textunicode.UTF16(textunicode.BigEndian, textunicode.UseBOM),
+		charmap.ISO8859_1,
+	}
+
+	for _, enc := range encodings {
+		reader := transform.NewReader(strings.NewReader(string(data)), enc.NewDecoder())
+		decoded, err := io.ReadAll(reader)
+		if err == nil && isUtf8(decoded) {
+			return string(decoded), nil
+		}
+	}
+	return "", errors.New("failed to decode the byte slice into a UTF-8 string")
+}
+
+func isUtf8(data []byte) bool {
+	return !strings.Contains(string(data), string(unicode.ReplacementChar))
+}

backend/utils/utils_test.go

@@ -432,3 +432,34 @@ func TestGetRenderedStatement(t *testing.T) {
 		assert.Equal(t, tc.expected, actual)
 	}
 }
+
+func TestConvertBytesToUTF8String(t *testing.T) {
+	tests := []struct {
+		input    []byte
+		expected string
+	}{
+		{
+			input:    []byte{},
+			expected: "",
+		},
+		{
+			input:    []byte("hello"),
+			expected: "hello",
+		},
+		{
+			input:    []byte("你好"),
+			expected: "你好",
+		},
+		{
+			// string: SELECT "�ݱ�˼"
+			input:    []byte{83, 69, 76, 69, 67, 84, 32, 34, 176, 221, 177, 180, 203, 188, 34},
+			expected: "SELECT \"拜贝思\"",
+		},
+	}
+
+	for _, test := range tests {
+		actual, err := ConvertBytesToUTF8String(test.input)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expected, actual)
+	}
+}

frontend/src/components/IssueV1/components/IssueCommentSection/IssueCommentView/ActionCreator.vue

@@ -8,7 +8,7 @@
   >
     {{ user?.title }}
   </component>
-  <div v-else class="inline-flex items-center">
+  <div v-else class="inline-flex items-center gap-1">
     <span class="font-medium text-main whitespace-nowrap">
       {{ user?.title }}
     </span>

frontend/src/components/IssueV1/components/Sidebar/VCSInfo.vue

@@ -3,7 +3,7 @@
     v-if="issue.planEntity?.vcsSource?.pullRequestUrl"
     class="text-sm text-control-light flex space-x-1 items-center"
   >
-    <VCSIcon :type="issue.planEntity?.vcsSource.vcsType" />
+    <VCSIcon custom-class="h-5" :type="issue.planEntity?.vcsSource.vcsType" />
     <EllipsisText>
       <a
         :href="issue.planEntity?.vcsSource.pullRequestUrl"