Merge pull request #1705 from anisurrahman75/fix-xml-decoding

Add charset converter to convert non `utf-8` char

internal/databases/sqlserver/blob/xml.go

@@ -2,14 +2,12 @@ package blob
 
 import (
 	"bytes"
-	"encoding/binary"
 	"encoding/xml"
 	"fmt"
+	"golang.org/x/net/html/charset"
 	"io"
 
 	"strings"
-	"unicode/utf16"
-	"unicode/utf8"
 )
 
 const (
@@ -121,9 +119,11 @@ func (bl *XBlockListIn) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 
 func ParseBlocklistXML(data []byte) (*XBlockListIn, error) {
 	bl := &XBlockListIn{}
-	data = utf16utf8(data, binary.LittleEndian)
 	d := xml.NewDecoder(bytes.NewBuffer(data))
 	d.CharsetReader = func(s string, r io.Reader) (io.Reader, error) {
+		if s == "utf-16" {
+			return charset.NewReader(r, "charset=utf-16")
+		}
 		return r, nil
 	}
 	err := d.Decode(bl)
@@ -133,18 +133,6 @@ func ParseBlocklistXML(data []byte) (*XBlockListIn, error) {
 	return bl, nil
 }
 
-// 21century, we can't convert charset in golang. nice
-func utf16utf8(b []byte, o binary.ByteOrder) []byte {
-	utf := make([]uint16, (len(b)+(2-1))/2)
-	for i := 0; i+(2-1) < len(b); i += 2 {
-		utf[i/2] = o.Uint16(b[i:])
-	}
-	if len(b)/2 < len(utf) {
-		utf[len(utf)-1] = utf8.RuneError
-	}
-	return []byte(string(utf16.Decode(utf)))
-}
-
 type XBlockListOut struct {
 	XMLName         xml.Name `xml:"BlockList"`
 	CommittedBlocks struct {