Defaulting to UTF-8 if charset is missing

Some servers don't return the charset. This causes german characters to be encoded incorrectly, since ISO_8859_1 does not work very well in such cases defaulting to UTF-8 if its missing. https://www.iana.org/assignments/media-types/text/csv
googleapis · Aug 5, 2021 · 765b1a7 · 765b1a7
1 parent 41e54fa
commit 765b1a7
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 0 deletions.
diff --git a/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java b/google-http-client/src/main/java/com/google/api/client/http/HttpResponse.java
@@ -534,6 +534,10 @@ public Charset getContentCharset() {
         // https://tools.ietf.org/html/rfc4627 - JSON must be encoded with UTF-8
         return StandardCharsets.UTF_8;
       }
+       // fallback to well-kown charset for text/csv
+      if ("text".equals(mediaType.getType()) && "csv".equals(mediaType.getSubType())) {
+        return StandardCharsets.UTF_8;
+      }
     }
     return StandardCharsets.ISO_8859_1;
   }

diff --git a/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java b/google-http-client/src/test/java/com/google/api/client/http/HttpResponseTest.java
@@ -68,6 +68,8 @@ public void testParseAsString_none() throws Exception {
   private static final String VALID_CONTENT_TYPE = "text/plain";
   private static final String VALID_CONTENT_TYPE_WITH_PARAMS =
       "application/vnd.com.google.datastore.entity+json; charset=utf-8; version=v1; q=0.9";
+  private static final String VALID_CONTENT_TYPE_WITHOUT_CHARSET =
+      "text/csv; version=v1; q=0.9";
   private static final String INVALID_CONTENT_TYPE = "!!!invalid!!!";
   private static final String JSON_CONTENT_TYPE = "application/json";
 
@@ -194,6 +196,32 @@ public LowLevelHttpResponse execute() throws IOException {
     assertEquals("ISO-8859-1", response.getContentCharset().name());
   }
 
+  public void testParseAsString_validContentTypeWithoutCharSetWithParams() throws Exception {
+    HttpTransport transport =
+        new MockHttpTransport() {
+          @Override
+          public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+            return new MockLowLevelHttpRequest() {
+              @Override
+              public LowLevelHttpResponse execute() throws IOException {
+                MockLowLevelHttpResponse result = new MockLowLevelHttpResponse();
+                result.setContent(SAMPLE2);
+                result.setContentType(VALID_CONTENT_TYPE_WITHOUT_CHARSET);
+                return result;
+              }
+            };
+          }
+        };
+    HttpRequest request =
+        transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL);
+
+    HttpResponse response = request.execute();
+    assertEquals(SAMPLE2, response.parseAsString());
+    assertEquals(VALID_CONTENT_TYPE_WITHOUT_CHARSET, response.getContentType());
+    assertNotNull(response.getMediaType());
+    assertEquals("UTF-8", response.getContentCharset().name());
+  }
+
   public void testParseAsString_jsonContentType() throws IOException {
     HttpTransport transport =
         new MockHttpTransport() {