Fix HTTP response charset parameter parsing

sentinel-transport/sentinel-transport-simple-http/src/main/java/com/alibaba/csp/sentinel/transport/heartbeat/client/SimpleHttpResponse.java

@@ -18,6 +18,8 @@
 import com.alibaba.csp.sentinel.config.SentinelConfig;
 
 import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -45,13 +47,78 @@ public SimpleHttpResponse(String statusLine, Map<String, String> headers, byte[]
         this.body = body;
     }
 
+    /**
+     * Resolve the charset from the {@code Content-Type} header, which is a media type with optional
+     * parameters, e.g. {@code text/plain; charset="UTF-8"} (RFC 7231, section 3.1.1.5). When the
+     * header is absent or carries no usable {@code charset} parameter, the default charset is used.
+     * An unsupported charset value still fails with {@link java.nio.charset.UnsupportedCharsetException}.
+     */
     private void parseCharset() {
+        // Reset to the default on every call, as the headers may have been mutated in between.
+        charset = Charset.forName(SentinelConfig.charset());
         String contentType = getHeader("Content-Type");
-        for (String str : contentType.split(" ")) {
-            if (str.toLowerCase().startsWith("charset=")) {
-                charset = Charset.forName(str.split("=")[1]);
+        if (contentType == null) {
+            return;
+        }
+        for (String parameter : splitParameters(contentType)) {
+            int idx = parameter.indexOf('=');
+            // Skip the media type itself and any parameter that is not a charset.
+            if (idx < 0 || !"charset".equalsIgnoreCase(parameter.substring(0, idx).trim())) {
+                continue;
             }
+            String value = unquote(parameter.substring(idx + 1).trim());
+            if (!value.isEmpty()) {
+                charset = Charset.forName(value);
+            }
+            return;
+        }
+    }
+
+    /**
+     * Split the parameters of a media type value on {@code ;}, ignoring separators inside a quoted
+     * value, so that e.g. {@code text/plain; boundary="a;b"; charset=UTF-8} is split correctly.
+     * A quoted-pair ({@code \} followed by any character) inside a quoted value is kept as-is and
+     * never ends the quoted value, per RFC 7231, section 3.1.1.1.
+     *
+     * @param contentType the media type value
+     * @return the media type and its parameters
+     */
+    private static List<String> splitParameters(String contentType) {
+        List<String> parameters = new ArrayList<String>();
+        StringBuilder current = new StringBuilder();
+        boolean quoted = false;
+        for (int i = 0; i < contentType.length(); i++) {
+            char c = contentType.charAt(i);
+            if (quoted && c == '\\' && i + 1 < contentType.length()) {
+                // Keep the escaped character verbatim: it may be a quote or a separator.
+                current.append(c).append(contentType.charAt(++i));
+                continue;
+            }
+            if (c == '"') {
+                quoted = !quoted;
+            }
+            if (c == ';' && !quoted) {
+                parameters.add(current.toString());
+                current.setLength(0);
+            } else {
+                current.append(c);
+            }
+        }
+        parameters.add(current.toString());
+        return parameters;
+    }
+
+    /**
+     * Strip the surrounding balanced double quotes of a parameter value, if any.
+     *
+     * @param value the parameter value
+     * @return the unquoted value
+     */
+    private static String unquote(String value) {
+        if (value.length() >= 2 && value.charAt(0) == '"' && value.charAt(value.length() - 1) == '"') {
+            return value.substring(1, value.length() - 1).trim();
         }
+        return value;
     }
 
     private void parseCode() {

sentinel-transport/sentinel-transport-simple-http/src/test/java/com/alibaba/csp/sentinel/transport/heartbeat/client/SimpleHttpResponseTest.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright 1999-2018 Alibaba Group Holding Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.csp.sentinel.transport.heartbeat.client;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import com.alibaba.csp.sentinel.config.SentinelConfig;
+
+/**
+ * Test cases for {@link SimpleHttpResponse#getBodyAsString()}, focused on the charset declared by
+ * the {@code Content-Type} header (a media type with optional parameters, RFC 7231, 3.1.1.5).
+ *
+ * <p>The bodies below are encoded with a non-default charset, so that a decode performed with the
+ * wrong charset yields replacement characters and fails the assertion.</p>
+ */
+public class SimpleHttpResponseTest {
+
+    private static final String STATUS_LINE = "HTTP/1.1 200 OK";
+
+    private static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1");
+    private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+    /** Text with bytes that are valid ISO-8859-1 but not valid UTF-8. */
+    private static final String ISO_8859_1_TEXT = "caf\u00e9 \u00e0 la carte";
+    private static final byte[] ISO_8859_1_BODY = ISO_8859_1_TEXT.getBytes(ISO_8859_1);
+
+    /** Chinese text requiring multi-byte UTF-8 sequences, to keep the UTF-8 cases honest. */
+    private static final String UTF_8_TEXT = "\u4e2d\u6587\u6d4b\u8bd5";
+    private static final byte[] UTF_8_BODY = UTF_8_TEXT.getBytes(UTF_8);
+
+    private static Charset defaultCharset() {
+        return Charset.forName(SentinelConfig.charset());
+    }
+
+    private static SimpleHttpResponse response(String headerName, String headerValue, byte[] body) {
+        Map<String, String> headers = new HashMap<String, String>();
+        if (headerName != null) {
+            headers.put(headerName, headerValue);
+        }
+        return new SimpleHttpResponse(STATUS_LINE, headers, body);
+    }
+
+    private static SimpleHttpResponse response(String contentType, byte[] body) {
+        return response("Content-Type", contentType, body);
+    }
+
+    @Test
+    public void getBodyAsStringShouldUseDefaultCharsetWhenContentTypeHeaderIsAbsent() {
+        assertEquals(new String(ISO_8859_1_BODY, defaultCharset()),
+            response(null, ISO_8859_1_BODY).getBodyAsString());
+        assertEquals(new String(UTF_8_BODY, defaultCharset()),
+            response(null, UTF_8_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldUseDefaultCharsetWhenNoCharsetParameterIsGiven() {
+        assertEquals(new String(ISO_8859_1_BODY, defaultCharset()),
+            response("text/plain", ISO_8859_1_BODY).getBodyAsString());
+        assertEquals(new String(UTF_8_BODY, defaultCharset()),
+            response("text/html; boundary=xyz", UTF_8_BODY).getBodyAsString());
+        assertEquals(new String(ISO_8859_1_BODY, defaultCharset()),
+            response("text/plain; foo=bar; baz=qux", ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldHonorCharsetParameterWithoutWhitespaceAfterSemicolon() {
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain;charset=ISO-8859-1", ISO_8859_1_BODY).getBodyAsString());
+        // Both the header name and the charset parameter name are case-insensitive.
+        assertEquals(ISO_8859_1_TEXT,
+            response("content-type", "text/plain;CHARSET=ISO-8859-1", ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldHonorQuotedCharsetParameter() {
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain; charset=\"ISO-8859-1\"", ISO_8859_1_BODY).getBodyAsString());
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain;charset=\"ISO-8859-1\"", ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldHonorCharsetParameterFollowedByOtherParameters() {
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain; charset=ISO-8859-1; boundary=xyz", ISO_8859_1_BODY).getBodyAsString());
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain;charset=ISO-8859-1;boundary=xyz", ISO_8859_1_BODY).getBodyAsString());
+        // The charset parameter must be honored whatever its position among the parameters.
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain; boundary=xyz; charset=ISO-8859-1", ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldHonorExplicitUtf8Charset() {
+        assertEquals(UTF_8_TEXT,
+            response("application/json;charset=UTF-8", UTF_8_BODY).getBodyAsString());
+        assertEquals(UTF_8_TEXT,
+            response("application/json; charset=utf-8", UTF_8_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldNotPromoteACharsetInsideAQuotedPair() {
+        // Here `boundary="a\";charset=US-ASCII"` is one quoted value whose quoted-pair `\"` is a
+        // literal quote, so the `charset=US-ASCII` it contains is not a parameter: only the real
+        // charset outside the quoted value must be honored.
+        assertEquals(ISO_8859_1_TEXT,
+            response("multipart/form-data; boundary=\"a\\\";charset=US-ASCII\"; charset=ISO-8859-1",
+                ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldTolerateWhitespaceAroundTheEqualsSign() {
+        // Tolerate whitespace around the parameter delimiter.
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain; charset = ISO-8859-1", ISO_8859_1_BODY).getBodyAsString());
+        assertEquals(ISO_8859_1_TEXT,
+            response("text/plain;charset= ISO-8859-1", ISO_8859_1_BODY).getBodyAsString());
+    }
+
+    @Test
+    public void getBodyAsStringShouldResetToTheDefaultCharsetWhenHeadersAreMutated() {
+        SimpleHttpResponse response = response("text/plain; charset=ISO-8859-1", ISO_8859_1_BODY);
+        assertEquals(ISO_8859_1_TEXT, response.getBodyAsString());
+
+        response.getHeaders().put("Content-Type", "text/plain");
+
+        assertEquals(new String(ISO_8859_1_BODY, defaultCharset()), response.getBodyAsString());
+    }
+
+    @Test(expected = UnsupportedCharsetException.class)
+    public void getBodyAsStringShouldThrowOnAnUnsupportedCharsetParameter() {
+        response("text/plain; charset=no-such-charset", UTF_8_BODY).getBodyAsString();
+    }
+}