    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017package org.apache.commons.io.input;
    018
    019import java.io.BufferedInputStream;
    020import java.io.BufferedReader;
    021import java.io.File;
    022import java.io.FileInputStream;
    023import java.io.IOException;
    024import java.io.InputStream;
    025import java.io.InputStreamReader;
    026import java.io.Reader;
    027import java.io.StringReader;
    028import java.net.HttpURLConnection;
    029import java.net.URL;
    030import java.net.URLConnection;
    031import java.text.MessageFormat;
    032import java.util.Locale;
    033import java.util.Objects;
    034import java.util.regex.Matcher;
    035import java.util.regex.Pattern;
    036
    037import org.apache.commons.io.ByteOrderMark;
    038import org.apache.commons.io.IOUtils;
    039
    040/**
    041 * Character stream that handles all the necessary Voodoo to figure out the
    042 * charset encoding of the XML document within the stream.
    043 * <p>
    044 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader.
    045 * This one IS a character stream.
    046 * </p>
    047 * <p>
    048 * All this has to be done without consuming characters from the stream, if not
    049 * the XML parser will not recognized the document as a valid XML. This is not
    050 * 100% true, but it's close enough (UTF-8 BOM is not handled by all parsers
    051 * right now, XmlStreamReader handles it and things work in all parsers).
    052 * </p>
    053 * <p>
    054 * The XmlStreamReader class handles the charset encoding of XML documents in
    055 * Files, raw streams and HTTP streams by offering a wide set of constructors.
    056 * </p>
    057 * <p>
    058 * By default the charset encoding detection is lenient, the constructor with
    059 * the lenient flag can be used for a script (following HTTP MIME and XML
    060 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a
    061 * href="http://diveintomark.org/archives/2004/02/13/xml-media-types">
    062 * Determining the character encoding of a feed</a>.
    063 * </p>
    064 * <p>
    065 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under
    066 * Apache License 2.0.
    067 * </p>
    068 *
    069 * @see org.apache.commons.io.output.XmlStreamWriter
    070 * @since 2.0
    071 */
    072public class XmlStreamReader extends Reader {
    073    private static final int BUFFER_SIZE = IOUtils.DEFAULT_BUFFER_SIZE;
    074
    075    private static final String UTF_8 = "UTF-8";
    076
    077    private static final String US_ASCII = "US-ASCII";
    078
    079    private static final String UTF_16BE = "UTF-16BE";
    080
    081    private static final String UTF_16LE = "UTF-16LE";
    082
    083    private static final String UTF_32BE = "UTF-32BE";
    084
    085    private static final String UTF_32LE = "UTF-32LE";
    086
    087    private static final String UTF_16 = "UTF-16";
    088
    089    private static final String UTF_32 = "UTF-32";
    090
    091    private static final String EBCDIC = "CP1047";
    092
    093    private static final ByteOrderMark[] BOMS = new ByteOrderMark[] {
    094        ByteOrderMark.UTF_8,
    095        ByteOrderMark.UTF_16BE,
    096        ByteOrderMark.UTF_16LE,
    097        ByteOrderMark.UTF_32BE,
    098        ByteOrderMark.UTF_32LE
    099    };
    100
    101    // UTF_16LE and UTF_32LE have the same two starting BOM bytes.
    102    private static final ByteOrderMark[] XML_GUESS_BYTES = new ByteOrderMark[] {
    103        new ByteOrderMark(UTF_8,    0x3C, 0x3F, 0x78, 0x6D),
    104        new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F),
    105        new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
    106        new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C,
    107                0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
    108        new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00,
    109                0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
    110        new ByteOrderMark(EBCDIC,   0x4C, 0x6F, 0xA7, 0x94)
    111    };
    112
    113    private final Reader reader;
    114
    115    private final String encoding;
    116
    117    private final String defaultEncoding;
    118
    119    /**
    120     * Returns the default encoding to use if none is set in HTTP content-type,
    121     * XML prolog and the rules based on content-type are not adequate.
    122     * <p>
    123     * If it is NULL the content-type based rules are used.
    124     *
    125     * @return the default encoding to use.
    126     */
    127    public String getDefaultEncoding() {
    128        return defaultEncoding;
    129    }
    130
    131    /**
    132     * Creates a Reader for a File.
    133     * <p>
    134     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset,
    135     * if this is also missing defaults to UTF-8.
    136     * <p>
    137     * It does a lenient charset encoding detection, check the constructor with
    138     * the lenient parameter for details.
    139     *
    140     * @param file File to create a Reader from.
    141     * @throws IOException thrown if there is a problem reading the file.
    142     */
    143    public XmlStreamReader(final File file) throws IOException {
    144        this(new FileInputStream(Objects.requireNonNull(file)));
    145    }
    146
    147    /**
    148     * Creates a Reader for a raw InputStream.
    149     * <p>
    150     * It follows the same logic used for files.
    151     * <p>
    152     * It does a lenient charset encoding detection, check the constructor with
    153     * the lenient parameter for details.
    154     *
    155     * @param inputStream InputStream to create a Reader from.
    156     * @throws IOException thrown if there is a problem reading the stream.
    157     */
    158    public XmlStreamReader(final InputStream inputStream) throws IOException {
    159        this(inputStream, true);
    160    }
    161
    162    /**
    163     * Creates a Reader for a raw InputStream.
    164     * <p>
    165     * It follows the same logic used for files.
    166     * <p>
    167     * If lenient detection is indicated and the detection above fails as per
    168     * specifications it then attempts the following:
    169     * <p>
    170     * If the content type was 'text/html' it replaces it with 'text/xml' and
    171     * tries the detection again.
    172     * <p>
    173     * Else if the XML prolog had a charset encoding that encoding is used.
    174     * <p>
    175     * Else if the content type had a charset encoding that encoding is used.
    176     * <p>
    177     * Else 'UTF-8' is used.
    178     * <p>
    179     * If lenient detection is indicated an XmlStreamReaderException is never
    180     * thrown.
    181     *
    182     * @param inputStream InputStream to create a Reader from.
    183     * @param lenient indicates if the charset encoding detection should be
    184     *        relaxed.
    185     * @throws IOException thrown if there is a problem reading the stream.
    186     * @throws XmlStreamReaderException thrown if the charset encoding could not
    187     *         be determined according to the specs.
    188     */
    189    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
    190        this(inputStream, lenient, null);
    191    }
    192
    193    /**
    194     * Creates a Reader for a raw InputStream.
    195     * <p>
    196     * It follows the same logic used for files.
    197     * <p>
    198     * If lenient detection is indicated and the detection above fails as per
    199     * specifications it then attempts the following:
    200     * <p>
    201     * If the content type was 'text/html' it replaces it with 'text/xml' and
    202     * tries the detection again.
    203     * <p>
    204     * Else if the XML prolog had a charset encoding that encoding is used.
    205     * <p>
    206     * Else if the content type had a charset encoding that encoding is used.
    207     * <p>
    208     * Else 'UTF-8' is used.
    209     * <p>
    210     * If lenient detection is indicated an XmlStreamReaderException is never
    211     * thrown.
    212     *
    213     * @param inputStream InputStream to create a Reader from.
    214     * @param lenient indicates if the charset encoding detection should be
    215     *        relaxed.
    216     * @param defaultEncoding The default encoding
    217     * @throws IOException thrown if there is a problem reading the stream.
    218     * @throws XmlStreamReaderException thrown if the charset encoding could not
    219     *         be determined according to the specs.
    220     */
    221    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding)
    222            throws IOException {
    223        Objects.requireNonNull(inputStream, "inputStream");
    224        this.defaultEncoding = defaultEncoding;
    225        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    226        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    227        this.encoding = doRawStream(bom, pis, lenient);
    228        this.reader = new InputStreamReader(pis, encoding);
    229    }
    230
    231    /**
    232     * Creates a Reader using the InputStream of a URL.
    233     * <p>
    234     * If the URL is not of type HTTP and there is not 'content-type' header in
    235     * the fetched data it uses the same logic used for Files.
    236     * <p>
    237     * If the URL is a HTTP Url or there is a 'content-type' header in the
    238     * fetched data it uses the same logic used for an InputStream with
    239     * content-type.
    240     * <p>
    241     * It does a lenient charset encoding detection, check the constructor with
    242     * the lenient parameter for details.
    243     *
    244     * @param url URL to create a Reader from.
    245     * @throws IOException thrown if there is a problem reading the stream of
    246     *         the URL.
    247     */
    248    public XmlStreamReader(final URL url) throws IOException {
    249        this(Objects.requireNonNull(url, "url").openConnection(), null);
    250    }
    251
    252    /**
    253     * Creates a Reader using the InputStream of a URLConnection.
    254     * <p>
    255     * If the URLConnection is not of type HttpURLConnection and there is not
    256     * 'content-type' header in the fetched data it uses the same logic used for
    257     * files.
    258     * <p>
    259     * If the URLConnection is a HTTP Url or there is a 'content-type' header in
    260     * the fetched data it uses the same logic used for an InputStream with
    261     * content-type.
    262     * <p>
    263     * It does a lenient charset encoding detection, check the constructor with
    264     * the lenient parameter for details.
    265     *
    266     * @param conn URLConnection to create a Reader from.
    267     * @param defaultEncoding The default encoding
    268     * @throws IOException thrown if there is a problem reading the stream of
    269     *         the URLConnection.
    270     */
    271    public XmlStreamReader(final URLConnection conn, final String defaultEncoding) throws IOException {
    272        Objects.requireNonNull(conn, "conm");
    273        this.defaultEncoding = defaultEncoding;
    274        final boolean lenient = true;
    275        final String contentType = conn.getContentType();
    276        final InputStream inputStream = conn.getInputStream();
    277        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    278        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    279        if (conn instanceof HttpURLConnection || contentType != null) {
    280            this.encoding = processHttpStream(bom, pis, contentType, lenient);
    281        } else {
    282            this.encoding = doRawStream(bom, pis, lenient);
    283        }
    284        this.reader = new InputStreamReader(pis, encoding);
    285    }
    286
    287    /**
    288     * Creates a Reader using an InputStream and the associated content-type
    289     * header.
    290     * <p>
    291     * First it checks if the stream has BOM. If there is not BOM checks the
    292     * content-type encoding. If there is not content-type encoding checks the
    293     * XML prolog encoding. If there is not XML prolog encoding uses the default
    294     * encoding mandated by the content-type MIME type.
    295     * <p>
    296     * It does a lenient charset encoding detection, check the constructor with
    297     * the lenient parameter for details.
    298     *
    299     * @param inputStream InputStream to create the reader from.
    300     * @param httpContentType content-type header to use for the resolution of
    301     *        the charset encoding.
    302     * @throws IOException thrown if there is a problem reading the file.
    303     */
    304    public XmlStreamReader(final InputStream inputStream, final String httpContentType)
    305            throws IOException {
    306        this(inputStream, httpContentType, true);
    307    }
    308
    309    /**
    310     * Creates a Reader using an InputStream and the associated content-type
    311     * header. This constructor is lenient regarding the encoding detection.
    312     * <p>
    313     * First it checks if the stream has BOM. If there is not BOM checks the
    314     * content-type encoding. If there is not content-type encoding checks the
    315     * XML prolog encoding. If there is not XML prolog encoding uses the default
    316     * encoding mandated by the content-type MIME type.
    317     * <p>
    318     * If lenient detection is indicated and the detection above fails as per
    319     * specifications it then attempts the following:
    320     * <p>
    321     * If the content type was 'text/html' it replaces it with 'text/xml' and
    322     * tries the detection again.
    323     * <p>
    324     * Else if the XML prolog had a charset encoding that encoding is used.
    325     * <p>
    326     * Else if the content type had a charset encoding that encoding is used.
    327     * <p>
    328     * Else 'UTF-8' is used.
    329     * <p>
    330     * If lenient detection is indicated an XmlStreamReaderException is never
    331     * thrown.
    332     *
    333     * @param inputStream InputStream to create the reader from.
    334     * @param httpContentType content-type header to use for the resolution of
    335     *        the charset encoding.
    336     * @param lenient indicates if the charset encoding detection should be
    337     *        relaxed.
    338     * @param defaultEncoding The default encoding
    339     * @throws IOException thrown if there is a problem reading the file.
    340     * @throws XmlStreamReaderException thrown if the charset encoding could not
    341     *         be determined according to the specs.
    342     */
    343    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
    344            final boolean lenient, final String defaultEncoding) throws IOException {
    345        Objects.requireNonNull(inputStream, "inputStream");
    346        this.defaultEncoding = defaultEncoding;
    347        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    348        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    349        this.encoding = processHttpStream(bom, pis, httpContentType, lenient);
    350        this.reader = new InputStreamReader(pis, encoding);
    351    }
    352
    353    /**
    354     * Creates a Reader using an InputStream and the associated content-type
    355     * header. This constructor is lenient regarding the encoding detection.
    356     * <p>
    357     * First it checks if the stream has BOM. If there is not BOM checks the
    358     * content-type encoding. If there is not content-type encoding checks the
    359     * XML prolog encoding. If there is not XML prolog encoding uses the default
    360     * encoding mandated by the content-type MIME type.
    361     * <p>
    362     * If lenient detection is indicated and the detection above fails as per
    363     * specifications it then attempts the following:
    364     * <p>
    365     * If the content type was 'text/html' it replaces it with 'text/xml' and
    366     * tries the detection again.
    367     * <p>
    368     * Else if the XML prolog had a charset encoding that encoding is used.
    369     * <p>
    370     * Else if the content type had a charset encoding that encoding is used.
    371     * <p>
    372     * Else 'UTF-8' is used.
    373     * <p>
    374     * If lenient detection is indicated an XmlStreamReaderException is never
    375     * thrown.
    376     *
    377     * @param inputStream InputStream to create the reader from.
    378     * @param httpContentType content-type header to use for the resolution of
    379     *        the charset encoding.
    380     * @param lenient indicates if the charset encoding detection should be
    381     *        relaxed.
    382     * @throws IOException thrown if there is a problem reading the file.
    383     * @throws XmlStreamReaderException thrown if the charset encoding could not
    384     *         be determined according to the specs.
    385     */
    386    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
    387            final boolean lenient) throws IOException {
    388        this(inputStream, httpContentType, lenient, null);
    389    }
    390
    391    /**
    392     * Returns the charset encoding of the XmlStreamReader.
    393     *
    394     * @return charset encoding.
    395     */
    396    public String getEncoding() {
    397        return encoding;
    398    }
    399
    400    /**
    401     * Invokes the underlying reader's <code>read(char[], int, int)</code> method.
    402     * @param buf the buffer to read the characters into
    403     * @param offset The start offset
    404     * @param len The number of bytes to read
    405     * @return the number of characters read or -1 if the end of stream
    406     * @throws IOException if an I/O error occurs
    407     */
    408    @Override
    409    public int read(final char[] buf, final int offset, final int len) throws IOException {
    410        return reader.read(buf, offset, len);
    411    }
    412
    413    /**
    414     * Closes the XmlStreamReader stream.
    415     *
    416     * @throws IOException thrown if there was a problem closing the stream.
    417     */
    418    @Override
    419    public void close() throws IOException {
    420        reader.close();
    421    }
    422
    423    /**
    424     * Process the raw stream.
    425     *
    426     * @param bom BOMInputStream to detect byte order marks
    427     * @param pis BOMInputStream to guess XML encoding
    428     * @param lenient indicates if the charset encoding detection should be
    429     *        relaxed.
    430     * @return the encoding to be used
    431     * @throws IOException thrown if there is a problem reading the stream.
    432     */
    433    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient)
    434            throws IOException {
    435        final String bomEnc      = bom.getBOMCharsetName();
    436        final String xmlGuessEnc = pis.getBOMCharsetName();
    437        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
    438        try {
    439            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
    440        } catch (final XmlStreamReaderException ex) {
    441            if (lenient) {
    442                return doLenientDetection(null, ex);
    443            }
    444            throw ex;
    445        }
    446    }
    447
    448    /**
    449     * Process a HTTP stream.
    450     *
    451     * @param bom BOMInputStream to detect byte order marks
    452     * @param pis BOMInputStream to guess XML encoding
    453     * @param httpContentType The HTTP content type
    454     * @param lenient indicates if the charset encoding detection should be
    455     *        relaxed.
    456     * @return the encoding to be used
    457     * @throws IOException thrown if there is a problem reading the stream.
    458     */
    459    private String processHttpStream(final BOMInputStream bom, final BOMInputStream pis, final String httpContentType,
    460        final boolean lenient) throws IOException {
    461        final String bomEnc = bom.getBOMCharsetName();
    462        final String xmlGuessEnc = pis.getBOMCharsetName();
    463        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
    464        try {
    465            return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient);
    466        } catch (final XmlStreamReaderException ex) {
    467            if (lenient) {
    468                return doLenientDetection(httpContentType, ex);
    469            }
    470            throw ex;
    471        }
    472    }
    473
    474    /**
    475     * Do lenient detection.
    476     *
    477     * @param httpContentType content-type header to use for the resolution of
    478     *        the charset encoding.
    479     * @param ex The thrown exception
    480     * @return the encoding
    481     * @throws IOException thrown if there is a problem reading the stream.
    482     */
    483    private String doLenientDetection(String httpContentType,
    484            XmlStreamReaderException ex) throws IOException {
    485        if (httpContentType != null && httpContentType.startsWith("text/html")) {
    486            httpContentType = httpContentType.substring("text/html".length());
    487            httpContentType = "text/xml" + httpContentType;
    488            try {
    489                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(),
    490                        ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
    491            } catch (final XmlStreamReaderException ex2) {
    492                ex = ex2;
    493            }
    494        }
    495        String encoding = ex.getXmlEncoding();
    496        if (encoding == null) {
    497            encoding = ex.getContentTypeEncoding();
    498        }
    499        if (encoding == null) {
    500            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
    501        }
    502        return encoding;
    503    }
    504
    505    /**
    506     * Calculate the raw encoding.
    507     *
    508     * @param bomEnc BOM encoding
    509     * @param xmlGuessEnc XML Guess encoding
    510     * @param xmlEnc XML encoding
    511     * @return the raw encoding
    512     * @throws IOException thrown if there is a problem reading the stream.
    513     */
    514    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc,
    515            final String xmlEnc) throws IOException {
    516
    517        // BOM is Null
    518        if (bomEnc == null) {
    519            if (xmlGuessEnc == null || xmlEnc == null) {
    520                return defaultEncoding == null ? UTF_8 : defaultEncoding;
    521            }
    522            if (xmlEnc.equals(UTF_16) &&
    523               (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
    524                return xmlGuessEnc;
    525            }
    526            return xmlEnc;
    527        }
    528
    529        // BOM is UTF-8
    530        if (bomEnc.equals(UTF_8)) {
    531            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
    532                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    533                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    534            }
    535            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
    536                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    537                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    538            }
    539            return bomEnc;
    540        }
    541
    542        // BOM is UTF-16BE or UTF-16LE
    543        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
    544            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
    545                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    546                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    547            }
    548            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
    549                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    550                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    551            }
    552            return bomEnc;
    553        }
    554
    555        // BOM is UTF-32BE or UTF-32LE
    556        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
    557            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
    558                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    559                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    560            }
    561            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
    562                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    563                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    564            }
    565            return bomEnc;
    566        }
    567
    568        // BOM is something else
    569        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
    570        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    571    }
    572
    573
    574    /**
    575     * Calculate the HTTP encoding.
    576     *
    577     * @param httpContentType The HTTP content type
    578     * @param bomEnc BOM encoding
    579     * @param xmlGuessEnc XML Guess encoding
    580     * @param xmlEnc XML encoding
    581     * @param lenient indicates if the charset encoding detection should be
    582     *        relaxed.
    583     * @return the HTTP encoding
    584     * @throws IOException thrown if there is a problem reading the stream.
    585     */
    586    String calculateHttpEncoding(final String httpContentType,
    587            final String bomEnc, final String xmlGuessEnc, final String xmlEnc,
    588            final boolean lenient) throws IOException {
    589
    590        // Lenient and has XML encoding
    591        if (lenient && xmlEnc != null) {
    592            return xmlEnc;
    593        }
    594
    595        // Determine mime/encoding content types from HTTP Content Type
    596        final String cTMime = getContentTypeMime(httpContentType);
    597        final String cTEnc  = getContentTypeEncoding(httpContentType);
    598        final boolean appXml  = isAppXml(cTMime);
    599        final boolean textXml = isTextXml(cTMime);
    600
    601        // Mime type NOT "application/xml" or "text/xml"
    602        if (!appXml && !textXml) {
    603            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    604            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    605        }
    606
    607        // No content type encoding
    608        if (cTEnc == null) {
    609            if (appXml) {
    610                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
    611            }
    612            return defaultEncoding == null ? US_ASCII : defaultEncoding;
    613        }
    614
    615        // UTF-16BE or UTF-16LE content type encoding
    616        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
    617            if (bomEnc != null) {
    618                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    619                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    620            }
    621            return cTEnc;
    622        }
    623
    624        // UTF-16 content type encoding
    625        if (cTEnc.equals(UTF_16)) {
    626            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
    627                return bomEnc;
    628            }
    629            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    630            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    631        }
    632
    633        // UTF-32BE or UTF-132E content type encoding
    634        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
    635            if (bomEnc != null) {
    636                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    637                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    638            }
    639            return cTEnc;
    640        }
    641
    642        // UTF-32 content type encoding
    643        if (cTEnc.equals(UTF_32)) {
    644            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
    645                return bomEnc;
    646            }
    647            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    648            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    649        }
    650
    651        return cTEnc;
    652    }
    653
    654    /**
    655     * Returns MIME type or NULL if httpContentType is NULL.
    656     *
    657     * @param httpContentType the HTTP content type
    658     * @return The mime content type
    659     */
    660    static String getContentTypeMime(final String httpContentType) {
    661        String mime = null;
    662        if (httpContentType != null) {
    663            final int i = httpContentType.indexOf(";");
    664            if (i >= 0) {
    665                mime = httpContentType.substring(0, i);
    666            } else {
    667                mime = httpContentType;
    668            }
    669            mime = mime.trim();
    670        }
    671        return mime;
    672    }
    673
    674    private static final Pattern CHARSET_PATTERN = Pattern
    675            .compile("charset=[\"']?([.[^; \"']]*)[\"']?");
    676
    677    /**
    678     * Returns charset parameter value, NULL if not present, NULL if
    679     * httpContentType is NULL.
    680     *
    681     * @param httpContentType the HTTP content type
    682     * @return The content type encoding (upcased)
    683     */
    684    static String getContentTypeEncoding(final String httpContentType) {
    685        String encoding = null;
    686        if (httpContentType != null) {
    687            final int i = httpContentType.indexOf(";");
    688            if (i > -1) {
    689                final String postMime = httpContentType.substring(i + 1);
    690                final Matcher m = CHARSET_PATTERN.matcher(postMime);
    691                encoding = m.find() ? m.group(1) : null;
    692                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
    693            }
    694        }
    695        return encoding;
    696    }
    697
    698    /**
    699     * Pattern capturing the encoding of the "xml" processing instruction.
    700     */
    701    public static final Pattern ENCODING_PATTERN = Pattern.compile(
    702            "<\\?xml.*encoding[\\s]*=[\\s]*((?:\".[^\"]*\")|(?:'.[^']*'))",
    703            Pattern.MULTILINE);
    704
    705    /**
    706     * Returns the encoding declared in the <?xml encoding=...?>, NULL if none.
    707     *
    708     * @param inputStream InputStream to create the reader from.
    709     * @param guessedEnc guessed encoding
    710     * @return the encoding declared in the <?xml encoding=...?>
    711     * @throws IOException thrown if there is a problem reading the stream.
    712     */
    713    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc)
    714            throws IOException {
    715        String encoding = null;
    716        if (guessedEnc != null) {
    717            final byte[] bytes = new byte[BUFFER_SIZE];
    718            inputStream.mark(BUFFER_SIZE);
    719            int offset = 0;
    720            int max = BUFFER_SIZE;
    721            int c = inputStream.read(bytes, offset, max);
    722            int firstGT = -1;
    723            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
    724            while (c != -1 && firstGT == -1 && offset < BUFFER_SIZE) {
    725                offset += c;
    726                max -= c;
    727                c = inputStream.read(bytes, offset, max);
    728                xmlProlog = new String(bytes, 0, offset, guessedEnc);
    729                firstGT = xmlProlog.indexOf('>');
    730            }
    731            if (firstGT == -1) {
    732                if (c == -1) {
    733                    throw new IOException("Unexpected end of XML stream");
    734                }
    735                throw new IOException(
    736                        "XML prolog or ROOT element not found on first "
    737                                + offset + " bytes");
    738            }
    739            final int bytesRead = offset;
    740            if (bytesRead > 0) {
    741                inputStream.reset();
    742                final BufferedReader bReader = new BufferedReader(new StringReader(
    743                        xmlProlog.substring(0, firstGT + 1)));
    744                final StringBuffer prolog = new StringBuffer();
    745                String line;
    746                while ((line = bReader.readLine()) != null) {
    747                    prolog.append(line);
    748                }
    749                final Matcher m = ENCODING_PATTERN.matcher(prolog);
    750                if (m.find()) {
    751                    encoding = m.group(1).toUpperCase(Locale.ROOT);
    752                    encoding = encoding.substring(1, encoding.length() - 1);
    753                }
    754            }
    755        }
    756        return encoding;
    757    }
    758
    759    /**
    760     * Indicates if the MIME type belongs to the APPLICATION XML family.
    761     *
    762     * @param mime The mime type
    763     * @return true if the mime type belongs to the APPLICATION XML family,
    764     * otherwise false
    765     */
    766    static boolean isAppXml(final String mime) {
    767        return mime != null &&
    768               (mime.equals("application/xml") ||
    769                mime.equals("application/xml-dtd") ||
    770                mime.equals("application/xml-external-parsed-entity") ||
    771               mime.startsWith("application/") && mime.endsWith("+xml"));
    772    }
    773
    774    /**
    775     * Indicates if the MIME type belongs to the TEXT XML family.
    776     *
    777     * @param mime The mime type
    778     * @return true if the mime type belongs to the TEXT XML family,
    779     * otherwise false
    780     */
    781    static boolean isTextXml(final String mime) {
    782        return mime != null &&
    783              (mime.equals("text/xml") ||
    784               mime.equals("text/xml-external-parsed-entity") ||
    785              mime.startsWith("text/") && mime.endsWith("+xml"));
    786    }
    787
    788    private static final String RAW_EX_1 =
    789        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
    790
    791    private static final String RAW_EX_2 =
    792        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
    793
    794    private static final String HTTP_EX_1 =
    795        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be NULL";
    796
    797    private static final String HTTP_EX_2 =
    798        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
    799
    800    private static final String HTTP_EX_3 =
    801        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Invalid MIME";
    802
    803}