    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017package org.apache.commons.io.input;
    018
    019import java.io.BufferedInputStream;
    020import java.io.BufferedReader;
    021import java.io.File;
    022import java.io.FileInputStream;
    023import java.io.IOException;
    024import java.io.InputStream;
    025import java.io.InputStreamReader;
    026import java.io.Reader;
    027import java.io.StringReader;
    028import java.net.HttpURLConnection;
    029import java.net.URL;
    030import java.net.URLConnection;
    031import java.text.MessageFormat;
    032import java.util.Locale;
    033import java.util.Objects;
    034import java.util.regex.Matcher;
    035import java.util.regex.Pattern;
    036
    037import org.apache.commons.io.ByteOrderMark;
    038import org.apache.commons.io.IOUtils;
    039
    040/**
    041 * Character stream that handles all the necessary Voodoo to figure out the
    042 * charset encoding of the XML document within the stream.
    043 * <p>
    044 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader.
    045 * This one IS a character stream.
    046 * <p>
    047 * All this has to be done without consuming characters from the stream, if not
    048 * the XML parser will not recognized the document as a valid XML. This is not
    049 * 100% true, but it's close enough (UTF-8 BOM is not handled by all parsers
    050 * right now, XmlStreamReader handles it and things work in all parsers).
    051 * <p>
    052 * The XmlStreamReader class handles the charset encoding of XML documents in
    053 * Files, raw streams and HTTP streams by offering a wide set of constructors.
    054 * <p>
    055 * By default the charset encoding detection is lenient, the constructor with
    056 * the lenient flag can be used for a script (following HTTP MIME and XML
    057 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a
    058 * href="http://diveintomark.org/archives/2004/02/13/xml-media-types">
    059 * Determining the character encoding of a feed</a>.
    060 * <p>
    061 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under
    062 * Apache License 2.0.
    063 *
    064 * @see org.apache.commons.io.output.XmlStreamWriter
    065 * @since 2.0
    066 */
    067public class XmlStreamReader extends Reader {
    068    private static final int BUFFER_SIZE = IOUtils.DEFAULT_BUFFER_SIZE;
    069
    070    private static final String UTF_8 = "UTF-8";
    071
    072    private static final String US_ASCII = "US-ASCII";
    073
    074    private static final String UTF_16BE = "UTF-16BE";
    075
    076    private static final String UTF_16LE = "UTF-16LE";
    077
    078    private static final String UTF_32BE = "UTF-32BE";
    079
    080    private static final String UTF_32LE = "UTF-32LE";
    081
    082    private static final String UTF_16 = "UTF-16";
    083
    084    private static final String UTF_32 = "UTF-32";
    085
    086    private static final String EBCDIC = "CP1047";
    087
    088    private static final ByteOrderMark[] BOMS = new ByteOrderMark[] {
    089        ByteOrderMark.UTF_8,
    090        ByteOrderMark.UTF_16BE,
    091        ByteOrderMark.UTF_16LE,
    092        ByteOrderMark.UTF_32BE,
    093        ByteOrderMark.UTF_32LE
    094    };
    095
    096    // UTF_16LE and UTF_32LE have the same two starting BOM bytes.
    097    private static final ByteOrderMark[] XML_GUESS_BYTES = new ByteOrderMark[] {
    098        new ByteOrderMark(UTF_8,    0x3C, 0x3F, 0x78, 0x6D),
    099        new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F),
    100        new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
    101        new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C,
    102                0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
    103        new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00,
    104                0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
    105        new ByteOrderMark(EBCDIC,   0x4C, 0x6F, 0xA7, 0x94)
    106    };
    107
    108    private final Reader reader;
    109
    110    private final String encoding;
    111
    112    private final String defaultEncoding;
    113
    114    /**
    115     * Returns the default encoding to use if none is set in HTTP content-type,
    116     * XML prolog and the rules based on content-type are not adequate.
    117     * <p>
    118     * If it is NULL the content-type based rules are used.
    119     *
    120     * @return the default encoding to use.
    121     */
    122    public String getDefaultEncoding() {
    123        return defaultEncoding;
    124    }
    125
    126    /**
    127     * Creates a Reader for a File.
    128     * <p>
    129     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset,
    130     * if this is also missing defaults to UTF-8.
    131     * <p>
    132     * It does a lenient charset encoding detection, check the constructor with
    133     * the lenient parameter for details.
    134     *
    135     * @param file File to create a Reader from.
    136     * @throws IOException thrown if there is a problem reading the file.
    137     */
    138    public XmlStreamReader(final File file) throws IOException {
    139        this(new FileInputStream(Objects.requireNonNull(file)));
    140    }
    141
    142    /**
    143     * Creates a Reader for a raw InputStream.
    144     * <p>
    145     * It follows the same logic used for files.
    146     * <p>
    147     * It does a lenient charset encoding detection, check the constructor with
    148     * the lenient parameter for details.
    149     *
    150     * @param inputStream InputStream to create a Reader from.
    151     * @throws IOException thrown if there is a problem reading the stream.
    152     */
    153    public XmlStreamReader(final InputStream inputStream) throws IOException {
    154        this(inputStream, true);
    155    }
    156
    157    /**
    158     * Creates a Reader for a raw InputStream.
    159     * <p>
    160     * It follows the same logic used for files.
    161     * <p>
    162     * If lenient detection is indicated and the detection above fails as per
    163     * specifications it then attempts the following:
    164     * <p>
    165     * If the content type was 'text/html' it replaces it with 'text/xml' and
    166     * tries the detection again.
    167     * <p>
    168     * Else if the XML prolog had a charset encoding that encoding is used.
    169     * <p>
    170     * Else if the content type had a charset encoding that encoding is used.
    171     * <p>
    172     * Else 'UTF-8' is used.
    173     * <p>
    174     * If lenient detection is indicated an XmlStreamReaderException is never
    175     * thrown.
    176     *
    177     * @param inputStream InputStream to create a Reader from.
    178     * @param lenient indicates if the charset encoding detection should be
    179     *        relaxed.
    180     * @throws IOException thrown if there is a problem reading the stream.
    181     * @throws XmlStreamReaderException thrown if the charset encoding could not
    182     *         be determined according to the specs.
    183     */
    184    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
    185        this(inputStream, lenient, null);
    186    }
    187
    188    /**
    189     * Creates a Reader for a raw InputStream.
    190     * <p>
    191     * It follows the same logic used for files.
    192     * <p>
    193     * If lenient detection is indicated and the detection above fails as per
    194     * specifications it then attempts the following:
    195     * <p>
    196     * If the content type was 'text/html' it replaces it with 'text/xml' and
    197     * tries the detection again.
    198     * <p>
    199     * Else if the XML prolog had a charset encoding that encoding is used.
    200     * <p>
    201     * Else if the content type had a charset encoding that encoding is used.
    202     * <p>
    203     * Else 'UTF-8' is used.
    204     * <p>
    205     * If lenient detection is indicated an XmlStreamReaderException is never
    206     * thrown.
    207     *
    208     * @param inputStream InputStream to create a Reader from.
    209     * @param lenient indicates if the charset encoding detection should be
    210     *        relaxed.
    211     * @param defaultEncoding The default encoding
    212     * @throws IOException thrown if there is a problem reading the stream.
    213     * @throws XmlStreamReaderException thrown if the charset encoding could not
    214     *         be determined according to the specs.
    215     */
    216    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding)
    217            throws IOException {
    218        Objects.requireNonNull(inputStream, "inputStream");
    219        this.defaultEncoding = defaultEncoding;
    220        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    221        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    222        this.encoding = doRawStream(bom, pis, lenient);
    223        this.reader = new InputStreamReader(pis, encoding);
    224    }
    225
    226    /**
    227     * Creates a Reader using the InputStream of a URL.
    228     * <p>
    229     * If the URL is not of type HTTP and there is not 'content-type' header in
    230     * the fetched data it uses the same logic used for Files.
    231     * <p>
    232     * If the URL is a HTTP Url or there is a 'content-type' header in the
    233     * fetched data it uses the same logic used for an InputStream with
    234     * content-type.
    235     * <p>
    236     * It does a lenient charset encoding detection, check the constructor with
    237     * the lenient parameter for details.
    238     *
    239     * @param url URL to create a Reader from.
    240     * @throws IOException thrown if there is a problem reading the stream of
    241     *         the URL.
    242     */
    243    public XmlStreamReader(final URL url) throws IOException {
    244        this(Objects.requireNonNull(url, "url").openConnection(), null);
    245    }
    246
    247    /**
    248     * Creates a Reader using the InputStream of a URLConnection.
    249     * <p>
    250     * If the URLConnection is not of type HttpURLConnection and there is not
    251     * 'content-type' header in the fetched data it uses the same logic used for
    252     * files.
    253     * <p>
    254     * If the URLConnection is a HTTP Url or there is a 'content-type' header in
    255     * the fetched data it uses the same logic used for an InputStream with
    256     * content-type.
    257     * <p>
    258     * It does a lenient charset encoding detection, check the constructor with
    259     * the lenient parameter for details.
    260     *
    261     * @param conn URLConnection to create a Reader from.
    262     * @param defaultEncoding The default encoding
    263     * @throws IOException thrown if there is a problem reading the stream of
    264     *         the URLConnection.
    265     */
    266    public XmlStreamReader(final URLConnection conn, final String defaultEncoding) throws IOException {
    267        Objects.requireNonNull(conn, "conm");
    268        this.defaultEncoding = defaultEncoding;
    269        final boolean lenient = true;
    270        final String contentType = conn.getContentType();
    271        final InputStream inputStream = conn.getInputStream();
    272        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    273        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    274        if (conn instanceof HttpURLConnection || contentType != null) {
    275            this.encoding = doHttpStream(bom, pis, contentType, lenient);
    276        } else {
    277            this.encoding = doRawStream(bom, pis, lenient);
    278        }
    279        this.reader = new InputStreamReader(pis, encoding);
    280    }
    281
    282    /**
    283     * Creates a Reader using an InputStream and the associated content-type
    284     * header.
    285     * <p>
    286     * First it checks if the stream has BOM. If there is not BOM checks the
    287     * content-type encoding. If there is not content-type encoding checks the
    288     * XML prolog encoding. If there is not XML prolog encoding uses the default
    289     * encoding mandated by the content-type MIME type.
    290     * <p>
    291     * It does a lenient charset encoding detection, check the constructor with
    292     * the lenient parameter for details.
    293     *
    294     * @param inputStream InputStream to create the reader from.
    295     * @param httpContentType content-type header to use for the resolution of
    296     *        the charset encoding.
    297     * @throws IOException thrown if there is a problem reading the file.
    298     */
    299    public XmlStreamReader(final InputStream inputStream, final String httpContentType)
    300            throws IOException {
    301        this(inputStream, httpContentType, true);
    302    }
    303
    304    /**
    305     * Creates a Reader using an InputStream and the associated content-type
    306     * header. This constructor is lenient regarding the encoding detection.
    307     * <p>
    308     * First it checks if the stream has BOM. If there is not BOM checks the
    309     * content-type encoding. If there is not content-type encoding checks the
    310     * XML prolog encoding. If there is not XML prolog encoding uses the default
    311     * encoding mandated by the content-type MIME type.
    312     * <p>
    313     * If lenient detection is indicated and the detection above fails as per
    314     * specifications it then attempts the following:
    315     * <p>
    316     * If the content type was 'text/html' it replaces it with 'text/xml' and
    317     * tries the detection again.
    318     * <p>
    319     * Else if the XML prolog had a charset encoding that encoding is used.
    320     * <p>
    321     * Else if the content type had a charset encoding that encoding is used.
    322     * <p>
    323     * Else 'UTF-8' is used.
    324     * <p>
    325     * If lenient detection is indicated an XmlStreamReaderException is never
    326     * thrown.
    327     *
    328     * @param inputStream InputStream to create the reader from.
    329     * @param httpContentType content-type header to use for the resolution of
    330     *        the charset encoding.
    331     * @param lenient indicates if the charset encoding detection should be
    332     *        relaxed.
    333     * @param defaultEncoding The default encoding
    334     * @throws IOException thrown if there is a problem reading the file.
    335     * @throws XmlStreamReaderException thrown if the charset encoding could not
    336     *         be determined according to the specs.
    337     */
    338    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
    339            final boolean lenient, final String defaultEncoding) throws IOException {
    340        Objects.requireNonNull(inputStream, "inputStream");
    341        this.defaultEncoding = defaultEncoding;
    342        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
    343        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
    344        this.encoding = doHttpStream(bom, pis, httpContentType, lenient);
    345        this.reader = new InputStreamReader(pis, encoding);
    346    }
    347
    348    /**
    349     * Creates a Reader using an InputStream and the associated content-type
    350     * header. This constructor is lenient regarding the encoding detection.
    351     * <p>
    352     * First it checks if the stream has BOM. If there is not BOM checks the
    353     * content-type encoding. If there is not content-type encoding checks the
    354     * XML prolog encoding. If there is not XML prolog encoding uses the default
    355     * encoding mandated by the content-type MIME type.
    356     * <p>
    357     * If lenient detection is indicated and the detection above fails as per
    358     * specifications it then attempts the following:
    359     * <p>
    360     * If the content type was 'text/html' it replaces it with 'text/xml' and
    361     * tries the detection again.
    362     * <p>
    363     * Else if the XML prolog had a charset encoding that encoding is used.
    364     * <p>
    365     * Else if the content type had a charset encoding that encoding is used.
    366     * <p>
    367     * Else 'UTF-8' is used.
    368     * <p>
    369     * If lenient detection is indicated an XmlStreamReaderException is never
    370     * thrown.
    371     *
    372     * @param inputStream InputStream to create the reader from.
    373     * @param httpContentType content-type header to use for the resolution of
    374     *        the charset encoding.
    375     * @param lenient indicates if the charset encoding detection should be
    376     *        relaxed.
    377     * @throws IOException thrown if there is a problem reading the file.
    378     * @throws XmlStreamReaderException thrown if the charset encoding could not
    379     *         be determined according to the specs.
    380     */
    381    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
    382            final boolean lenient) throws IOException {
    383        this(inputStream, httpContentType, lenient, null);
    384    }
    385
    386    /**
    387     * Returns the charset encoding of the XmlStreamReader.
    388     *
    389     * @return charset encoding.
    390     */
    391    public String getEncoding() {
    392        return encoding;
    393    }
    394
    395    /**
    396     * Invokes the underlying reader's <code>read(char[], int, int)</code> method.
    397     * @param buf the buffer to read the characters into
    398     * @param offset The start offset
    399     * @param len The number of bytes to read
    400     * @return the number of characters read or -1 if the end of stream
    401     * @throws IOException if an I/O error occurs
    402     */
    403    @Override
    404    public int read(final char[] buf, final int offset, final int len) throws IOException {
    405        return reader.read(buf, offset, len);
    406    }
    407
    408    /**
    409     * Closes the XmlStreamReader stream.
    410     *
    411     * @throws IOException thrown if there was a problem closing the stream.
    412     */
    413    @Override
    414    public void close() throws IOException {
    415        reader.close();
    416    }
    417
    418    /**
    419     * Process the raw stream.
    420     *
    421     * @param bom BOMInputStream to detect byte order marks
    422     * @param pis BOMInputStream to guess XML encoding
    423     * @param lenient indicates if the charset encoding detection should be
    424     *        relaxed.
    425     * @return the encoding to be used
    426     * @throws IOException thrown if there is a problem reading the stream.
    427     */
    428    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient)
    429            throws IOException {
    430        final String bomEnc      = bom.getBOMCharsetName();
    431        final String xmlGuessEnc = pis.getBOMCharsetName();
    432        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
    433        try {
    434            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
    435        } catch (final XmlStreamReaderException ex) {
    436            if (lenient) {
    437                return doLenientDetection(null, ex);
    438            }
    439            throw ex;
    440        }
    441    }
    442
    443    /**
    444     * Process a HTTP stream.
    445     *
    446     * @param bom BOMInputStream to detect byte order marks
    447     * @param pis BOMInputStream to guess XML encoding
    448     * @param httpContentType The HTTP content type
    449     * @param lenient indicates if the charset encoding detection should be
    450     *        relaxed.
    451     * @return the encoding to be used
    452     * @throws IOException thrown if there is a problem reading the stream.
    453     */
    454    private String doHttpStream(final BOMInputStream bom, final BOMInputStream pis, final String httpContentType,
    455            final boolean lenient) throws IOException {
    456        final String bomEnc      = bom.getBOMCharsetName();
    457        final String xmlGuessEnc = pis.getBOMCharsetName();
    458        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
    459        try {
    460            return calculateHttpEncoding(httpContentType, bomEnc,
    461                    xmlGuessEnc, xmlEnc, lenient);
    462        } catch (final XmlStreamReaderException ex) {
    463            if (lenient) {
    464                return doLenientDetection(httpContentType, ex);
    465            }
    466            throw ex;
    467        }
    468    }
    469
    470    /**
    471     * Do lenient detection.
    472     *
    473     * @param httpContentType content-type header to use for the resolution of
    474     *        the charset encoding.
    475     * @param ex The thrown exception
    476     * @return the encoding
    477     * @throws IOException thrown if there is a problem reading the stream.
    478     */
    479    private String doLenientDetection(String httpContentType,
    480            XmlStreamReaderException ex) throws IOException {
    481        if (httpContentType != null && httpContentType.startsWith("text/html")) {
    482            httpContentType = httpContentType.substring("text/html".length());
    483            httpContentType = "text/xml" + httpContentType;
    484            try {
    485                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(),
    486                        ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
    487            } catch (final XmlStreamReaderException ex2) {
    488                ex = ex2;
    489            }
    490        }
    491        String encoding = ex.getXmlEncoding();
    492        if (encoding == null) {
    493            encoding = ex.getContentTypeEncoding();
    494        }
    495        if (encoding == null) {
    496            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
    497        }
    498        return encoding;
    499    }
    500
    501    /**
    502     * Calculate the raw encoding.
    503     *
    504     * @param bomEnc BOM encoding
    505     * @param xmlGuessEnc XML Guess encoding
    506     * @param xmlEnc XML encoding
    507     * @return the raw encoding
    508     * @throws IOException thrown if there is a problem reading the stream.
    509     */
    510    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc,
    511            final String xmlEnc) throws IOException {
    512
    513        // BOM is Null
    514        if (bomEnc == null) {
    515            if (xmlGuessEnc == null || xmlEnc == null) {
    516                return defaultEncoding == null ? UTF_8 : defaultEncoding;
    517            }
    518            if (xmlEnc.equals(UTF_16) &&
    519               (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
    520                return xmlGuessEnc;
    521            }
    522            return xmlEnc;
    523        }
    524
    525        // BOM is UTF-8
    526        if (bomEnc.equals(UTF_8)) {
    527            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
    528                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    529                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    530            }
    531            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
    532                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    533                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    534            }
    535            return bomEnc;
    536        }
    537
    538        // BOM is UTF-16BE or UTF-16LE
    539        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
    540            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
    541                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    542                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    543            }
    544            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
    545                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    546                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    547            }
    548            return bomEnc;
    549        }
    550
    551        // BOM is UTF-32BE or UTF-32LE
    552        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
    553            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
    554                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    555                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    556            }
    557            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
    558                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
    559                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    560            }
    561            return bomEnc;
    562        }
    563
    564        // BOM is something else
    565        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
    566        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
    567    }
    568
    569
    570    /**
    571     * Calculate the HTTP encoding.
    572     *
    573     * @param httpContentType The HTTP content type
    574     * @param bomEnc BOM encoding
    575     * @param xmlGuessEnc XML Guess encoding
    576     * @param xmlEnc XML encoding
    577     * @param lenient indicates if the charset encoding detection should be
    578     *        relaxed.
    579     * @return the HTTP encoding
    580     * @throws IOException thrown if there is a problem reading the stream.
    581     */
    582    String calculateHttpEncoding(final String httpContentType,
    583            final String bomEnc, final String xmlGuessEnc, final String xmlEnc,
    584            final boolean lenient) throws IOException {
    585
    586        // Lenient and has XML encoding
    587        if (lenient && xmlEnc != null) {
    588            return xmlEnc;
    589        }
    590
    591        // Determine mime/encoding content types from HTTP Content Type
    592        final String cTMime = getContentTypeMime(httpContentType);
    593        final String cTEnc  = getContentTypeEncoding(httpContentType);
    594        final boolean appXml  = isAppXml(cTMime);
    595        final boolean textXml = isTextXml(cTMime);
    596
    597        // Mime type NOT "application/xml" or "text/xml"
    598        if (!appXml && !textXml) {
    599            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    600            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    601        }
    602
    603        // No content type encoding
    604        if (cTEnc == null) {
    605            if (appXml) {
    606                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
    607            }
    608            return defaultEncoding == null ? US_ASCII : defaultEncoding;
    609        }
    610
    611        // UTF-16BE or UTF-16LE content type encoding
    612        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
    613            if (bomEnc != null) {
    614                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    615                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    616            }
    617            return cTEnc;
    618        }
    619
    620        // UTF-16 content type encoding
    621        if (cTEnc.equals(UTF_16)) {
    622            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
    623                return bomEnc;
    624            }
    625            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    626            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    627        }
    628
    629        // UTF-32BE or UTF-132E content type encoding
    630        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
    631            if (bomEnc != null) {
    632                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    633                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    634            }
    635            return cTEnc;
    636        }
    637
    638        // UTF-32 content type encoding
    639        if (cTEnc.equals(UTF_32)) {
    640            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
    641                return bomEnc;
    642            }
    643            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    644            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
    645        }
    646
    647        return cTEnc;
    648    }
    649
    650    /**
    651     * Returns MIME type or NULL if httpContentType is NULL.
    652     *
    653     * @param httpContentType the HTTP content type
    654     * @return The mime content type
    655     */
    656    static String getContentTypeMime(final String httpContentType) {
    657        String mime = null;
    658        if (httpContentType != null) {
    659            final int i = httpContentType.indexOf(";");
    660            if (i >= 0) {
    661                mime = httpContentType.substring(0, i);
    662            } else {
    663                mime = httpContentType;
    664            }
    665            mime = mime.trim();
    666        }
    667        return mime;
    668    }
    669
    670    private static final Pattern CHARSET_PATTERN = Pattern
    671            .compile("charset=[\"']?([.[^; \"']]*)[\"']?");
    672
    673    /**
    674     * Returns charset parameter value, NULL if not present, NULL if
    675     * httpContentType is NULL.
    676     *
    677     * @param httpContentType the HTTP content type
    678     * @return The content type encoding (upcased)
    679     */
    680    static String getContentTypeEncoding(final String httpContentType) {
    681        String encoding = null;
    682        if (httpContentType != null) {
    683            final int i = httpContentType.indexOf(";");
    684            if (i > -1) {
    685                final String postMime = httpContentType.substring(i + 1);
    686                final Matcher m = CHARSET_PATTERN.matcher(postMime);
    687                encoding = m.find() ? m.group(1) : null;
    688                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
    689            }
    690        }
    691        return encoding;
    692    }
    693
    694    public static final Pattern ENCODING_PATTERN = Pattern.compile(
    695            "<\\?xml.*encoding[\\s]*=[\\s]*((?:\".[^\"]*\")|(?:'.[^']*'))",
    696            Pattern.MULTILINE);
    697
    698    /**
    699     * Returns the encoding declared in the <?xml encoding=...?>, NULL if none.
    700     *
    701     * @param inputStream InputStream to create the reader from.
    702     * @param guessedEnc guessed encoding
    703     * @return the encoding declared in the <?xml encoding=...?>
    704     * @throws IOException thrown if there is a problem reading the stream.
    705     */
    706    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc)
    707            throws IOException {
    708        String encoding = null;
    709        if (guessedEnc != null) {
    710            final byte[] bytes = new byte[BUFFER_SIZE];
    711            inputStream.mark(BUFFER_SIZE);
    712            int offset = 0;
    713            int max = BUFFER_SIZE;
    714            int c = inputStream.read(bytes, offset, max);
    715            int firstGT = -1;
    716            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
    717            while (c != -1 && firstGT == -1 && offset < BUFFER_SIZE) {
    718                offset += c;
    719                max -= c;
    720                c = inputStream.read(bytes, offset, max);
    721                xmlProlog = new String(bytes, 0, offset, guessedEnc);
    722                firstGT = xmlProlog.indexOf('>');
    723            }
    724            if (firstGT == -1) {
    725                if (c == -1) {
    726                    throw new IOException("Unexpected end of XML stream");
    727                }
    728                throw new IOException(
    729                        "XML prolog or ROOT element not found on first "
    730                                + offset + " bytes");
    731            }
    732            final int bytesRead = offset;
    733            if (bytesRead > 0) {
    734                inputStream.reset();
    735                final BufferedReader bReader = new BufferedReader(new StringReader(
    736                        xmlProlog.substring(0, firstGT + 1)));
    737                final StringBuffer prolog = new StringBuffer();
    738                String line = bReader.readLine();
    739                while (line != null) {
    740                    prolog.append(line);
    741                    line = bReader.readLine();
    742                }
    743                final Matcher m = ENCODING_PATTERN.matcher(prolog);
    744                if (m.find()) {
    745                    encoding = m.group(1).toUpperCase(Locale.ROOT);
    746                    encoding = encoding.substring(1, encoding.length() - 1);
    747                }
    748            }
    749        }
    750        return encoding;
    751    }
    752
    753    /**
    754     * Indicates if the MIME type belongs to the APPLICATION XML family.
    755     *
    756     * @param mime The mime type
    757     * @return true if the mime type belongs to the APPLICATION XML family,
    758     * otherwise false
    759     */
    760    static boolean isAppXml(final String mime) {
    761        return mime != null &&
    762               (mime.equals("application/xml") ||
    763                mime.equals("application/xml-dtd") ||
    764                mime.equals("application/xml-external-parsed-entity") ||
    765               mime.startsWith("application/") && mime.endsWith("+xml"));
    766    }
    767
    768    /**
    769     * Indicates if the MIME type belongs to the TEXT XML family.
    770     *
    771     * @param mime The mime type
    772     * @return true if the mime type belongs to the TEXT XML family,
    773     * otherwise false
    774     */
    775    static boolean isTextXml(final String mime) {
    776        return mime != null &&
    777              (mime.equals("text/xml") ||
    778               mime.equals("text/xml-external-parsed-entity") ||
    779              mime.startsWith("text/") && mime.endsWith("+xml"));
    780    }
    781
    782    private static final String RAW_EX_1 =
    783        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
    784
    785    private static final String RAW_EX_2 =
    786        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
    787
    788    private static final String HTTP_EX_1 =
    789        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be NULL";
    790
    791    private static final String HTTP_EX_2 =
    792        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
    793
    794    private static final String HTTP_EX_3 =
    795        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Invalid MIME";
    796
    797}