/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tika.parser.html;


import org.apache.tika.parser.html.charsetdetector.StandardHtmlEncodingDetector;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.html.charsetdetector.charsets.ReplacementCharset;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;

public class StandardHtmlEncodingDetectorTest {
    private Metadata metadata = new Metadata();

    @Before
    public void setUp() {
        this.metadata = new Metadata();
    }

    @Test
    public void basic() throws IOException {
        assertWindows1252("<meta charset=WINDOWS-1252>");
    }

    @Test
    public void quoted() throws IOException {
        assertWindows1252("<meta charset='WINDOWS-1252'>");
    }

    @Test
    public void duplicateMeta() throws IOException {
        assertWindows1252("<meta charset='WINDOWS-1252'>" +
                "<meta charset='UTF-8'>");
    }

    @Test
    public void duplicateAttribute() throws IOException {
        assertWindows1252("<meta charset='WINDOWS-1252' charset='UTF-8'>");
    }

    @Test
    public void invalidThenValid() throws IOException {
        assertCharset("<meta charset=blah>" +
                "<meta charset=WINDOWS-1252>", null);
    }

    @Test
    public void spacesInAttributes() throws IOException {
        assertWindows1252("<meta charset\u000C=  \t  WINDOWS-1252>");
    }

    @Test
    public void httpEquiv() throws IOException {
        assertWindows1252("<meta " +
                "http-equiv='content-type' " +
                "content='text/html; charset=\"WINDOWS-1252\"'>"); // quotes around the charset are allowed
        assertWindows1252("<meta " +
                "content=' charset  =  WINDOWS-1252' " + // The charset may be anywhere in the content attribute
                "http-equiv='content-type' >");
    }

    @Test
    public void emptyAttributeEnd() throws IOException {
        assertWindows1252("<meta charset=WINDOWS-1252 a>");
    }

    @Test
    public void httpEquivDuplicateCharset() throws IOException {
        assertWindows1252("<meta " +
                "http-equiv='content-type' " +
                "content='charset=WINDOWS-1252;" + // The detection should stop after the semicolon
                "charset=UTF-8'>");
    }

    @Test
    public void htmlFragment() throws IOException {
        assertWindows1252("<!doctype html><html class=nojs><head><meta charset='WINDOWS-1252'>");
    }

    @Test
    public void veryBadHtml() throws IOException {
        // check that the parser is not confused by garbage before the declaration
        assertWindows1252("<< l \" == / '=x\n >" +
                "<!--> " +
                "< <x'/ <=> " +
                "<meta/>" +
                "<meta>" +
                "<a x/>" +
                "<meta charset='WINDOWS-1252'>");
    }

    @Test
    public void specialTag() throws IOException {
        // special tags cannot have arguments, any '>' ends them
        assertWindows1252("<? x='><meta charset='WINDOWS-1252'>");
    }

    @Test
    public void longHtml() throws IOException {
        StringBuilder sb = new StringBuilder("<!doctype html>\n" +
                "<html>\n" +
                "<head>\n" +
                "<title>Hello world</title>\n");
        String repeated = "<meta x='y' />\n";
        String charsetMeta = "<meta charset='windows-1252'>";

        while (sb.length() + repeated.length() + charsetMeta.length() < 1024) sb.append(repeated);

        sb.append(charsetMeta);

        assertWindows1252(sb.toString());
    }

    @Test
    public void tooLong() throws IOException {
        // Create a string with 1Mb of '\0' followed by a meta
        String padded = new String(new byte[1000000], StandardCharsets.ISO_8859_1) + "<meta charset='windows-1252'>";
        // Only the first bytes should be prescanned, so the algorithm should stop before the meta tag
        assertCharset(padded, null);
    }

    @Test
    public void incompleteMeta() throws IOException {
        assertCharset("<meta charset='WINDOWS-1252'", null); // missing '>' at the end
    }

    @Test
    public void charsetWithWhiteSpaces() throws IOException {
        assertWindows1252("<meta charset='   \t\n  WINDOWS-1252 \t\n'>");
    }

    @Test
    public void mixedCase() throws IOException {
        assertWindows1252("<mEtA chArsEt='WInDOWs-1252'>");
    }

    @Test
    public void utf16() throws IOException {
        // According to the specification 'If charset is a UTF-16 encoding, then set charset to UTF-8.'
        assertCharset("<meta charset='UTF-16BE'>", StandardCharsets.UTF_8);
    }

    @Test
    public void xUserDefined() throws IOException {
        // According to the specification 'If charset is x-user-defined, then set charset to windows-1252.'
        assertWindows1252("<meta charset='x-user-defined'>");
    }

    @Test
    public void replacement() throws IOException {
        // Several dangerous charsets should are aliases of 'replacement' in the spec
        String inString = "<meta charset='iso-2022-cn'>";
        assertCharset(new ByteArrayInputStream(inString.getBytes(StandardCharsets.ISO_8859_1)), new ReplacementCharset());
    }

    @Test
    public void iso88591() throws IOException {
        // In the spec, iso-8859-1 is an alias for WINDOWS-1252
        assertWindows1252("<meta charset='iso-8859-1'>");
    }

    @Test
    public void macintoshEncoding() throws IOException {
        // The mac roman encoding exists in java, but under the name x-MacRoman
        assertCharset("<meta charset='macintosh'>", Charset.forName("x-MacRoman"));
    }

    @Test
    public void bom() throws IOException {
        // A BOM should have precedence over the meta
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_8);
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_16LE);
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_16BE);
    }

    @Test
    public void withSlash() throws IOException {
        assertWindows1252("<meta/charset='WINDOWS-1252'>");
    }

    @Test
    public void insideDescription() throws IOException {
        assertWindows1252("<meta name='description'" +
                "content='If I write charset=UTF-8 here, it doesnt mean the page is in UTF-8'/>" +
                "<meta charset='WINDOWS-1252'>");
    }

    @Test
    public void insideTag() throws IOException {
        assertWindows1252("<tag " +
                "attribute=\"<meta charset='UTF-8'>\" " + // inside attribute
                "<meta charset='UTF-8' " + // still inside tag
                "/>" + // tag end
                "<meta charset='WINDOWS-1252'>");
    }

    @Test
    public void missingAttribute() throws IOException {
        assertWindows1252(
                "<meta content='charset=UTF-8'>" + // missing http-equiv attribute
                        "<meta charset='WINDOWS-1252'>" // valid declaration
        );
    }

    @Test
    public void insideSpecialTag() throws IOException {
        // Content inside <?, <!, and </ should be ignored
        for (byte b : "?!/".getBytes(StandardCharsets.US_ASCII))
            assertWindows1252(
                    "<" + (char) b + // start comment
                            "<meta charset='UTF-8'>" + // inside special tag
                            "<meta charset='WINDOWS-1252'>" // real charset declaration
            );
    }

    @Test
    public void spaceBeforeTag() throws IOException {
        assertWindows1252(
                "< meta charset='UTF-8'>" + // invalid charset declaration
                        "<meta charset='WINDOWS-1252'>" // real charset declaration
        );
    }

    @Test
    public void invalidAttribute() throws IOException {
        assertWindows1252(
                "<meta " +
                        "badcharset='UTF-8' " + // invalid charset declaration
                        "charset='WINDOWS-1252'>" // real charset declaration
        );
    }

    @Test
    public void unmatchedQuote() throws IOException {
        assertWindows1252(
                "<meta http-equiv='content-type' content='charset=\"UTF-8'>" + // invalid charset declaration
                        "<meta charset='WINDOWS-1252'>" // real charset declaration
        );
    }

    @Test
    public void realWorld() throws IOException {
        assertWindows1252("<!DOCTYPE html>\n" +
                "<html lang=\"fr\">\n" +
                "<head>\n" +
                "<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':\n" +
                "\t\t\tnew Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],\n" +
                "\t\t\tj=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=\n" +
                "\t\t\t'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);\n" +
                "\t\t\t})(window,document,'script','dataLayer','GTM-PNX8H8X');</script>\n" +
                "<title>Horaires Transilien 2018 - Lignes A B C D E H J K L N P R U</title>\n" +
                "<meta name=\"description\" content=\"Consultez les horaires du Transilien en temps réel. Lignes A et B du RER. Lignes C D E H J K L N P R U du Transilien.\">\n" +
                "<meta name=\"keywords\" content=\"horaires transilien\">\n" +
                "<meta charset=\"windows-1252\">\n" +
                "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n" +
                "<meta name=\"robots\" content=\"follow, index\">\n" +
                "<base hr");
    }

    @Test
    public void withCompactComment() throws IOException {
        // <!--> is a valid comment
        assertWindows1252(
                "<!--" + // start comment
                        "<meta charset='UTF-8'>" + // inside comment
                        "-->" + // end comment
                        "<!-->" + // compact comment
                        "<meta charset='WINDOWS-1252'>" // outside comment, charset declaration
        );
    }

    @Test
    public void withCharsetInContentType() throws IOException {
        metadata.set(Metadata.CONTENT_TYPE, "text/html; Charset=ISO-8859-1");
        // ISO-8859-1 is an alias for WINDOWS-1252, even if it's set at the transport layer level
        assertWindows1252("");
        assertWindows1252("<meta charset='UTF-8'>");
        assertWindows1252("<meta http-equiv='content-type' content='charset=utf-8'>");
        // if a BOM is present, it has precedence over transport layer information
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_8);
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_16LE);
        assertCharset("\ufeff<meta charset='WINDOWS-1252'>", StandardCharsets.UTF_16BE);
    }

    @Test
    public void throwResistance() throws IOException {
        // The preprocessing should return right after having found the charset
        // So if an error is thrown in the stream AFTER the declaration,
        // it shouldn't see it
        assertWindows1252(throwAfter("<meta charset='WINDOWS-1252'>"));
        assertWindows1252(throwAfter("<meta charset='WINDOWS-1252'><some other tag"));

        // But if an error is thrown before the end of the meta tag, it should see it
        // and return unsuccessfully
        assertCharset(throwAfter("<meta charset='WINDOWS-1252'"), null);

        // If there is no meta, but an error is thrown, the detector simply returns
        // unsuccessfully (it should not throw runtime errors)
        assertCharset(throwAfter("<"), null);
        assertCharset(throwAfter("<!"), null);
        assertCharset(throwAfter("<!doctype"), null);
        assertCharset(throwAfter("<!doctype html><html"), null);
        assertCharset(throwAfter("<!doctype html><html attr"), null);
        assertCharset(throwAfter("<!doctype html><html attr="), null);
        assertCharset(throwAfter("<!doctype html><html attr=x"), null);
        assertCharset(throwAfter("<!doctype html><html attr='x"), null);
    }

    @Test
    public void streamReset() throws IOException {
        // The stream should be reset after detection
        byte[] inBytes = {0,1,2,3,4};
        byte[] outBytes = new byte[5];
        InputStream inStream = new ByteArrayInputStream(inBytes);
        detectCharset(inStream);
        // The stream should still be readable from the beginning after detection
        inStream.read(outBytes);
        assertArrayEquals(inBytes, outBytes);
    }

    private void assertWindows1252(String html) throws IOException {
        assertCharset(html, Charset.forName("WINDOWS-1252"));
    }

    private void assertWindows1252(InputStream inStream) throws IOException {
        assertCharset(inStream, Charset.forName("WINDOWS-1252"));
    }

    private void assertCharset(String html, Charset charset) throws IOException {
        final Charset contentsCharset = (charset == null) ? StandardCharsets.UTF_8 : charset;
        InputStream inStream = new ByteArrayInputStream(html.getBytes(contentsCharset));
        final Charset detected = detectCharset(inStream);
        assertEquals(html + " should be detected as " + charset, charset, detected);
    }

    private void assertCharset(InputStream inStream, Charset charset) throws IOException {
        final Charset detected = detectCharset(inStream);
        assertEquals(charset, detected);
    }

    private Charset detectCharset(InputStream inStream) throws IOException {
        return new StandardHtmlEncodingDetector().detect(inStream, metadata);
    }

    private InputStream throwAfter(String html) {
        byte[] contents = html.getBytes(StandardCharsets.UTF_8);
        InputStream contentsInStream = new ByteArrayInputStream(contents);
        InputStream errorThrowing = new InputStream() {
            @Override
            public int read() throws IOException {
                throw new IOException("test exception");
            }
        };
        return new BufferedInputStream(new SequenceInputStream(contentsInStream, errorThrowing));
    }
}
