/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.daffodil.xml

import java.io.BufferedInputStream
import java.net.URI

import scala.io.Source
import scala.xml._
import scala.xml.include.sax.EncodingHeuristics
import scala.xml.parsing.ConstructingParser

import org.apache.daffodil.exceptions.Assert

/**
 * Scala 2.11 deprecated the Position object so it is no longer public.
 * However, we still need a way to decode the integer positions that contain
 * both line/col information. Scala provided no other way to decode this, so
 * this copies the relevant bits from:
 *
 * https://github.com/scala/scala/blob/2.11.x/src/library/scala/io/Position.scala
 *
 * Note that if scala ever changes thes values, line/column numbers will be off
 */
object Position {
  /** Number of bits used to encode the line number */
  final val LINE_BITS = 20
  /** Number of bits used to encode the column number */
  final val COLUMN_BITS = 31 - LINE_BITS // no negatives => 31
  /** Mask to decode the line number */
  final val LINE_MASK = (1 << LINE_BITS) - 1
  /** Mask to decode the column number */
  final val COLUMN_MASK = (1 << COLUMN_BITS) - 1

  final def line(pos: Int): Int = (pos >> COLUMN_BITS) & LINE_MASK

  final def column(pos: Int): Int = pos & COLUMN_MASK
}

/**
 * Loads XML using the Scala ConstructingParser for XML.
 *
 * Necessary as this xml loading technique handles the <![CDATA[...]]>
 * properly, creating PCData nodes for the contents of these, and not otherwise
 * messing with the contents.
 *
 * This code is effectively our fork of the Scala ConstructingParser. This
 * works around some bugs in it.
 *
 * Xerces, unfortunately, messes with the contents of these CDATA regions,
 * normalizes whitespace inside them, and generally makes it impossible to do things
 * in XML that depend on line-structure of element content to being preserved.
 *
 * We have places where line structure matters. Specifically regular expressions
 * have a free-form syntax with comments that extend to end-of-line. If we always
 * wrap these with CDATA, and use this loader, not Xerces, then these will be
 * preserved properly.
 *
 * Also, enhanced so that when addPositionAttributes is true, it will capture
 * file/line/column info for every element and add it
 * as attributes onto each XML element.
 *
 * The way the constructing loader (aka ConstructingParser (for XML))
 * gets positions is different. It is given just an offset into the document file/stream,
 * and it therefore must synthesize line number/col number info itself.
 *
 * This primary constructor is package private as the normalizeCRLFtoLF feature
 * is only for test/exploratory usage, or if a future need arises to preserve the
 * non-normalizaing behavior.
 *
 * @param uri URI for the XML to be loaded.
 * @param errorHandler Called back on load errors.
 * @param addPositionAttributes Use true if you want dafint:file,
 *                              dafint:col, and dafint:line attributes.
 *                              Defaults to false.
 * @param normalizeCRLFtoLF Use true to emulate the scala XML load
 *                          behavior of normalizing CRLF to LF, and solitary CR to LF.
 *                          Defaults to true. Should only be changed in special circumstances
 *                          as not normalizing CRLFs is non-standard for XML.
 *
 */
class DaffodilConstructingLoader private[xml] (uri: URI,
  errorHandler: org.xml.sax.ErrorHandler,
  addPositionAttributes: Boolean,
  normalizeCRLFtoLF: Boolean)
  extends ConstructingParser({
    // Note: we must open the XML carefully since it might be in some non
    // default encoding (we have tests that have UTF-16 for example)

    // must be buffered to support mark(), needed by heuristics
    val is = new BufferedInputStream(uri.toURL.openStream())
    val enc = EncodingHeuristics.readEncodingFromStream(is)
    Source.fromInputStream(is, enc)
  }, true) {

  /**
   * Public constructor insists on normalizingCRLFtoLF behavior.
   */
  def this (uri: URI,
    errorHandler: org.xml.sax.ErrorHandler,
    addPositionAttributes: Boolean = false) =
    this(uri, errorHandler, addPositionAttributes, normalizeCRLFtoLF = true)

  /**
   * Ensures that DOCTYPES aka DTDs, if encountered, are rejected.
   *
   * Coverage is off, because this should never be hit, because
   * the loader always has loaded the data with xerces prior to
   * this loader (for validation purposes), and that will have caught
   * the doctype being in the XML.
   *
   * However, under code maintenance, suppose someone turned that off
   * or made that pass optional (for performance reasons perhaps). Then this
   * provides a last-gasp attempt to protect from DOCTYPE-related
   * insecurity.
   */
  // $COVERAGE-OFF$
  override def parseDTD(): Unit = {
    val e = makeSAXParseException(pos, "DOCTYPE is disallowed.")
    throw e
  }
  // $COVERAGE-ON$

  // This one line is a bit of a hack to get consistent line numbers. The
  // scala-xml libary reads XML from a scala.io.Source which maintains private
  // line/col information about where in the Source we are reading from (i.e.
  // scala.io.Source.pos). The problem is that when CDATA or a processing
  // instruction is encountered, the library switches to a custom
  // "WithLookAhead" scala.io.Source that buffers the original Source. This
  // lookahead Source allows it to peek ahead a few characters, which is used
  // to find the end of CDATA and processing instructions. The problem is that
  // when it switches to this new Source, we lose position information since
  // that information is private to each Source. This causes line information
  // to reset to zero when the first CDATA or processing instruction is found.
  // And there is no good way to copy position information from one source to
  // another. So, what we can do is call this lookahead() function before any
  // XML is parsed. This causes the ConstructingLoader to immediately switch to
  // the buffering source. There may be some slight overhead for buffering, but
  // at last our line numbers are correct.
  lookahead()


  private def makeSAXParseException(pos: Int, msg: String) = {
    val line = Position.line(pos)
    val col = Position.column(pos)
    val exc = new org.xml.sax.SAXParseException(msg, null, uri.toString, line, col)
    exc
  }

  override def reportSyntaxError(pos: Int, msg: String): Unit = {
    val exc = makeSAXParseException(pos, msg)
    errorHandler.error(exc)
  }

  /*
   * Callback method invoked by MarkupParser after parsing an element, between
   * the elemStart and elemEnd callbacks. This adds daffodil file/line/column
   * information as attributes to the existing input attrs, modifying the scope
   * if necessary, then creates an element using the super def elem function.
   *
   *  @param pos      the position in the source file
   *  @param pre      the prefix
   *  @param local    the local name
   *  @param attrs    the attributes (metadata)
   *  @param scope    the namespace binding scope
   *  @param empty    `true` if the element was previously empty; `false` otherwise.
   *  @param args     the children of this element
   */
  override def elem(
    pos: Int,
    pre: String,
    local: String,
    attrs: MetaData,
    scope: NamespaceBinding,
    empty: Boolean,
    nodes: NodeSeq): NodeSeq = {

    val nsURI = NS(scope.getURI(pre))
    val isFileRootNode = (local.equalsIgnoreCase("schema") && nsURI == XMLUtils.XSD_NAMESPACE) ||
      (local.equalsIgnoreCase("testSuite") && nsURI == XMLUtils.TDML_NAMESPACE)
    val alreadyHasLineCol = attrs.exists {
      case PrefixedAttribute(XMLUtils.INT_PREFIX, attr, _, _) => {
        attr.equalsIgnoreCase(XMLUtils.COLUMN_ATTRIBUTE_NAME) ||
          attr.equalsIgnoreCase(XMLUtils.LINE_ATTRIBUTE_NAME)
      }
      case _ => false
    }

    val newAttrs: MetaData = {
      if (addPositionAttributes && !alreadyHasLineCol) {
        val withFile: MetaData =
          if (isFileRootNode) {
            new PrefixedAttribute(XMLUtils.INT_PREFIX, XMLUtils.FILE_ATTRIBUTE_NAME, uri.toString, attrs)
          } else {
            attrs
          }
        val withCol: MetaData = new PrefixedAttribute(XMLUtils.INT_PREFIX, XMLUtils.COLUMN_ATTRIBUTE_NAME, Position.column(pos).toString, withFile)
        val withLine: MetaData = new PrefixedAttribute(XMLUtils.INT_PREFIX, XMLUtils.LINE_ATTRIBUTE_NAME, Position.line(pos).toString, withCol)
        withLine
      } else {
        attrs
      }
    }

    // add the dafint prefix if it doesn't already exist
    val intPrefix = scope.getPrefix(XMLUtils.INT_NS)
    val newScope = if (addPositionAttributes && intPrefix == null) {
      NamespaceBinding(XMLUtils.INT_PREFIX, XMLUtils.INT_NS, scope)
    } else {
      Assert.usage(intPrefix == null || intPrefix == XMLUtils.INT_PREFIX) // can't deal with some other binding for dafint
      scope
    }

    super.elem(pos, pre, local, newAttrs, newScope, empty, nodes)
  }

  /**
   * To emulate the behavior of Xerces loader (standard scala loader)
   * we have to normalize CRLF to LF, and solitary CR to LF.
   *
   * This is optional controlled by a constructor parameter.
   */
  override def text(pos: Int, txt: String): Text = {
    val newText:String = {
      if (normalizeCRLFtoLF && txt.contains("\r")) {
        txt.
          replaceAll("\r\n", "\n").
          replaceAll("\r", "\n")
      } else {
        txt
      }
    }
    //
    // On MS-Windows the TDML Runner previously would load XML
    // files and due to git autoCRLF=true, they would
    // have CRLFs in them. The loader the TDML Runner WAS
    // using (not any more) was preserving these CRLFs
    // in the XML infoset data, and so tests could come
    // to depend on this and be non-portable between
    // unix (LF only) and windows (CRLF only).
    //
    // Furthermore, the TDML file itself used to be loaded with this
    // CRLF-preserving loader.
    //
    // The TDML Runner now always normalizes CRLF or
    // isolated CR to LF like regular XML loaders do,
    // for both the TDML file itself, and any files it
    // loads. So this is no longer an issue.
    //
    super.text(pos, newText)
  }

  /**
   * We override this to force the ConstrutingParser to process CDATA regions
   * specially with an override-able method named cdata.
   *
   * Strangely, if you look at the implementation of this in the MarkupParser
   * trait, it calls the handler for text, but then it ignores the result of that
   * and constructs a PCDATA node from the original text.
   *
   * It's possible this is a bug fix.
   */
  override def xCharData: NodeSeq = {
    xToken("[CDATA[")
    def mkResult(pos: Int, s: String): NodeSeq = {
      val s1 = cdata(pos, s).text
      PCData(s1)
    }
    xTakeUntil(mkResult, () => pos, "]]>")
  }

  /**
   * Same CRLF/CR => LF processing as text gets.
   */
  def cdata(pos: Int, s: String): NodeSeq = {
    text(pos, s)
  }

  /**
   * Drops comments
   */
  override def comment(pos: Int, s: String): Comment = {
    // returning null drops comments
    null
  }

  /**
   * Drops processing instructions
   */
  override def procInstr(pos: Int, target: String, txt: String) = {
    // returning null drops processing instructions
    null
  }

  private def parseXMLPrologAttributes(m: MetaData): (Option[String], Option[String], Option[Boolean]) = {

    var info_ver: Option[String] = None
    var info_enc: Option[String] = None
    var info_stdl: Option[Boolean] = None

    var n = 0
    m("version") match {
      case null =>
      case Text("1.0") =>
        info_ver = Some("1.0"); n += 1
      case _ => reportSyntaxError("cannot deal with versions != 1.0")
    }

    m("encoding") match {
      case null =>
      case Text(enc) =>
        if (!isValidIANAEncoding(enc))
          reportSyntaxError("\"" + enc + "\" is not a valid encoding")
        else {
          info_enc = Some(enc)
          n += 1
        }
    }

    m("standalone") match {
      case null =>
      case Text("yes") =>
        info_stdl = Some(true); n += 1
      case Text("no") =>
        info_stdl = Some(false); n += 1
      case _ => reportSyntaxError("either 'yes' or 'no' expected")
    }

    if (m.length - n != 0) {
      reportSyntaxError(
        "only 'version', 'encoding', and 'standalone' attributes are expected in xml prolog. Found: " + m)
    }

    (info_ver, info_enc, info_stdl)
  }

  /**
   * Override of document to make it tolerant of the start of the file
   * being whitespace instead of a "<" character
   *
   * This does not handle DOCTYPEs (aka DTDs) at all. Hence, is not
   * a true replacement (bug fix) on the original ConstructingParser method
   * that it overrides.
   */
  override def document(): Document = {
    doc = new Document()
    this.dtd = null
    var children: NodeSeq = null

    if ('<' == ch) {
      nextch()
      if ('?' == ch) {
        nextch()
        // It's probably an XML prolog, but
        // there are cases where there is no XML Prolog, but a starting
        // PI of <?xml-model href="...."?>
        // So we have to recognize as a general PI, then look and see if
        // it is a prolog.
        val name = xName
        xSpace()
        val (md, scp) = xAttributes(TopScope)
        if (scp != TopScope)
          reportSyntaxError("no xmlns definitions allowed.")
        xToken('?')
        xToken('>')
        if (name == "xml") {
          val info_prolog = parseXMLPrologAttributes(md)
          doc.version = info_prolog._1
          doc.encoding = info_prolog._2
          doc.standAlone = info_prolog._3
        } else {
          // not an xml prolog. It's some other PI
          // do nothing. We're just skipping those PIs
        }
        children = content(TopScope)
      } else {
        val ts = new NodeBuffer()
        content1(TopScope, ts) // the 1 suffix means "without the first < character"
        ts &+ content(TopScope)
        children = NodeSeq.fromSeq(ts)
      }
    } else {
      children = content(TopScope)
    }

    var isErr = false
    var elemCount = 0
    var theNode: Node = null
    children.foreach { c =>
      c match {
        case _: ProcInstr => // skip
        case _: Comment => // skip
        // $COVERAGE-OFF$ // constructing parser never creates these - probably due to a bug
        case _: EntityRef => {
          reportSyntaxError("no entity references allowed here")
          isErr = true
        }
        // $COVERAGE-ON$
        case s: SpecialNode => {
          val txt = s.toString.trim()
          if (txt.length > 0) {
            reportSyntaxError("non-empty text nodes not allowed: '" + txt + "'.")
            isErr = true
          }
        }
        case m: Elem =>
          elemCount += 1
          theNode = m
      }
    }
    if (1 != elemCount) {
      reportSyntaxError("document must contain exactly one element")
      isErr = true
    }

    if (!isErr) {
      doc.children = children
      doc.docElem = theNode
      doc
    } else {
      null
    }
  }

  def load(): Node = {
    val res =
      try {
        this.initialize
        val doc = this.document()
        if (doc == null) null
        else doc.docElem
      } catch {
        case e: Exception => {
          val exc = makeSAXParseException(curInput.pos, e.toString)
          errorHandler.fatalError(exc) // good place for a breakpoint
          null
        }
      }
    res
  }
}
