#!/usr/bin/env php
<?php
/*
    This file is part of Erebot.

    Erebot is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    Erebot is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Erebot.  If not, see <http://www.gnu.org/licenses/>.
*/

/**
 * This class converts wordlists from the old (.txt-based)
 * format to a new SQLite-based format.
 *
 * You may call this script directly from the commandline
 * to convert one or more .txt files to their .sqlite
 * counterparts.
 */
class Erebot_Module_Wordlists_Converter
{
    /// PDO database handle.
    protected $_db;

    /// Path to the directory containing the wordlists.
    protected $_path;

    /// Name of the file this wordlists is stored in.
    protected $_file;


    /**
     * Constructs a new wordlist.
     *
     * \param string $path
     *      Path to the directory containing the lists
     *      of words to convert. The user running this
     *      script must have write permissions in that
     *      directory.
     *
     * \param string $file
     *      Name of the file containing the wordlist
     *      to convert, without any extension.
     */
    public function __construct($path, $file)
    {
        $this->_path = $path;
        $this->_file = $file;

        if (!file_exists($path . DIRECTORY_SEPARATOR . $file . '.txt'))
            throw new Exception('File not found or insufficient permissions');
        $this->_parseFile($path . DIRECTORY_SEPARATOR . $file);
    }

    /**
     * Parses the contents of a file.
     *
     * \param string $file
     *      Path to the file where the wordlist is stored,
     *      without the ".txt" extension.
     *
     * \note
     *      You may add comments (lines beginning with a #)
     *      in the file. Comments at the beginning of the file
     *      play a special role. They are used to provide extra
     *      information about the content of the file.
     *      These special comments are of the form "# key: value"
     *      where "key" can be one of: "name", "version",
     *      "description", "author", "locale", "license", "url"
     *      or "keyword". The "author" and "keyword" keys may
     *      be used more than once.
     *      It does not matter in what order these special
     *      comments appear in the file, as long as they appear
     *      at the very beginning of the file (except that there
     *      MAY be a Byte Order Mark before them).
     *
     * \note
     *      This method automatically converts any word
     *      contained in the wordlist to UTF-8.
     *      The encoding for the input file may be specified
     *      using a Byte Order Mark (BOM).
     *      By default, this method assumes the file is encoded
     *      in UTF-8.
     *
     * \note
     *      You MUST specify the locale the wordlist is intended
     *      for (eg. "en-US") in the file. You may do so by putting
     *      a comment at the beginning of the file, eg.:
     *          # locale: en-US
     *      This indication is used to sort words in the wordlist in
     *      alphabetical order, using the proper rules for that locale.
     */
    protected function _parseFile($file)
    {
        $dbFile     = $file . '.sqlite';
        $file      .= '.txt';

        fprintf(STDOUT, "Converting %s to %s ...\n", $file, $dbFile);

        /// Metadata associated with this list.
        $metadata = array(
            'name'          => NULL,
            'version'       => NULL,
            'description'   => NULL,
            'author'        => array(),
            'locale'        => NULL,
            'license'       => NULL,
            'url'           => NULL,
            'keyword'       => array(),
        );

        $content = file_get_contents($file);
        if ($content === FALSE)
            throw new Erebot_Module_Wordlists_UnreadableFileException($file);

        $encoding = self::_handleBOM($content);
        if ($encoding !== NULL)
            $content = $this->_toUTF8($content, $encoding);

        // Replace "\r\n" and "\r" sequences with "\n"
        // (for compatibility with Windows and old Mac).
        $content = str_replace(array("\r\n", "\r"), "\n", $content);
        $content = explode("\n", $content);
        while (count($content) && substr($content[0], 0, 1) == "#") {
            // Remove the line and strip the leading "#".
            $line   = (string) substr(array_shift($content), 1);
            $pos    = strpos($line, ':');

            if ($pos === FALSE)
                break;

            $key    = strtolower(trim((string) substr($line, 0, $pos)));
            $value  = ltrim((string) substr($line, $pos + 1));

            if (!array_key_exists($key, $metadata))
                continue;
            if (is_array($metadata[$key]))
                $metadata[$key][] = $value;
            else
                $metadata[$key] = $value;
        }

        if (!isset($metadata['locale']))
            throw new Exception('Missing locale');

        $collator = new Collator(
            str_replace('-', '_', $metadata['locale'])
        );
        // -127 = U_USING_DEFAULT_WARNING
        // -128 = U_USING_FALLBACK_WARNING.
        //    0 = U_ZERO_ERROR (no error).
        if (!in_array(intl_get_error_code(), array(-127, -128, 0))) {
            throw new Erebot_InvalidValueException(
                "Invalid locale (".$metadata['locale']."): ".
                intl_get_error_message()
            );
        }
        $collator->setStrength(Collator::PRIMARY);

        $this->_createDb($dbFile);
        $this->_writeMetadata($metadata);
        $this->_writeWords($collator, $content);
    }

    /**
     * Create a new SQLite database for
     * a list of words.
     *
     * \param string $file
     *      Full path to the sqlite database
     *      to create.
     *
     * \warning
     *      If the file already existed, it is overwritten
     *      with new data without any warning.
     */
    protected function _createDb($file)
    {
        @unlink($file);

        $this->_db = new PDO("sqlite:$file");
        $this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);

        $res = $this->_db->exec('DROP TABLE IF EXISTS metadata');
        assert('$res !== FALSE');
        $res = $this->_db->exec('DROP TABLE IF EXISTS words');
        assert('$res !== FALSE');

        $res = $this->_db->exec(
            'CREATE TABLE metadata ('.
                'id INTEGER PRIMARY KEY, '.
                'type VARCHAR(32) NOT NULL, '.
                'value TEXT NOT NULL'.
            ')'
        );
        assert('$res !== FALSE');

        $res = $this->_db->exec(
            'CREATE TABLE words ('.
                'sortkey BLOB NOT NULL PRIMARY KEY, '.
                'value TEXT NOT NULL'.
            ')'
        );
        assert('$res !== FALSE');
    }

    /**
     * Write the metadata for the current wordlist.
     *
     * \param array $metadata
     *      An array containing the metadata to write.
     */
    protected function _writeMetadata($metadata)
    {
        $insert = $this->_db->prepare(
            'INSERT INTO metadata(type, value) '.
            'VALUES(:type, :value)'
        );
        foreach ($metadata as $type => $values) {
            $values = (array) $values;
            foreach ($values as $value) {
                $res = $insert->execute(array(':type' => $type, ':value' => $value));
                assert('$res === TRUE');
                $res = $insert->closeCursor();
                assert('$res === TRUE');
            }
        }
    }

    /**
     * Write the words into the wordlist.
     *
     * \param Collator $collator
     *      Collator to use to get each
     *      word's sort key.
     *
     * \param array $words
     *      A list with the words to add
     *      to the wordlist.
     *
     * \note
     *      The list of words is modified
     *      in-place.
     */
    protected function _writeWords(Collator $collator, &$words)
    {
        $mbstring = function_exists('mb_strtolower');
        $tolower = 'strtolower';
        if ($mbstring) {
            $previousMB = mb_internal_encoding();
            mb_internal_encoding('UTF-8');
            $tolower = 'mb_strtolower';
        }

        // Tweaking these improves performances big time!
        $res = $this->_db->exec('PRAGMA synchronous=OFF');
        $res = $this->_db->exec('PRAGMA journal_mode=MEMORY');
        $res = $this->_db->exec('PRAGMA temp_store=MEMORY');

        $res = $this->_db->beginTransaction();
        assert('$res === TRUE');

        $insert = $this->_db->prepare(
            'INSERT OR IGNORE INTO words(sortkey, value) '.
            'VALUES(:key, :value)'
        );
        // Collator::getSortKey() tends to generate warnings.
        $previousER = error_reporting(error_reporting() & ~E_WARNING);
        $n = count($words);
        $this->_updateProgress(0, $n);
        for ($i = 0; $i < $n; $i++) {
            $word = array_pop($words);
            if (substr($word, 0, 1) == '#')
                continue;
            $word = $tolower(trim($word));
            if ($word == '')
                continue;

            $key = $collator->getSortKey($word);
            /* In intl > 3.0.0a1, the key now ends with a trailing NUL byte.
             * We remove it for backward-compatibility with older releases. */
            if (substr($key, -1) === "\0")
                $key = substr($key, 0, -1);

            $res = $insert->execute(
                array(
                    ':key'      => $key,
                    ':value'    => $word,
                )
            );
            assert('$res === TRUE');
            $res = $insert->closeCursor();
            assert('$res === TRUE');

            if ($i % 10 == 0)
                $this->_updateProgress($i + 1, $n);
        }
        $this->_updateProgress($n, $n);
        fprintf(STDOUT, "\nDone.\n");

        error_reporting($previousER);
        if ($mbstring)
            mb_internal_encoding($previousMB);

        $res = $this->_db->commit();
        assert('$res === TRUE');
    }

    /**
     * Display an updated bar indicating the progress
     * made on the conversion.
     *
     * \param int $done
     *      Number of lines already converted.
     *
     * \param int $total
     *      Total number of lines to convert.
     *
     * \note
     *      The progress bar is sent directly to STDOUT.
     *      When the conversion is finished (ie. $done == $total),
     *      you should send a single "\n" character to STDOUT
     *      before sending the next status line, this method
     *      WILL NOT do it for you.
     */
    static protected function _updateProgress($done, $total)
    {
        $length = 100;
        $percent = (int) ($done * $length / $total);
        $left = $length - $percent;
        $res = str_pad('>', $percent + 1, '=', STR_PAD_LEFT);
        $res = substr($res, 0, $length);
        $res = str_pad($res, $length, ' ');
        fprintf(
            STDOUT,
            "[%s] %6.2f%% (%d/%d)\r",
            $res,
            $done * 100 / $total,
            $done,
            $total
        );
    }

    /**
     * Returns the path to the directory containing
     * all wordlists.
     *
     * \retval string
     *      Path to the directory containing wordlists.
     */
    public function getPath()
    {
        return $this->_path;
    }

    /**
     * Returns the name of the file in which the list
     * is stored.
     *
     * \retval string
     *      Name of the file containing the list.
     */
    public function getFile()
    {
        return $this->_file;
    }

    /**
     * Given some text, returns the name of its encoding,
     * if one could be deduced from a BOM (Byte Order Mark).
     *
     * \param string $text
     *      The text to analyze to try to deduce its encoding
     *      based on a Byte Order Mark.
     *
     * \retval mixed
     *      Either the name of an encoding (eg. "UTF-16BE")
     *      if one could be deduced from a Byte Order Mark
     *      present in the file; NULL otherwise.
     */
    static protected function _handleBOM(&$text)
    {
        if (substr($text, 0, 3) == "\xEF\xBB\xBF") {
            $text = (string) substr($text, 3);
            return 'UTF-8';
        }

        $four = substr($text, 0, 4);
        if ($four == "\x00\x00\xFF\xFE") {
            $text = (string) substr($text, 4);
            return "UTF-32BE";
        }

        if ($four == "\xFF\xFE\x00\x00") {
            $text = (string) substr($text, 4);
            return "UTF-32LE";
        }

        $two = substr($text, 0, 2);
        if ($two == "\xFE\xFF") {
            $text = (string) substr($text, 2);
            return "UTF-16BE";
        }

        if ($two == "\xFF\xFE") {
            $text = (string) substr($text, 2);
            return "UTF-16LE";
        }

        // This is some other 8-byte based character set.
        return NULL;
    }

    /**
     * Encodes some text in UTF-8.
     *
     * \param string $text
     *      Text to encode in UTF-8.
     *
     * \param string $from
     *      The text's current encoding.
     *
     * \retval string
     *      The text, encoded in UTF-8.
     *
     * \note
     *      This method has been duplicated from Erebot_Utils
     *      as we need a way to convert some random text to UTF-8
     *      without depending on Erebot's inner workings.
     *
     * \throw Erebot_NotImplementedException
     *      No mechanism was found that could be used
     *      to convert the given text to UTF-8.
     */
    protected function _toUTF8($text, $from)
    {
        if ($from == 'UTF-8')
            return $text;

        if ($from == 'ISO-8859-1' && function_exists('utf8_encode'))
            return utf8_encode($text);

        if (function_exists('iconv'))
            return iconv($from, 'UTF-8//TRANSLIT', $text);

        if (function_exists('recode'))
            return recode($from.'..utf-8', $text);

        if (function_exists('mb_convert_encoding')) {
            $subst  = mb_substitute_character('none');
            $text   = mb_convert_encoding($text, 'UTF-8', $from);
            mb_substitute_character($subst);
            return $text;
        }

        if (function_exists('html_entity_decode')) {
            return html_entity_decode(
                htmlentities($text, ENT_QUOTES, $from),
                ENT_QUOTES,
                'UTF-8'
            );
        }

        throw new Erebot_NotImplementedException(
            "No way to do UTF-8 conversions"
        );
    }
}

// Not running from CLI.
if (!isset($_SERVER['argv']))
    die(2);

// Invalid usage.
if (count($_SERVER['argv']) == 1) {
    fprintf(
        STDERR,
        "Usage: %s /path/to/wordlist.txt ...\n",
        $_SERVER['argv'][0]
    );
    die(2);
}

// Convert each .txt file passed as an argument
// to its equivalent .sqlite file.
for ($i = 1, $n = count($_SERVER['argv']); $i < $n; $i++) {
    $path = dirname($_SERVER['argv'][$i]);
    $file = basename($_SERVER['argv'][$i], '.txt');
    try {
        new Erebot_Module_Wordlists_Converter($path, $file);
    }
    catch (Exception $e) {
        fprintf(
            STDERR,
            "Exception: %s\n%s\n",
            $e->getMessage(),
            $e->getTraceAsString()
        );
        die(1);
    }
}
die(0);

