    001/*
    002 * Licensed to the Apache Software Foundation (ASF) under one or more
    003 * contributor license agreements.  See the NOTICE file distributed with
    004 * this work for additional information regarding copyright ownership.
    005 * The ASF licenses this file to You under the Apache License, Version 2.0
    006 * (the "License"); you may not use this file except in compliance with
    007 * the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 * Unless required by applicable law or agreed to in writing, software
    012 * distributed under the License is distributed on an "AS IS" BASIS,
    013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 * See the License for the specific language governing permissions and
    015 * limitations under the License.
    016 */
    017
    018package org.apache.commons.codec.language.bm;
    019
    020import java.util.ArrayList;
    021import java.util.Arrays;
    022import java.util.Collections;
    023import java.util.EnumMap;
    024import java.util.HashSet;
    025import java.util.LinkedHashSet;
    026import java.util.List;
    027import java.util.Locale;
    028import java.util.Map;
    029import java.util.Objects;
    030import java.util.Set;
    031import java.util.TreeMap;
    032import java.util.stream.Collectors;
    033
    034import org.apache.commons.codec.language.bm.Languages.LanguageSet;
    035import org.apache.commons.codec.language.bm.Rule.Phoneme;
    036
    037/**
    038 * Converts words into potential phonetic representations.
    039 * <p>
    040 * This is a two-stage process. Firstly, the word is converted into a phonetic representation that takes
    041 * into account the likely source language. Next, this phonetic representation is converted into a
    042 * pan-European 'average' representation, allowing comparison between different versions of essentially
    043 * the same word from different languages.
    044 * </p>
    045 * <p>
    046 * This class is intentionally immutable and thread-safe.
    047 * If you wish to alter the settings for a PhoneticEngine, you
    048 * must make a new one with the updated settings.
    049 * </p>
    050 * <p>
    051 * Ported from phoneticengine.php
    052 * </p>
    053 *
    054 * @since 1.6
    055 */
    056public class PhoneticEngine {
    057
    058    private static final int DEFAULT_MAX_PHONEMES = 20;
    059
    060    private static final Map<NameType, Set<String>> NAME_PREFIXES = new EnumMap<>(NameType.class);
    061
    062    private final Lang lang;
    063
    064    private final NameType nameType;
    065
    066    private final RuleType ruleType;
    067
    068    private final boolean concat;
    069
    070    private final int maxPhonemes;
    071
    072    static {
    073        NAME_PREFIXES.put(NameType.ASHKENAZI,
    074                Collections.unmodifiableSet(
    075                        new HashSet<>(Arrays.asList("bar", "ben", "da", "de", "van", "von"))));
    076        NAME_PREFIXES.put(NameType.SEPHARDIC,
    077                Collections.unmodifiableSet(
    078                        new HashSet<>(Arrays.asList("al", "el", "da", "dal", "de", "del", "dela", "de la",
    079                                                          "della", "des", "di", "do", "dos", "du", "van", "von"))));
    080        NAME_PREFIXES.put(NameType.GENERIC,
    081                Collections.unmodifiableSet(
    082                        new HashSet<>(Arrays.asList("da", "dal", "de", "del", "dela", "de la", "della",
    083                                                          "des", "di", "do", "dos", "du", "van", "von"))));
    084    }
    085
    086    /**
    087     * Utility for manipulating a set of phonemes as they are being built up. Not intended for use outside
    088     * this package, and probably not outside the {@link PhoneticEngine} class.
    089     *
    090     * @since 1.6
    091     */
    092    static final class PhonemeBuilder {
    093
    094        private final Set<Rule.Phoneme> phonemes;
    095
    096        /**
    097         * An empty builder where all phonemes must come from some set of languages. This will contain a single
    098         * phoneme of zero characters. This can then be appended to. This should be the only way to create a new
    099         * phoneme from scratch.
    100         *
    101         * @param languages the set of languages
    102         * @return  a new, empty phoneme builder
    103         */
    104        public static PhonemeBuilder empty(final Languages.LanguageSet languages) {
    105            return new PhonemeBuilder(new Rule.Phoneme("", languages));
    106        }
    107
    108        private PhonemeBuilder(final Rule.Phoneme phoneme) {
    109            this.phonemes = new LinkedHashSet<>();
    110            this.phonemes.add(phoneme);
    111        }
    112
    113        private PhonemeBuilder(final Set<Rule.Phoneme> phonemes) {
    114            this.phonemes = phonemes;
    115        }
    116
    117        /**
    118         * Creates a new phoneme builder containing all phonemes in this one extended by {@code str}.
    119         *
    120         * @param str   the characters to append to the phonemes
    121         */
    122        public void append(final CharSequence str) {
    123            phonemes.forEach(ph -> ph.append(str));
    124        }
    125
    126        /**
    127         * Applies the given phoneme expression to all phonemes in this phoneme builder.
    128         * <p>
    129         * This will lengthen phonemes that have compatible language sets to the expression, and drop those that are
    130         * incompatible.
    131         * </p>
    132         *
    133         * @param phonemeExpr   the expression to apply
    134         * @param maxPhonemes   the maximum number of phonemes to build up
    135         */
    136        public void apply(final Rule.PhonemeExpr phonemeExpr, final int maxPhonemes) {
    137            final Set<Rule.Phoneme> newPhonemes = new LinkedHashSet<>(maxPhonemes);
    138
    139            EXPR: for (final Rule.Phoneme left : this.phonemes) {
    140                for (final Rule.Phoneme right : phonemeExpr.getPhonemes()) {
    141                    final LanguageSet languages = left.getLanguages().restrictTo(right.getLanguages());
    142                    if (!languages.isEmpty()) {
    143                        final Rule.Phoneme join = new Phoneme(left, right, languages);
    144                        if (newPhonemes.size() < maxPhonemes) {
    145                            newPhonemes.add(join);
    146                            if (newPhonemes.size() >= maxPhonemes) {
    147                                break EXPR;
    148                            }
    149                        }
    150                    }
    151                }
    152            }
    153
    154            this.phonemes.clear();
    155            this.phonemes.addAll(newPhonemes);
    156        }
    157
    158        /**
    159         * Gets underlying phoneme set. Please don't mutate.
    160         *
    161         * @return  the phoneme set
    162         */
    163        public Set<Rule.Phoneme> getPhonemes() {
    164            return this.phonemes;
    165        }
    166
    167        /**
    168         * Stringifies the phoneme set. This produces a single string of the strings of each phoneme,
    169         * joined with a pipe. This is explicitly provided in place of toString as it is a potentially
    170         * expensive operation, which should be avoided when debugging.
    171         *
    172         * @return  the stringified phoneme set
    173         */
    174        public String makeString() {
    175            return phonemes.stream().map(Rule.Phoneme::getPhonemeText).collect(Collectors.joining("|"));
    176        }
    177    }
    178
    179    /**
    180     * A function closure capturing the application of a list of rules to an input sequence at a particular offset.
    181     * After invocation, the values {@code i} and {@code found} are updated. {@code i} points to the
    182     * index of the next char in {@code input} that must be processed next (the input up to that index having been
    183     * processed already), and {@code found} indicates if a matching rule was found or not. In the case where a
    184     * matching rule was found, {@code phonemeBuilder} is replaced with a new builder containing the phonemes
    185     * updated by the matching rule.
    186     *
    187     * Although this class is not thread-safe (it has mutable unprotected fields), it is not shared between threads
    188     * as it is constructed as needed by the calling methods.
    189     * @since 1.6
    190     */
    191    private static final class RulesApplication {
    192        private final Map<String, List<Rule>> finalRules;
    193        private final CharSequence input;
    194
    195        private final PhonemeBuilder phonemeBuilder;
    196        private int i;
    197        private final int maxPhonemes;
    198        private boolean found;
    199
    200        public RulesApplication(final Map<String, List<Rule>> finalRules, final CharSequence input,
    201                                final PhonemeBuilder phonemeBuilder, final int i, final int maxPhonemes) {
    202            Objects.requireNonNull(finalRules, "finalRules");
    203            this.finalRules = finalRules;
    204            this.phonemeBuilder = phonemeBuilder;
    205            this.input = input;
    206            this.i = i;
    207            this.maxPhonemes = maxPhonemes;
    208        }
    209
    210        public int getI() {
    211            return this.i;
    212        }
    213
    214        public PhonemeBuilder getPhonemeBuilder() {
    215            return this.phonemeBuilder;
    216        }
    217
    218        /**
    219         * Invokes the rules. Loops over the rules list, stopping at the first one that has a matching context
    220         * and pattern. Then applies this rule to the phoneme builder to produce updated phonemes. If there was no
    221         * match, {@code i} is advanced one and the character is silently dropped from the phonetic spelling.
    222         *
    223         * @return {@code this}
    224         */
    225        public RulesApplication invoke() {
    226            this.found = false;
    227            int patternLength = 1;
    228            final List<Rule> rules = this.finalRules.get(input.subSequence(i, i+patternLength));
    229            if (rules != null) {
    230                for (final Rule rule : rules) {
    231                    final String pattern = rule.getPattern();
    232                    patternLength = pattern.length();
    233                    if (rule.patternAndContextMatches(this.input, this.i)) {
    234                        this.phonemeBuilder.apply(rule.getPhoneme(), maxPhonemes);
    235                        this.found = true;
    236                        break;
    237                    }
    238                }
    239            }
    240
    241            if (!this.found) {
    242                patternLength = 1;
    243            }
    244
    245            this.i += patternLength;
    246            return this;
    247        }
    248
    249        public boolean isFound() {
    250            return this.found;
    251        }
    252    }
    253
    254    /**
    255     * Joins some strings with an internal separator.
    256     *
    257     * @param strings   Strings to join
    258     * @param sep       String to separate them with
    259     * @return a single String consisting of each element of {@code strings} interleaved by {@code sep}
    260     */
    261    private static String join(final List<String> strings, final String sep) {
    262        return strings.stream().collect(Collectors.joining(sep));
    263    }
    264
    265    /**
    266     * Generates a new, fully-configured phonetic engine.
    267     *
    268     * @param nameType
    269     *            the type of names it will use
    270     * @param ruleType
    271     *            the type of rules it will apply
    272     * @param concat
    273     *            if it will concatenate multiple encodings
    274     */
    275    public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concat) {
    276        this(nameType, ruleType, concat, DEFAULT_MAX_PHONEMES);
    277    }
    278
    279    /**
    280     * Generates a new, fully-configured phonetic engine.
    281     *
    282     * @param nameType
    283     *            the type of names it will use
    284     * @param ruleType
    285     *            the type of rules it will apply
    286     * @param concat
    287     *            if it will concatenate multiple encodings
    288     * @param maxPhonemes
    289     *            the maximum number of phonemes that will be handled
    290     * @since 1.7
    291     */
    292    public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concat,
    293                          final int maxPhonemes) {
    294        if (ruleType == RuleType.RULES) {
    295            throw new IllegalArgumentException("ruleType must not be " + RuleType.RULES);
    296        }
    297        this.nameType = nameType;
    298        this.ruleType = ruleType;
    299        this.concat = concat;
    300        this.lang = Lang.instance(nameType);
    301        this.maxPhonemes = maxPhonemes;
    302    }
    303
    304    /**
    305     * Applies the final rules to convert from a language-specific phonetic representation to a
    306     * language-independent representation.
    307     *
    308     * @param phonemeBuilder the current phonemes
    309     * @param finalRules the final rules to apply
    310     * @return the resulting phonemes
    311     */
    312    private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder,
    313            final Map<String, List<Rule>> finalRules) {
    314        Objects.requireNonNull(finalRules, "finalRules");
    315        if (finalRules.isEmpty()) {
    316            return phonemeBuilder;
    317        }
    318
    319        final Map<Rule.Phoneme, Rule.Phoneme> phonemes = new TreeMap<>(Rule.Phoneme.COMPARATOR);
    320
    321        phonemeBuilder.getPhonemes().forEach(phoneme -> {
    322            PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages());
    323            final String phonemeText = phoneme.getPhonemeText().toString();
    324
    325            for (int i = 0; i < phonemeText.length();) {
    326                final RulesApplication rulesApplication = new RulesApplication(finalRules, phonemeText, subBuilder, i, maxPhonemes).invoke();
    327                final boolean found = rulesApplication.isFound();
    328                subBuilder = rulesApplication.getPhonemeBuilder();
    329
    330                if (!found) {
    331                    // not found, appending as-is
    332                    subBuilder.append(phonemeText.subSequence(i, i + 1));
    333                }
    334
    335                i = rulesApplication.getI();
    336            }
    337
    338            // the phonemes map orders the phonemes only based on their text, but ignores the language set
    339            // when adding new phonemes, check for equal phonemes and merge their language set, otherwise
    340            // phonemes with the same text but different language set get lost
    341            subBuilder.getPhonemes().forEach(newPhoneme -> {
    342                if (phonemes.containsKey(newPhoneme)) {
    343                    final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme);
    344                    final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages());
    345                    phonemes.put(mergedPhoneme, mergedPhoneme);
    346                } else {
    347                    phonemes.put(newPhoneme, newPhoneme);
    348                }
    349            });
    350        });
    351
    352        return new PhonemeBuilder(phonemes.keySet());
    353    }
    354
    355    /**
    356     * Encodes a string to its phonetic representation.
    357     *
    358     * @param input
    359     *            the String to encode
    360     * @return the encoding of the input
    361     */
    362    public String encode(final String input) {
    363        final Languages.LanguageSet languageSet = this.lang.guessLanguages(input);
    364        return encode(input, languageSet);
    365    }
    366
    367    /**
    368     * Encodes an input string into an output phonetic representation, given a set of possible origin languages.
    369     *
    370     * @param input
    371     *            String to phoneticise; a String with dashes or spaces separating each word
    372     * @param languageSet
    373     *            set of possible origin languages
    374     * @return a phonetic representation of the input; a String containing '-'-separated phonetic representations of the
    375     *         input
    376     */
    377    public String encode(String input, final Languages.LanguageSet languageSet) {
    378        final Map<String, List<Rule>> rules = Rule.getInstanceMap(this.nameType, RuleType.RULES, languageSet);
    379        // rules common across many (all) languages
    380        final Map<String, List<Rule>> finalRules1 = Rule.getInstanceMap(this.nameType, this.ruleType, "common");
    381        // rules that apply to a specific language that may be ambiguous or wrong if applied to other languages
    382        final Map<String, List<Rule>> finalRules2 = Rule.getInstanceMap(this.nameType, this.ruleType, languageSet);
    383
    384        // tidy the input
    385        // lower case is a locale-dependent operation
    386        input = input.toLowerCase(Locale.ENGLISH).replace('-', ' ').trim();
    387
    388        if (this.nameType == NameType.GENERIC) {
    389            if (input.startsWith("d'")) { // check for d'
    390                final String remainder = input.substring(2);
    391                final String combined = "d" + remainder;
    392                return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
    393            }
    394            for (final String l : NAME_PREFIXES.get(this.nameType)) {
    395                // handle generic prefixes
    396                if (input.startsWith(l + " ")) {
    397                    // check for any prefix in the words list
    398                    final String remainder = input.substring(l.length() + 1); // input without the prefix
    399                    final String combined = l + remainder; // input with prefix without space
    400                    return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
    401                }
    402            }
    403        }
    404
    405        final List<String> words = Arrays.asList(input.split("\\s+"));
    406        final List<String> words2 = new ArrayList<>();
    407
    408        // special-case handling of word prefixes based upon the name type
    409        switch (this.nameType) {
    410        case SEPHARDIC:
    411            words.forEach(aWord -> {
    412                final String[] parts = aWord.split("'");
    413                words2.add(parts[parts.length - 1]);
    414            });
    415            words2.removeAll(NAME_PREFIXES.get(this.nameType));
    416            break;
    417        case ASHKENAZI:
    418            words2.addAll(words);
    419            words2.removeAll(NAME_PREFIXES.get(this.nameType));
    420            break;
    421        case GENERIC:
    422            words2.addAll(words);
    423            break;
    424        default:
    425            throw new IllegalStateException("Unreachable case: " + this.nameType);
    426        }
    427
    428        if (this.concat) {
    429            // concat mode enabled
    430            input = join(words2, " ");
    431        } else if (words2.size() == 1) {
    432            // not a multi-word name
    433            input = words.iterator().next();
    434        } else {
    435            // encode each word in a multi-word name separately (normally used for approx matches)
    436            final StringBuilder result = new StringBuilder();
    437            words2.forEach(word -> result.append("-").append(encode(word)));
    438            // return the result without the leading "-"
    439            return result.substring(1);
    440        }
    441
    442        PhonemeBuilder phonemeBuilder = PhonemeBuilder.empty(languageSet);
    443
    444        // loop over each char in the input - we will handle the increment manually
    445        for (int i = 0; i < input.length();) {
    446            final RulesApplication rulesApplication =
    447                    new RulesApplication(rules, input, phonemeBuilder, i, maxPhonemes).invoke();
    448            i = rulesApplication.getI();
    449            phonemeBuilder = rulesApplication.getPhonemeBuilder();
    450        }
    451
    452        // Apply the general rules
    453        phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules1);
    454        // Apply the language-specific rules
    455        phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules2);
    456
    457        return phonemeBuilder.makeString();
    458    }
    459
    460    /**
    461     * Gets the Lang language guessing rules being used.
    462     *
    463     * @return the Lang in use
    464     */
    465    public Lang getLang() {
    466        return this.lang;
    467    }
    468
    469    /**
    470     * Gets the NameType being used.
    471     *
    472     * @return the NameType in use
    473     */
    474    public NameType getNameType() {
    475        return this.nameType;
    476    }
    477
    478    /**
    479     * Gets the RuleType being used.
    480     *
    481     * @return the RuleType in use
    482     */
    483    public RuleType getRuleType() {
    484        return this.ruleType;
    485    }
    486
    487    /**
    488     * Gets if multiple phonetic encodings are concatenated or if just the first one is kept.
    489     *
    490     * @return true if multiple phonetic encodings are returned, false if just the first is
    491     */
    492    public boolean isConcat() {
    493        return this.concat;
    494    }
    495
    496    /**
    497     * Gets the maximum number of phonemes the engine will calculate for a given input.
    498     *
    499     * @return the maximum number of phonemes
    500     * @since 1.7
    501     */
    502    public int getMaxPhonemes() {
    503        return this.maxPhonemes;
    504    }
    505}