convert_language_model.py

#!/usr/bin/env python

######################## BEGIN LICENSE BLOCK ########################
# Contributor(s):
#   10.02.2015 - helour - first attempt
#   08.08.2016 - Dan Blanchard - first usable release
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
#
######################### END LICENSE BLOCK #########################

"""
Convert old style SBCS model to new
"""

import os
import sys
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from string import ascii_letters

import chardet
from chardet import __version__
from chardet.metadata.languages import LANGUAGES
from chardet.sbcharsetprober import SingleByteCharSetModel

# Turn ascii_letters into a set to make other ops easier
ascii_letters = set(ascii_letters)


def normalize_name(charset_name):
    """Convert name to proper Python constant format"""
    # Title case to start
    charset_name = charset_name.upper()
    # Underscores instead of hyphens
    charset_name = charset_name.replace("-", "_")
    return charset_name


def convert_sbcs_model(old_model, alphabet):
    """Create a SingleByteCharSetModel object representing the charset."""
    # Setup tables necessary for computing transition frequencies for model
    char_to_order = dict(enumerate(old_model["char_to_order_map"]))
    pos_ratio = old_model["typical_positive_ratio"]
    keep_ascii_letters = old_model["keep_english_letter"]

    curr_model = SingleByteCharSetModel(
        charset_name=old_model["charset_name"],
        language=old_model["language"],
        char_to_order_map=char_to_order,
        # language_model is filled in later
        language_model=None,
        typical_positive_ratio=pos_ratio,
        keep_ascii_letters=keep_ascii_letters,
        alphabet=alphabet,
    )
    return curr_model


def print_char_to_order(var_name, order_map, charset_name, output_file):
    print(f"{var_name} = {{", file=output_file)
    for char, order in sorted(order_map.items()):
        char_bytes = bytes(bytearray((char,)))
        try:
            unicode_char = char_bytes.decode(charset_name)
        except UnicodeError:
            unicode_char = None
        print(f"     {char!r}: {order!r},  # {unicode_char!r}", file=output_file)
    print("}\n", file=output_file)


def print_language_model(var_name, language_model, output_file, char_ranks):
    print(
        "# 3: Positive\n# 2: Likely\n# 1: Unlikely\n# 0: Negative\n",
        file=output_file,
    )
    print(f"{var_name} = {{", file=output_file)
    for first_char, sub_dict in sorted(language_model.items()):
        # Skip empty sub_dicts
        if not sub_dict or first_char not in char_ranks:
            continue
        print(f"    {char_ranks[first_char]!r}: {{  # {first_char!r}", file=output_file)
        for second_char, likelihood in sorted(sub_dict.items()):
            if second_char not in char_ranks:
                continue
            print(
                f"        {char_ranks[second_char]!r}: {likelihood!r},  # "
                f"{second_char!r}",
                file=output_file,
            )
        print("    },", file=output_file)
    print("}\n", file=output_file)


def convert_models_for_lang(language):
    """Convert old SingleByteCharSetModels for the given language"""
    # Validate language
    language = language.title()
    lang_metadata = LANGUAGES.get(language)
    if not lang_metadata:
        raise ValueError(
            f"Unknown language: {language}. If you are adding a model for a"
            " new language, you must first update metadata/"
            "languages.py"
        )
    lang_mod_name = f"lang{language.lower()}model"
    if not os.path.exists(os.path.join("chardet", lang_mod_name + ".py")):
        print(f"Skipping {language} because it does not have an old model.")
        return
    lang_mod = getattr(chardet, lang_mod_name)
    print(
        f"\n{language}\n----------------------------------------------------------------"
    )
    print(f"Keep ASCII Letters: {lang_metadata.use_ascii}")
    print(f"Alphabet: {lang_metadata.alphabet}")

    # Create char-to-order maps (aka char-to-rank dicts)
    charset_models = {}
    char_ranks = {}
    order_to_chars = {}
    for var_name in dir(lang_mod):
        if not ("Model" in var_name and "LangModel" not in var_name):
            continue
        old_model = getattr(lang_mod, var_name)
        charset_name = old_model["charset_name"]

        print(f"Converting charset model for {charset_name}")
        sys.stdout.flush()
        charset_models[charset_name] = convert_sbcs_model(
            old_model, lang_metadata.alphabet
        )
        # Since we don't know which charsets have which characters, we have to
        # try to reconstruct char_ranks (for letters only, since that's all
        # the old language models contain)
        for byte_hex, order in charset_models[charset_name].char_to_order_map.items():
            # order 64 was basically ignored before because of the off by one
            # error, but it's hard to know if training took that into account
            if order > 64:
                continue
            # Convert to bytes in Python 2 and 3
            char = bytes(bytearray((byte_hex,)))
            try:
                unicode_char = char.decode(charset_name)
            except UnicodeDecodeError:
                continue
            if unicode_char not in char_ranks:
                char_ranks[unicode_char] = order
                order_to_chars[order] = unicode_char
            elif char_ranks[unicode_char] != order:
                raise ValueError(f"Unstable character ranking for {unicode_char}")

    old_lang_model = getattr(lang_mod, f"{language.title()}LangModel")
    language_model = {}
    # Preserve off-by-one error here by ignoring first column and row
    for i in range(1, 64):
        if i not in order_to_chars:
            continue
        lang_char = order_to_chars[i]
        language_model[lang_char] = {}
        for j in range(1, 64):
            if j not in order_to_chars:
                continue
            lang_char2 = order_to_chars[j]
            language_model[lang_char][lang_char2] = old_lang_model[(i * 64) + j]

    # Write output files
    print(f"Writing output file for {language}\n\n")
    sys.stdout.flush()
    with open(f"lang{language.lower()}model.py", "w", encoding="utf-8") as output_file:
        upper_lang = language.upper()
        # print header to set encoding
        print(
            "from chardet.sbcharsetprober import SingleByteCharSetModel\n\n",
            file=output_file,
        )

        lm_name = f"{upper_lang}_LANG_MODEL"
        print_language_model(lm_name, language_model, output_file, char_ranks)

        print(
            "# 255: Undefined characters that did not exist in training text\n"
            "# 254: Carriage/Return\n"
            "# 253: symbol (punctuation) that does not belong to word\n"
            "# 252: 0 - 9\n"
            "# 251: Control characters\n\n"
            "# Character Mapping Table(s):",
            file=output_file,
        )
        for charset_name, sbcs_model in charset_models.items():
            normal_name = normalize_name(charset_name)
            char_to_order_name = f"{normal_name}_{upper_lang}_CHAR_TO_ORDER"
            print_char_to_order(
                char_to_order_name,
                sbcs_model.char_to_order_map,
                charset_name,
                output_file,
            )

            sbcs_model_name = f"{normal_name}_{upper_lang}_MODEL"
            sbcs_model.char_to_order_map.clear()
            sbcs_model_repr = (
                repr(sbcs_model)
                .replace("None", lm_name)
                .replace("{}", char_to_order_name)
                .replace(", ", (",\n" + " " * (len(sbcs_model_name) + 26)))
            )
            print(f"{sbcs_model_name} = {sbcs_model_repr}\n", file=output_file)


def main():
    parser = ArgumentParser(
        description=__doc__, formatter_class=ArgumentDefaultsHelpFormatter
    )
    parser.add_argument(
        "language",
        help="The name of the language the input documents are "
        "in. Also the name of the language the generated "
        "model will detect. If no language is specified, "
        "models for all languages known to chardet will be"
        " trained.",
        nargs="*",
    )
    parser.add_argument("--version", action="version", version=__version__)
    args = parser.parse_args()

    if not args.language:
        args.language = list(sorted(LANGUAGES.keys()))

    for language in args.language:
        convert_models_for_lang(language)


if __name__ == "__main__":
    main()