text.py

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities for text input preprocessing.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict
from hashlib import md5
import string
import sys

import numpy as np
from six.moves import range  # pylint: disable=redefined-builtin
from six.moves import zip  # pylint: disable=redefined-builtin

from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util.tf_export import tf_export


if sys.version_info < (3,):
  maketrans = string.maketrans
else:
  maketrans = str.maketrans


@tf_export('keras.preprocessing.text.text_to_word_sequence')
def text_to_word_sequence(text,
                          filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
                          lower=True,
                          split=' '):
  r"""Converts a text to a sequence of words (or tokens).

  Arguments:
      text: Input text (string).
      filters: list (or concatenation) of characters to filter out, such as
          punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
          includes basic punctuation, tabs, and newlines.
      lower: boolean, whether to convert the input to lowercase.
      split: string, separator for word splitting.

  Returns:
      A list of words (or tokens).
  """
  if lower:
    text = text.lower()

  if sys.version_info < (3,):
    if isinstance(text, unicode):
      translate_map = dict((ord(c), unicode(split)) for c in filters)
      text = text.translate(translate_map)
    elif len(split) == 1:
      translate_map = maketrans(filters, split * len(filters))
      text = text.translate(translate_map)
    else:
      for c in filters:
        text = text.replace(c, split)
  else:
    translate_dict = dict((c, split) for c in filters)
    translate_map = maketrans(translate_dict)
    text = text.translate(translate_map)

  seq = text.split(split)
  return [i for i in seq if i]


@tf_export('keras.preprocessing.text.one_hot')
def one_hot(text,
            n,
            filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
            lower=True,
            split=' '):
  r"""One-hot encodes a text into a list of word indexes of size n.

  This is a wrapper to the `hashing_trick` function using `hash` as the
  hashing function; unicity of word to index mapping non-guaranteed.

  Arguments:
      text: Input text (string).
      n: int, size of vocabulary.
      filters: list (or concatenation) of characters to filter out, such as
          punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
          includes basic punctuation, tabs, and newlines.
      lower: boolean, whether to set the text to lowercase.
      split: string, separator for word splitting.

  Returns:
      List of integers in [1, n].
      Each integer encodes a word (unicity non-guaranteed).
  """
  return hashing_trick(
      text, n, hash_function=hash, filters=filters, lower=lower, split=split)


@tf_export('keras.preprocessing.text.hashing_trick')
def hashing_trick(text,
                  n,
                  hash_function=None,
                  filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
                  lower=True,
                  split=' '):
  r"""Converts a text to a sequence of indexes in a fixed-size hashing space.

  Arguments:
      text: Input text (string).
      n: Dimension of the hashing space.
      hash_function: defaults to python `hash` function, can be 'md5' or
          any function that takes in input a string and returns a int.
          Note that 'hash' is not a stable hashing function, so
          it is not consistent across different runs, while 'md5'
          is a stable hashing function.
      filters: list (or concatenation) of characters to filter out, such as
          punctuation. Default: '!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
          includes basic punctuation, tabs, and newlines.
      lower: boolean, whether to set the text to lowercase.
      split: string, separator for word splitting.

  Returns:
      A list of integer word indices (unicity non-guaranteed).

  `0` is a reserved index that won't be assigned to any word.

  Two or more words may be assigned to the same index, due to possible
  collisions by the hashing function.
  The
  probability
  of a collision is in relation to the dimension of the hashing space and
  the number of distinct objects.
  """
  if hash_function is None:
    hash_function = hash
  elif hash_function == 'md5':
    hash_function = lambda w: int(md5(w.encode()).hexdigest(), 16)

  seq = text_to_word_sequence(text, filters=filters, lower=lower, split=split)
  return [(hash_function(w) % (n - 1) + 1) for w in seq]


@tf_export('keras.preprocessing.text.Tokenizer')
class Tokenizer(object):
  """Text tokenization utility class.

  This class allows to vectorize a text corpus, by turning each
  text into either a sequence of integers (each integer being the index
  of a token in a dictionary) or into a vector where the coefficient
  for each token could be binary, based on word count, based on tf-idf...

  Arguments:
      num_words: the maximum number of words to keep, based
          on word frequency. Only the most common `num_words` words will
          be kept.
      filters: a string where each element is a character that will be
          filtered from the texts. The default is all punctuation, plus
          tabs and line breaks, minus the `'` character.
      lower: boolean. Whether to convert the texts to lowercase.
      split: string, separator for word splitting.
      char_level: if True, every character will be treated as a token.
      oov_token: if given, it will be added to word_index and used to
          replace out-of-vocabulary words during text_to_sequence calls

  By default, all punctuation is removed, turning the texts into
  space-separated sequences of words
  (words maybe include the `'` character). These sequences are then
  split into lists of tokens. They will then be indexed or vectorized.

  `0` is a reserved index that won't be assigned to any word.
  """

  def __init__(self,
               num_words=None,
               filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n',
               lower=True,
               split=' ',
               char_level=False,
               oov_token=None,
               **kwargs):
    # Legacy support
    if 'nb_words' in kwargs:
      logging.warning('The `nb_words` argument in `Tokenizer` '
                      'has been renamed `num_words`.')
      num_words = kwargs.pop('nb_words')
    if kwargs:
      raise TypeError('Unrecognized keyword arguments: ' + str(kwargs))

    self.word_counts = OrderedDict()
    self.word_docs = {}
    self.filters = filters
    self.split = split
    self.lower = lower
    self.num_words = num_words
    self.document_count = 0
    self.char_level = char_level
    self.oov_token = oov_token
    self.index_docs = {}

  def fit_on_texts(self, texts):
    """Updates internal vocabulary based on a list of texts.

    In the case where texts contains lists, we assume each entry of the lists
    to be a token.

    Required before using `texts_to_sequences` or `texts_to_matrix`.

    Arguments:
        texts: can be a list of strings,
            a generator of strings (for memory-efficiency),
            or a list of list of strings.
    """
    for text in texts:
      self.document_count += 1
      if self.char_level or isinstance(text, list):
        seq = text
      else:
        seq = text_to_word_sequence(text, self.filters, self.lower, self.split)
      for w in seq:
        if w in self.word_counts:
          self.word_counts[w] += 1
        else:
          self.word_counts[w] = 1
      for w in set(seq):
        if w in self.word_docs:
          self.word_docs[w] += 1
        else:
          self.word_docs[w] = 1

    wcounts = list(self.word_counts.items())
    wcounts.sort(key=lambda x: x[1], reverse=True)
    sorted_voc = [wc[0] for wc in wcounts]
    # note that index 0 is reserved, never assigned to an existing word
    self.word_index = dict(
        list(zip(sorted_voc, list(range(1,
                                        len(sorted_voc) + 1)))))

    if self.oov_token is not None:
      i = self.word_index.get(self.oov_token)
      if i is None:
        self.word_index[self.oov_token] = len(self.word_index) + 1

    for w, c in list(self.word_docs.items()):
      self.index_docs[self.word_index[w]] = c

  def fit_on_sequences(self, sequences):
    """Updates internal vocabulary based on a list of sequences.

    Required before using `sequences_to_matrix`
    (if `fit_on_texts` was never called).

    Arguments:
        sequences: A list of sequence.
            A "sequence" is a list of integer word indices.
    """
    self.document_count += len(sequences)
    for seq in sequences:
      seq = set(seq)
      for i in seq:
        if i not in self.index_docs:
          self.index_docs[i] = 1
        else:
          self.index_docs[i] += 1

  def texts_to_sequences(self, texts):
    """Transforms each text in texts in a sequence of integers.

    Only top "num_words" most frequent words will be taken into account.
    Only words known by the tokenizer will be taken into account.

    Arguments:
        texts: A list of texts (strings).

    Returns:
        A list of sequences.
    """
    res = []
    for vect in self.texts_to_sequences_generator(texts):
      res.append(vect)
    return res

  def texts_to_sequences_generator(self, texts):
    """Transforms each text in `texts` in a sequence of integers.

    Each item in texts can also be a list, in which case we assume each item of
    that list
    to be a token.

    Only top "num_words" most frequent words will be taken into account.
    Only words known by the tokenizer will be taken into account.

    Arguments:
        texts: A list of texts (strings).

    Yields:
        Yields individual sequences.
    """
    num_words = self.num_words
    for text in texts:
      if self.char_level or isinstance(text, list):
        seq = text
      else:
        seq = text_to_word_sequence(text, self.filters, self.lower, self.split)
      vect = []
      for w in seq:
        i = self.word_index.get(w)
        if i is not None:
          if num_words and i >= num_words:
            continue
          else:
            vect.append(i)
        elif self.oov_token is not None:
          i = self.word_index.get(self.oov_token)
          if i is not None:
            vect.append(i)
      yield vect

  def texts_to_matrix(self, texts, mode='binary'):
    """Convert a list of texts to a Numpy matrix.

    Arguments:
        texts: list of strings.
        mode: one of "binary", "count", "tfidf", "freq".

    Returns:
        A Numpy matrix.
    """
    sequences = self.texts_to_sequences(texts)
    return self.sequences_to_matrix(sequences, mode=mode)

  def sequences_to_matrix(self, sequences, mode='binary'):
    """Converts a list of sequences into a Numpy matrix.

    Arguments:
        sequences: list of sequences
            (a sequence is a list of integer word indices).
        mode: one of "binary", "count", "tfidf", "freq"

    Returns:
        A Numpy matrix.

    Raises:
        ValueError: In case of invalid `mode` argument,
            or if the Tokenizer requires to be fit to sample data.
    """
    if not self.num_words:
      if self.word_index:
        num_words = len(self.word_index) + 1
      else:
        raise ValueError('Specify a dimension (num_words argument), '
                         'or fit on some text data first.')
    else:
      num_words = self.num_words

    if mode == 'tfidf' and not self.document_count:
      raise ValueError('Fit the Tokenizer on some data '
                       'before using tfidf mode.')

    x = np.zeros((len(sequences), num_words))
    for i, seq in enumerate(sequences):
      if not seq:
        continue
      counts = {}
      for j in seq:
        if j >= num_words:
          continue
        if j not in counts:
          counts[j] = 1.
        else:
          counts[j] += 1
      for j, c in list(counts.items()):
        if mode == 'count':
          x[i][j] = c
        elif mode == 'freq':
          x[i][j] = c / len(seq)
        elif mode == 'binary':
          x[i][j] = 1
        elif mode == 'tfidf':
          # Use weighting scheme 2 in
          # https://en.wikipedia.org/wiki/Tf%E2%80%93idf
          tf = 1 + np.log(c)
          idf = np.log(1 + self.document_count /
                       (1 + self.index_docs.get(j, 0)))
          x[i][j] = tf * idf
        else:
          raise ValueError('Unknown vectorization mode:', mode)
    return x