Make tokens_substitute() to replace characters in tokens?

[koheiw]

Currently, we need to use both tokens_replace() and stri_replace_all_fixed() to replace "d.c" with "dc" or "u.s" with "us". Why we cannot do this by tokens_substitute(toks, ".", "")? This is a similar to tokens_split().

require(quanteda)
#> Loading required package: quanteda
#> Package version: 4.0.0
#> Unicode version: 15.1
#> ICU version: 74.1
#> Parallel computing: 16 of 16 threads used.
#> See https://quanteda.io for tutorials and examples.
toks <- tokens("Washington d.c is the u.s capital.")
toks
#> Tokens consisting of 1 document.
#> text1 :
#> [1] "Washington" "d.c"        "is"         "the"        "u.s"       
#> [6] "capital"    "."
tokens_replace(toks, types(toks), stringi::stri_replace_all_fixed(types(toks), ".", ""))
#> Tokens consisting of 1 document.
#> text1 :
#> [1] "Washington" "dc"         "is"         "the"        "us"        
#> [6] "capital"    ""

tokens_substitute <- function(x, pattern, replacement, valuetype = c("fixed", "regex")) {
  
  valuetype <- match.arg(valuetype)
  type <- types(toks)
  if (valuetype == "fixed") {
    attr(x, "types") <- stringi::stri_replace_all_fixed(type, pattern, replacement)
  } else {
    attr(x, "types") <- stringi::stri_replace_all_regex(type, pattern, replacement)
  }
  return(x)
}

tokens_substitute(toks, ".", "") 
#> Tokens consisting of 1 document.
#> text1 :
#> [1] "Washington" "dc"         "is"         "the"        "us"        
#> [6] "capital"    ""