Add docs about special token precedence. (#3349)

parlai/core/torch_agent.py

@@ -655,7 +655,11 @@ def add_cmdline_args(
             '--special-tok-lst',
             type=str,
             default=None,
-            help='Comma separated list of special tokens',
+            help=(
+                'Comma separated list of special tokens. '
+                'In case of ambiguous parses from special tokens, the ordering '
+                'provided in this arg sets precedence.'
+            ),
         )
         # GPU arguments
         # these gpu options are all mutually exclusive, and should error if the
@@ -867,6 +871,12 @@ def _get_special_tokens(self) -> List[str]:
         Return list of special tokens.
 
         Made easily overridable for special cases.
+
+        Note that in the case of ambiguity of special-token parsing, the
+        precedence is set by the ordering returned in this method.  For
+        example, if special tokens are ["OHB", "BOY"], parsing "OHBOY" will
+        become (special)OHB and (normal)OY. But with special tokens
+        ["BOY", "OHB"], then we will get (normal)OH and (special)BOY.
         """
         if self.opt.get('special_tok_lst'):
             return self.opt['special_tok_lst'].split(',')