Merge branch 'master' into gh-2104

.gitmodules

@@ -18,3 +18,7 @@
 	path = submodules/zotero-odf-scan-plugin
 	url = https://github.com/Juris-M/zotero-odf-scan-plugin.git
 	branch = master
+[submodule "submodules/zotero-utilities"]
+	path = submodules/zotero-utilities
+	url = https://github.com/zotero/utilities.git
+	branch = master

content/Preferences/preferences.pug

@@ -158,7 +158,7 @@ bbt:doc When you set this on, BBT will import bib files leaving any LaTeX comman
 preference(name="extensions.zotero.translators.better-bibtex.skipWords" bbt:affects="tex" type="string" default="a,ab,aboard,about,above,across,after,against,al,along,amid,among,an,and,anti,around,as,at,before,behind,below,beneath,beside,besides,between,beyond,but,by,d,da,das,de,del,dell,dello,dei,degli,della,dell,delle,dem,den,der,des,despite,die,do,down,du,during,ein,eine,einem,einen,einer,eines,el,en,et,except,for,from,gli,i,il,in,inside,into,is,l,la,las,le,les,like,lo,los,near,nor,of,off,on,onto,or,over,past,per,plus,round,save,since,so,some,sur,than,the,through,to,toward,towards,un,una,unas,under,underneath,une,unlike,uno,unos,until,up,upon,versus,via,von,while,with,within,without,yet,zu,zum")
 bbt:doc list of words to skip in title when generating citation keys
 
-preference(name="extensions.zotero.translators.better-bibtex.verbatimFields" bbt:affects="" type="string" default="url,doi,file,ids,eprint,verba,verbb,verbc,groups")
+preference(name="extensions.zotero.translators.better-bibtex.verbatimFields" bbt:affects="" type="string" default="url,doi,file,ids,eprint,/^verb[a-z]$/,groups,/^citeulike-linkout-[0-9]+$/, /^bdsk-url-[0-9]+$/")
 bbt:doc list of fields to treat as verbatim during import. If you're importing e.g. Mendeley-generated BibTeX, which is out of spec in various ways, try removing `file` from this list before import.
 
 preference#pref-better-bibtex-qualityReport(name="extensions.zotero.translators.better-bibtex.qualityReport" bbt:affects="tex" type="bool" default="false")

content/extra.ts

@@ -68,19 +68,23 @@ const casing = {
 }
 
 export function get(extra: string, mode: 'zotero' | 'csl', options?: GetOptions): { extra: string, extraFields: Fields } {
-  if (!options) options = { citationKey: true , aliases: true, kv: true, tex: true }
+  let defaults = false
+  if (!options) {
+    options = { citationKey: true , aliases: true, kv: true, tex: true }
+    defaults = true
+  }
 
   const other = {zotero: 'csl', csl: 'zotero'}[mode]
 
   extra = extra || ''
 
   const extraFields: Fields = {
-    kv: {},
+    kv: options.kv || defaults ? {} : undefined,
     creator: {},
     creators: [],
-    tex: {},
+    tex: options.tex || defaults ? {} : undefined,
     citationKey: '',
-    aliases: [],
+    aliases: options.aliases || defaults ? [] : undefined,
   }
 
   let ef
@@ -115,7 +119,7 @@ export function get(extra: string, mode: 'zotero' | 'csl', options?: GetOptions)
       return false
     }
 
-    if (options.kv && (ef = mapping[key]) && !tex) {
+    if (options.kv && key !== 'citation key' && (ef = mapping[key]) && !tex) {
       for (const field of (ef[mode] || ef[other])) {
         switch (ef.type) {
           case 'name':

content/json-rpc.ts

@@ -48,8 +48,6 @@ class NSAutoExport {
    * @param translator                             The name or GUID of a BBT translator
    * @param path                                   The absolute path to which the collection will be auto-exported
    * @param displayOptions                         Options which you would be able to select during an interactive export; `exportNotes`, default `false`, and `useJournalAbbreviation`, default `false`
-   * @param displayOptions.exportNotes             Export notes
-   * @param displayOptions.useJournalAbbreviation  Use Journal abbreviation in export
    * @param replace                                Replace the auto-export if it exists, default `false`
    * @returns                                      Collection ID of the target collection
    */

content/key-manager/formatter.ts

@@ -392,6 +392,7 @@ class PatternFormatter {
 
   /**
    * Tests whether the item is of any of the given types, and skips to the next pattern if not
+   * @param allowed one or more item type names
    */
   public $type(...allowed: string[]) {
     if (allowed.map(type => type.toLowerCase()).includes(this.item.itemType.toLowerCase())) {
@@ -404,6 +405,7 @@ class PatternFormatter {
 
   /**
    * Tests whether the item has the given language set, and skips to the next pattern if not
+   * @param name one or more language codes
    */
   public $language(...name: (BabelLanguage | BabelLanguageTag)[]) {
     if (name.concat(name.map(n => BabelTag[n] as string).filter(n => n)).includes(this.item.babelTag())) {
@@ -440,6 +442,7 @@ class PatternFormatter {
 
   /**
    * Gets the value of the item field
+   * @param name name of the field
    */
   public $getField(name: string) {
     const value = this.item.getField(name)
@@ -463,15 +466,15 @@ class PatternFormatter {
   }
 
   /**
-   * Author/editor information. Parameters are:
-   * - `n`: select the first `n` authors (when passing a number) or the authors in this range (inclusive, when passing two values); negative numbers mean "from the end", default = 0 = all,
-   * - `creator`: select type of creator (`author` or `editor`),
-   * - `name`: sprintf-js template. Available named parameters are: `f` (family name), `g` (given name), `i` (initials)
-   * - `etal`: use this term to replace authors after `n` authors have been named,
-   * - `sep`: use this character between authors
-   * - `clean`: transliterates the citation key and removes unsafe characters
-   * - `min`: skip to the next pattern if there are less than `min` creators
-   * - `max`: skip to the next pattern if there are more than `max` creators
+   * Author/editor information.
+   * @param n       select the first `n` authors (when passing a number) or the authors in this range (inclusive, when passing two values); negative numbers mean "from the end", default = 0 = all
+   * @param creator select type of creator (`author` or `editor`)
+   * @param name    sprintf-js template. Available named parameters are: `f` (family name), `g` (given name), `i` (initials)
+   * @param etal    use this term to replace authors after `n` authors have been named
+   * @param sep     use this character between authors
+   * @param clean   transliterates the citation key and removes unsafe characters
+   * @param min     skip to the next pattern if there are less than `min` creators, 0 = ignore
+   * @param max     skip to the next pattern if there are more than `max` creators, 0 = ignore
    */
   public $authors(
     n: number | [number, number] = 0,
@@ -505,7 +508,14 @@ class PatternFormatter {
     return this.$text(author)
   }
 
-  /** The first `N` (default: all) characters of the `M`th (default: first) author's last name. */
+  /**
+   * The first `n` (default: all) characters of the `m`th (default: first) author's last name.
+   * @param n         the number of characters to take from the name, 0 = all
+   * @param m         select the `m`th author
+   * @param creator   select from authors or only from editors
+   * @param initials  add author initials
+   * @param clean     transliterates the citation key and removes unsafe characters
+   */
   public $auth(n=0, m=1, creator: 'author' | 'editor' = 'author', initials=false, clean=true) {
     const family = n ? `%(f).${n}s` : '%(f)s'
     const name = initials ? `${family}%(I)s` : family
@@ -563,7 +573,7 @@ class PatternFormatter {
     return this.$text(author)
   }
 
-  /** The beginning of each author's last name, using no more than `N` characters. */
+  /** The beginning of each author's last name, using no more than `n` characters (0 = all). */
   public $authIni(n=0, creator: 'author' | 'editor' = 'author', initials=false, sep='.', clean=true) {
     const authors = this.creators(creator === 'editor', initials ? '%(f)s%(I)s' : '%(f)s')
     if (!authors.length) return this.$text('')
@@ -680,21 +690,21 @@ class PatternFormatter {
     return this.$text(pages.split(/[-\s,–]/).pop() || '')
   }
 
-  /** Tag number `N` */
+  /** Tag number `n` */
   public $keyword(n: number) {
     const tag: string | { tag: string} = this.item.getTags()?.[n] || ''
     return this.$text(typeof tag === 'string' ? tag : tag.tag)
   }
 
-  /** The first `N` (default: 3) words of the title, apply capitalization to first `M` (default: 0) of those */
+  /** The first `n` (default: 3) words of the title, apply capitalization to first `m` (default: 0) of those */
   public $shorttitle(n: number = 3, m: number = 0) { // eslint-disable-line no-magic-numbers, @typescript-eslint/no-inferrable-types
     const words = this.titleWords(this.item.title, { skipWords: true, asciiOnly: true})
     if (!words) return this.$text('')
 
     return this.$text(words.slice(0, n).map((word, i) => i < m ? word.charAt(0).toUpperCase() + word.slice(1) : word).join(' '))
   }
 
-  /** The first `N` (default: 1) words of the title, apply capitalization to first `M` (default: 0) of those */
+  /** The first `n` (default: 1) words of the title, apply capitalization to first `m` (default: 0) of those */
   public $veryshorttitle(n: number = 1, m: number = 0) { // eslint-disable-line no-magic-numbers, @typescript-eslint/no-inferrable-types
     return this.$shorttitle(n, m)
   }

content/logger.ts

@@ -13,14 +13,14 @@ class Logger {
 
   protected timestamp: number
 
-  private format({ error=false, worker='', translator=''}, msg) {
+  private format({ error=false, worker='', translator='', issue=0}, msg) {
     let diff = null
     const now = Date.now()
     if (this.timestamp) diff = now - this.timestamp
     this.timestamp = now
 
     if (typeof msg !== 'string') {
-      let output = ''
+      let output = issue ? `issue ${issue}: `: ''
       for (const m of msg) {
         const type = typeof m
         if (type === 'string' || m instanceof String || type === 'number' || type === 'undefined' || type === 'boolean' || m === null) {
@@ -79,6 +79,10 @@ class Logger {
     if (this.enabled) Zotero.debug(this.format({}, msg))
   }
 
+  public for(issue: number, ...msg) {
+    if (this.enabled) Zotero.debug(this.format({ issue }, msg))
+  }
+
   public error(...msg) {
     Zotero.debug(this.format({error: true}, msg))
   }

content/worker/zotero.ts

@@ -5,6 +5,29 @@ importScripts('resource://gre/modules/osfile.jsm')
 import type { ITranslator } from '../../translators/lib/translator'
 import type { Translators } from '../../typings/translators'
 
+import { DOMParser as XMLDOMParser } from '@xmldom/xmldom'
+
+function hasAttribute(attr: string): boolean {
+  return !!(this.getAttribute?.(attr))
+}
+function upgrade(node) {
+  if (!node.children) {
+    node.children = Array.from(node.childNodes || [])
+    for (const child of node.children) {
+      upgrade(child)
+    }
+  }
+  if (!node.hasAttribute) node.hasAttribute = hasAttribute
+}
+export class DOMParser extends XMLDOMParser {
+  parseFromString(text: string, contentType: string) { // eslint-disable-line @typescript-eslint/explicit-module-boundary-types
+    const doc = super.parseFromString(text, contentType)
+    upgrade(doc)
+    return doc
+  }
+}
+const ZU = require('../../submodules/zotero-utilities/utilities.js')
+
 declare const doExport: () => void
 declare const Translator: ITranslator
 declare const dump: (message: string) => void
@@ -96,6 +119,10 @@ class WorkerZoteroUtilities {
     return workerContext.version
   }
 
+  walkNoteDOM(note, visitors) {
+    ZU.walkNoteDOM(note, visitors)
+  }
+
   public text2html(str: string, singleNewlineIsParagraph: boolean) {
     str = str
       .replace(/&/g, '&')

esbuild.js

@@ -26,7 +26,7 @@ async function bundle(config) {
   config = {
     bundle: true,
     format: 'iife',
-    define: { BigInt: 'Number' },
+    // define: { BigInt: 'Number' },
     target: ['firefox60'],
     inject: [],
     ...config,
@@ -48,7 +48,7 @@ async function bundle(config) {
     target = `${config.outdir} [${config.entryPoints.join(', ')}]`
   }
   console.log('* bundling', target)
-  console.log('  aliasing BigInt to Number for https://github.com/benjamn/ast-types/issues/750')
+  // console.log('  aliasing BigInt to Number for https://github.com/benjamn/ast-types/issues/750')
   const meta = (await esbuild.build(config)).metafile
   if (typeof metafile === 'string') await fs.promises.writeFile(metafile, JSON.stringify(meta, null, 2))
 }
@@ -76,7 +76,7 @@ async function rebuild() {
   })
 
   // worker code
-  const vars = [ 'Zotero', 'workerContext' ]
+  const vars = [ 'Zotero', 'workerContext', 'DOMParser' ]
   const globalName = vars.join('__')
   await bundle({
     entryPoints: [ 'content/worker/zotero.ts' ],
@@ -102,6 +102,7 @@ async function rebuild() {
       ].join('\n'),
     },
     metafile: 'gen/worker.json',
+    external: [ 'jsdom' ],
   })
 
   // translators