Merge pull request #9159 from quarto-dev/feature/escape-pandoc-ast

Feature: `pandoc-native` and `pandoc-json` rawblock and rawinline elements

news/changelog-1.5.md

@@ -106,6 +106,10 @@ All changes included in 1.5:
 
 - ([#8939](https://github.com/quarto-dev/quarto-cli/pull/8939)): `quarto inspect` now takes an additional optional parameter to specify the output file, and provides the graph of include dependencies for the inspection target.
 
+## Quarto's input format
+
+- Quarto now supports raw block and raw inline elements of types `pandoc-native` and `pandoc-json`, and will use Pandoc's `native` and `json` reader to convert these elements to Pandoc's AST. This is useful in situations where emitting Markdown is not sufficient or convient enough to express the desired structure of a document.
+
 ## Other Fixes and Improvements
 
 - ([#8119](https://github.com/quarto-dev/quarto-cli/issues/8119)): More intelligently detect when ejs templates are modified during development, improving quality of life during preview.

src/resources/filters/ast/customnodes.lua

@@ -79,6 +79,7 @@ function run_emulated_filter(doc, filter)
         return node
       else
         -- luacov: disable
+        quarto.utils.dump(node)
         internal_error()
         -- luacov: enable
       end
@@ -259,6 +260,42 @@ function create_emulated_node(t, tbl, context, forwarder)
   return result, custom_node_data[id]
 end
 
+-- walk_meta walks a Pandoc Meta object, applying the filter to each node
+-- and recursing on lists and objects. It mutates the meta object in place
+-- and returns it.
+--
+-- It performs slightly more work than a regular walk filter because of the
+-- ambiguity around single-element lists and objects.
+function walk_meta(meta, filter)
+  local skip = {
+    ["nil"] = true,
+    number = true,
+    boolean = true,
+    string = true,
+    ["function"] = true,
+  }
+  local iterate = {
+    Meta = true,
+    List = true,
+  }
+  local function walk(obj)
+    local t = type(obj)
+    if skip[t] then
+      return obj
+    end
+    local pt = quarto.utils.type(obj)
+    if iterate[pt] then
+      for k, v in pairs(obj) do
+        obj[k] = walk(v)
+      end
+    else
+      return _quarto.ast.walk(obj, filter)
+    end
+    return obj
+  end
+  return walk(meta)
+end
+
 _quarto.ast = {
   vault = {
     _uuid = "3ade8a4a-fb1d-4a6c-8409-ac45482d5fc9",
@@ -495,6 +532,8 @@ _quarto.ast = {
 
   walk = run_emulated_filter,
 
+  walk_meta = walk_meta,
+
   writer_walk = function(doc, filter)
     local old_custom_walk = filter.Custom
     local function custom_walk(node)

src/resources/filters/normalize/extractquartodom.lua

@@ -25,7 +25,35 @@ function parse_md_in_html_rawblocks()
         end
         return make_scaffold(pandoc.Span, inlines)
       end
-    end
+    end,
+    RawBlock = function(raw)
+      local result
+      if raw.format == "pandoc-native" then
+        result = pandoc.read(raw.text, "native").blocks
+      elseif raw.format == "pandoc-json" then
+        result = pandoc.read(raw.text, "json").blocks
+      else
+        return raw
+      end
+      return result
+    end,
+    RawInline = function(raw)
+      local result
+      if raw.format == "pandoc-native" then
+        result = quarto.utils.as_inlines(pandoc.read(raw.text, "native").blocks)
+      elseif raw.format == "pandoc-json" then
+        -- let's try to be minimally smart here, and handle lists differently from a single top-level element
+        result = quarto.utils.as_inlines(pandoc.read(raw.text, "json").blocks)
+      else
+        return raw
+      end
+      return result
+    end,
+    -- Meta = function(meta)
+    --   local filter = parse_md_in_html_rawblocks()
+    --   local result = _quarto.ast.walk_meta(meta, filter)
+    --   return result
+    -- end,
   }
 end
 

tests/docs/ast/test-pandoc-native-raw-in-meta.qmd

@@ -0,0 +1,13 @@
+---
+format: html
+key-good: "`Str \"path-to-directory-with--two-dashes/file\"`{=pandoc-native}"
+key-bad: path-to-directory-with--two-dashes/file
+_quarto:
+  tests:
+    html:
+      ensureFileRegexMatches:
+        - ["--"]
+        - ["–"]
+---
+
+Hello {{< meta key-good >}}. (This would fail with `{{{< meta key-bad >}}}`)

tests/docs/ast/test-quarto-pandoc-json-raw.qmd

@@ -0,0 +1,13 @@
+---
+format: html
+_quarto:
+  tests:
+    html:
+      ensureFileRegexMatches:
+        - ["--"]
+        - ["–"]
+---
+
+(The JSON syntax is decidedly more verbose and not as nice, and really intended to be used by machine-generated markdown.)
+
+Here is a string that shouldn't be escaped: `{"pandoc-api-version":[1,23,1],"meta":{},"blocks":[{"t":"Plain","c":[{"t":"Str","c":"do--not--escape--this"}]}]}`{=pandoc-json}.

tests/docs/ast/test-quarto-pandoc-native-raw.qmd

@@ -0,0 +1,11 @@
+---
+format: html
+_quarto:
+  tests:
+    html:
+      ensureFileRegexMatches:
+        - ["--"]
+        - ["–"]
+---
+
+Here is a string that shouldn't be escaped: `Str "do--not--escape--this"`{=pandoc-native}.