add google doc parser

Co-authored-by: austinpower1258 <austinpower1258@users.noreply.github.com>

Author: lawrencecchen

package.json

@@ -24,6 +24,12 @@
     "eslint": "^7.22.0",
     "eslint-config-prettier": "^8.1.0",
     "eslint-plugin-svelte3": "^3.2.0",
+    "googleapis": "^76.0.0",
+    "hast": "^1.0.0",
+    "hast-util-heading": "^2.0.0",
+    "hast-util-heading-rank": "^2.1.0",
+    "hast-util-to-text": "^3.0.0",
+    "hastscript": "^7.0.1",
     "performant-array-to-tree": "^1.9.1",
     "postcss": "^8.3.0",
     "prettier": "~2.2.1",

schema.sql

@@ -47,6 +47,7 @@ CREATE OR REPLACE RECURSIVE VIEW topics_tree (
   parent_id,
   item_id,
   slug,
+  doc_url,
   path,
   slug_arr
 ) AS
@@ -56,6 +57,7 @@ CREATE OR REPLACE RECURSIVE VIEW topics_tree (
     t1.parent_id,
     t1.item_id,
     t1.slug,
+    t1.doc_url,
     t1.slug as path,
     array[t1.slug]::text[] as slug_arr
   FROM
@@ -69,6 +71,7 @@ CREATE OR REPLACE RECURSIVE VIEW topics_tree (
       t2.parent_id,
       t2.item_id,
       t2.slug,
+      t2.doc_url,
       t_tree.path || '/' || t2.slug as path,
       t_tree.slug_arr || t2.slug as slug_arr
     FROM

src/lib/CourseEditor/Lesson.svelte

@@ -44,11 +44,11 @@
 	<div class="rounded-b space-x-4 bg-gray-50 border-t flex items-center justify-start px-6 py-3">
 		<button
 			on:click={addLesson}
-			class="inline-flex justify-center py-2 px-4 border border-transparent shadow-sm text-xs font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500"
+			class="inline-flex justify-center py-2 px-3 border border-transparent shadow-sm text-xs font-medium rounded-md text-white bg-indigo-600 hover:bg-indigo-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500"
 			>Add new lesson
 		</button>
 		<button
-			class="flex px-3 py-2 text-sm justify-center items-center transition border rounded-md text-gray-600  dark svelte-1fb58l0"
+			class="flex px-3 py-2 text-xs justify-center items-center transition border rounded-md text-gray-600  dark svelte-1fb58l0"
 			type="submit">Save</button
 		>
 	</div>

src/lib/utils/adapters/googledocs/@types/hast-util-classnames.d.ts

@@ -0,0 +1,4 @@
+declare module 'hast-util-classnames' {
+	function classnames(...node: any[]): any;
+	export = classnames;
+}

src/lib/utils/adapters/googledocs/@types/hast-util-heading-rank.d.ts

@@ -0,0 +1,6 @@
+import type { Element } from 'hast';
+
+declare module 'hast-util-heading-rank' {
+	function headingRank(node: Element): number;
+	export = isHeading;
+}

src/lib/utils/adapters/googledocs/@types/hast-util-heading.d.ts

@@ -0,0 +1,6 @@
+import type { Element } from 'hast';
+
+declare module 'hast-util-heading' {
+	function isHeading(node: Element): boolean;
+	export = isHeading;
+}

src/lib/utils/adapters/googledocs/@types/hast-util-to-text.d.ts

@@ -0,0 +1,6 @@
+import type { Element } from 'hast';
+
+declare module 'hast-util-to-text' {
+	function toText(node: Element): string;
+	export = toText;
+}

src/lib/utils/adapters/googledocs/@types/remark-rehype.d.ts

@@ -0,0 +1,4 @@
+declare module 'remark-rehype' {
+	const x: any;
+	export = x;
+}

src/lib/utils/adapters/googledocs/lisp is cool

@@ -0,0 +1,48 @@
+lisp is cool
+
+<div>
+  </h1>
+    jsdlakfjas
+  </h1>
+</div>
+
+s-expression
+(div
+  (h1
+    (jsdlakfsdja)))
+
+HTML is XML and XML is a treateeeeeeey
+
+express ANY code as a treee
+
+### LISP BIG MAN IDEAS
+
+<factorial>
+  <if>
+    <zerop>
+  </if>
+</factorial>
+
+"Lisp is worth learning for the profound enlightenment experience you will have when you finally get it; that experience will make you a better programmer for the rest of your days, even if you never actually use Lisp itself a lot."
+
+
+"Greenspun's Tenth Rule of Programming: any sufficiently complicated C or Fortran program contains an ad hoc informally-specified bug-ridden slow implementation of half of Common Lisp."
+
+- Philip Greenspun
+
+(defun factorial (x)
+   (if (zerop x)
+       1
+       (* x (factorial (- x 1)))))
+
+basically a list
+
+intermediate format
+from google doc -> our custom html
+- table of contents
+- other metadata
+
+
+Things we need:
+- Parse json into sexprs
+- Parse sexprs into the DOM

src/lib/utils/adapters/googledocs/parser.ts

@@ -0,0 +1,124 @@
+// Multiple H1s ARE semantic: https://www.youtube.com/watch?v=WsgrSxCmMbM
+import type { docs_v1 } from 'googleapis/build/src/apis/docs/v1';
+import { h } from 'hastscript';
+import { resolveTableOfContents } from './utils/resolveTableOfContents';
+import type { Element } from 'hast';
+import { resolveChildren } from './utils/resolveChildren';
+import { resolveHeadingLinks } from './utils/resolveHeadingLinks';
+import { resolveListElements } from './utils/resolveListElements';
+import { resolveList } from './utils/resolveList';
+import type { ResolveListProps } from './utils/resolveList';
+
+import toText from 'hast-util-to-text';
+import { Slugger } from '$lib/utils/slugger';
+
+const STYLE_TAG_MAP = new Map([
+	['NORMAL_TEXT', 'p'],
+	['SUBTITLE', 'p'],
+	['TITLE', 'h1'],
+	['HEADING_1', 'h1'],
+	['HEADING_2', 'h2'],
+	['HEADING_3', 'h3'],
+	['HEADING_4', 'h4'],
+	['HEADING_5', 'h5'],
+	['HEADING_6', 'h6']
+]);
+
+interface ParsedDocument {
+	documentRoot: Element;
+	tableOfContents: Element;
+	title: string | null | undefined;
+	documentId: string | null | undefined;
+	revisionId: string | null | undefined;
+}
+
+export function parseGoogleDoc(document: docs_v1.Schema$Document): ParsedDocument {
+	if (!document) throw new Error('Document is undefined.');
+
+	const slugger = new Slugger();
+	const {
+		body,
+		documentStyle,
+		lists,
+		documentId,
+		namedStyles,
+		revisionId,
+		title,
+		inlineObjects
+	} = document;
+	const listsMap = new Map(Object.entries(lists ?? {}));
+	const inlineObjectsMap = new Map(Object.entries(inlineObjects ?? {}));
+	const resolvedListsIds = new Map<string, boolean>();
+	const listElementsMap = resolveListElements(body?.content, listsMap);
+	let customTitle = null;
+
+	const _resolveChildren = (elements?: docs_v1.Schema$ParagraphElement[]): Element[] =>
+		resolveChildren(elements, inlineObjectsMap);
+	const _resolveHeadingLinks = (element: Element): Element => resolveHeadingLinks(element, slugger);
+	const _resolveList = (listElements: ResolveListProps | undefined): Element =>
+		resolveList(listElements, _resolveChildren);
+
+	const content: Element[] | undefined = body?.content
+		?.filter((structuralElement) => !!structuralElement.paragraph)
+		.map((structuralElement) => {
+			if (!structuralElement.paragraph?.bullet?.listId) return structuralElement;
+
+			const alreadyVisitedList = resolvedListsIds.get(structuralElement.paragraph.bullet.listId);
+
+			if (structuralElement.paragraph.bullet.listId && !alreadyVisitedList) {
+				resolvedListsIds.set(structuralElement.paragraph.bullet.listId, true);
+				return structuralElement;
+			}
+
+			return null;
+		})
+		.filter((structuralElement) => !!structuralElement?.paragraph)
+		.map((structuralElement) => {
+			if (!structuralElement?.paragraph) throw new Error('Empty paragraph');
+
+			const { paragraph } = structuralElement;
+			const paragraphStyleKey = paragraph?.paragraphStyle?.namedStyleType;
+			const paragraphTag = paragraphStyleKey ? STYLE_TAG_MAP.get(paragraphStyleKey) : 'div';
+			const inlineObjectElement = paragraph?.elements?.[0].inlineObjectElement;
+			const bullet = paragraph?.bullet;
+			const children = _resolveChildren(paragraph?.elements);
+
+			if (bullet && bullet.listId) {
+				const { listId } = bullet;
+				const listElements = listElementsMap.get(listId);
+
+				return _resolveList(listElements);
+			}
+
+			if (inlineObjectElement) {
+				return h('div', children);
+			}
+
+			if (paragraphStyleKey === 'TITLE') {
+				customTitle = toText(children as any)[0];
+				return h(
+					'h1',
+					{
+						class: 'font-serif-display tracking-tight text-6xl pb-10 heading-title',
+						dataType: 'title',
+						id: 'heading-title'
+					},
+					children
+				);
+			}
+
+			return h(paragraphTag ?? 'p', children);
+		})
+		.map(_resolveHeadingLinks);
+
+	const documentRoot = h('div', content);
+	const tableOfContents = h('div', [resolveTableOfContents(documentRoot) || '']);
+
+	return {
+		documentRoot,
+		tableOfContents,
+		title: customTitle ?? title,
+		documentId,
+		revisionId
+	};
+}