`;
+
+ const response = await sequelize.transaction((transaction) =>
+ documentImporter({
+ user,
+ mimeType: "text/html",
+ fileName: "notes.html",
+ content: html,
+ ctx: createContext({ user, transaction }),
+ })
+ );
+
+ const attachments = await Attachment.count({
+ where: {
+ teamId: user.teamId,
+ },
+ });
+ expect(attachments).toEqual(1);
+ expect(response.text).toContain("[the spec](/api/attachments.redirect?id=");
+ expect(response.text).not.toContain("data:application/pdf");
+ });
+
it("should not strip content after period in title", async () => {
const user = await buildUser();
const fileName = "01. test";
diff --git a/server/commands/documentImporter.ts b/server/commands/documentImporter.ts
index 48f5a94d61..456d191156 100644
--- a/server/commands/documentImporter.ts
+++ b/server/commands/documentImporter.ts
@@ -7,7 +7,7 @@ import { traceFunction } from "@server/logging/tracing";
import type { User } from "@server/models";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
import type { APIContext } from "@server/types";
-import { DocumentConverter } from "@server/utils/DocumentConverter";
+import { DocumentConverter } from "@server/converters/DocumentConverter";
import { InvalidRequestError } from "../errors";
type Props = {
@@ -64,6 +64,7 @@ async function documentImporter({
"mhtml",
"mht",
"eml",
+ "textpack",
...(mime.extensions[mimeType] ?? []),
];
const fileTitle = fileName.replace(
diff --git a/server/converters/BaseConverter.ts b/server/converters/BaseConverter.ts
new file mode 100644
index 0000000000..c0125e6c1c
--- /dev/null
+++ b/server/converters/BaseConverter.ts
@@ -0,0 +1,50 @@
+import yaml from "js-yaml";
+
+/**
+ * Base class for the individual file format converters, holding the handling
+ * that is common to reading any incoming file.
+ */
+export abstract class BaseConverter {
+ /**
+ * Convert a Buffer to a string.
+ *
+ * @param content The content as a Buffer or string.
+ * @returns The content as a string.
+ */
+ protected static bufferToString(content: Buffer | string): string {
+ return typeof content === "string" ? content : content.toString("utf8");
+ }
+
+ /**
+ * Parse and convert frontmatter to a YAML codeblock.
+ *
+ * @param content The markdown content that may contain frontmatter.
+ * @returns The markdown content with frontmatter converted to a YAML codeblock.
+ */
+ protected static processFrontmatter(content: string): string {
+ // Frontmatter must start at the beginning of the document
+ const frontmatterRegex = /^---\n([\s\S]*?)\n---(?:\n|$)/;
+ const match = content.match(frontmatterRegex);
+
+ if (!match) {
+ return content;
+ }
+
+ const frontmatterContent = match[1];
+ const remainingContent = content.slice(match[0].length);
+
+ // Validate that the frontmatter is valid YAML
+ try {
+ yaml.load(frontmatterContent);
+ } catch {
+ // If it's not valid YAML, return content unchanged
+ return content;
+ }
+
+ // Convert frontmatter to a YAML codeblock
+ const codeBlockDelimiter = "```";
+ const yamlCodeblock = `${codeBlockDelimiter}yaml\n${frontmatterContent}\n${codeBlockDelimiter}\n\n`;
+
+ return yamlCodeblock + remainingContent;
+ }
+}
diff --git a/server/converters/CsvConverter.ts b/server/converters/CsvConverter.ts
new file mode 100644
index 0000000000..8e76601c83
--- /dev/null
+++ b/server/converters/CsvConverter.ts
@@ -0,0 +1,117 @@
+import { parse } from "@fast-csv/parse";
+import { escapeRegExp } from "es-toolkit/compat";
+import { FileImportError } from "@server/errors";
+import { BaseConverter } from "./BaseConverter";
+
+/**
+ * Converts delimiter separated values (CSV, TSV) to a markdown table.
+ */
+export class CsvConverter extends BaseConverter {
+ /**
+ * Convert a CSV file to a markdown table.
+ *
+ * @param content The CSV file content.
+ * @returns A markdown table representation.
+ */
+ public static async toMarkdown(content: Buffer | string): Promise {
+ return new Promise((resolve, reject) => {
+ const text = this.bufferToString(content).trim();
+ const textLines = text.split("\n");
+
+ // Find the first non-empty line to determine the delimiter
+ const firstNonEmptyLine =
+ textLines.find((line) => line.trim().length > 0) || "";
+
+ // Determine the separator used in the CSV file based on number of occurrences of each separator on first line
+ const delimiter = [";", ",", "\t"].reduce(
+ (acc, separator) => {
+ const count = (
+ firstNonEmptyLine.match(new RegExp(escapeRegExp(separator), "g")) ||
+ []
+ ).length;
+ return count > acc.count ? { count, separator } : acc;
+ },
+ { count: 0, separator: "," }
+ ).separator;
+
+ const lines: string[][] = [];
+ const stream = parse({ delimiter })
+ .on("error", (error) => {
+ reject(
+ FileImportError(`There was an error parsing the CSV file: ${error}`)
+ );
+ })
+ .on("data", (row) => lines.push(row))
+ .on("end", () => {
+ // Filter out completely empty rows
+ const nonEmptyLines = lines.filter((row) =>
+ row.some((cell) => cell.trim() !== "")
+ );
+
+ if (nonEmptyLines.length === 0) {
+ resolve("");
+ return;
+ }
+
+ // Check if all rows have a trailing empty cell (trailing comma artifact)
+ // Only trim if ALL non-empty rows end with an empty cell
+ let trimmedLines = nonEmptyLines;
+ while (
+ trimmedLines.length > 0 &&
+ trimmedLines.every(
+ (row) => row.length > 0 && row[row.length - 1].trim() === ""
+ )
+ ) {
+ trimmedLines = trimmedLines.map((row) => row.slice(0, -1));
+ }
+
+ // Find the most common column count
+ const columnCounts = new Map();
+ for (const row of trimmedLines) {
+ if (row.length > 0) {
+ columnCounts.set(
+ row.length,
+ (columnCounts.get(row.length) || 0) + 1
+ );
+ }
+ }
+
+ // Get the column count that appears most frequently
+ let expectedColumns = 0;
+ let maxFrequency = 0;
+ for (const [count, frequency] of columnCounts) {
+ if (frequency > maxFrequency) {
+ maxFrequency = frequency;
+ expectedColumns = count;
+ }
+ }
+
+ // Find the first row with the expected column count (this is the header)
+ const headerIndex = trimmedLines.findIndex(
+ (row) => row.length === expectedColumns
+ );
+ if (headerIndex === -1) {
+ resolve("");
+ return;
+ }
+
+ const headers = trimmedLines[headerIndex];
+ const dataRows = trimmedLines
+ .slice(headerIndex + 1)
+ .filter((row) => row.length === expectedColumns);
+
+ const table = dataRows
+ .map((cells) => `| ${cells.join(" | ")} |`)
+ .join("\n");
+
+ const headerLine = `| ${headers.join(" | ")} |`;
+ const separatorLine = `| ${headers.map(() => "---").join(" | ")} |`;
+
+ resolve(`${headerLine}\n${separatorLine}\n${table}\n`);
+ });
+
+ stream.write(text);
+ stream.end();
+ });
+ }
+}
diff --git a/server/utils/DocumentConverter.test.ts b/server/converters/DocumentConverter.test.ts
similarity index 68%
rename from server/utils/DocumentConverter.test.ts
rename to server/converters/DocumentConverter.test.ts
index f8919a01ce..473fe53222 100644
--- a/server/utils/DocumentConverter.test.ts
+++ b/server/converters/DocumentConverter.test.ts
@@ -1,10 +1,29 @@
import path from "node:path";
import fs from "fs-extra";
+import { buildZip } from "@server/test/support";
import { DocumentConverter } from "./DocumentConverter";
const fixture = (fileName: string) =>
fs.readFile(path.resolve(__dirname, "..", "test", "fixtures", fileName));
+/** A 1x1 transparent PNG, small enough to embed in a test bundle. */
+const PNG_PIXEL = Buffer.from(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==",
+ "base64"
+);
+
+/** A TextBundle as an app would write it: wrapper folder, metadata and assets. */
+const BASIC_BUNDLE = {
+ "Note.textbundle/info.json": JSON.stringify({
+ version: 2,
+ type: "net.daringfireball.markdown",
+ transient: false,
+ }),
+ "Note.textbundle/text.markdown":
+ "# My Note\n\nHello world!\n\n\n",
+ "Note.textbundle/assets/image.png": PNG_PIXEL,
+};
+
describe("DocumentConverter", () => {
describe("convert", () => {
describe("csv", () => {
@@ -176,6 +195,43 @@ John,25`;
expect(result.text).not.toMatch(/^🚀/);
});
+ it("should extract emoji leading the title", async () => {
+ const html = "
🚀 My Title
Content here
";
+ const result = await DocumentConverter.convert(
+ html,
+ "test.html",
+ "text/html"
+ );
+
+ expect(result.icon).toEqual("🚀");
+ expect(result.title).toEqual("My Title");
+ });
+
+ it("should leave the body alone when the title supplied an emoji", async () => {
+ const html = "
🚀 My Title
🎉 Content here
";
+ const result = await DocumentConverter.convert(
+ html,
+ "test.html",
+ "text/html"
+ );
+
+ expect(result.icon).toEqual("🚀");
+ expect(result.title).toEqual("My Title");
+ expect(result.text).toContain("🎉 Content here");
+ });
+
+ it("should not treat an emoji later in the title as an icon", async () => {
+ const html = "
My Title 🚀
Content here
";
+ const result = await DocumentConverter.convert(
+ html,
+ "test.html",
+ "text/html"
+ );
+
+ expect(result.icon).toBeUndefined();
+ expect(result.title).toEqual("My Title 🚀");
+ });
+
it("should convert htm when the mime type is not recognized", async () => {
const html = "
My Title
Content here
";
const result = await DocumentConverter.convert(html, "test.HTM", "");
@@ -437,6 +493,260 @@ Content`;
expect(result.text).not.toContain("```yaml");
});
});
+
+ describe("textpack", () => {
+ it("should convert a wrapped TextBundle to markdown and embed its assets", async () => {
+ const content = await buildZip(BASIC_BUNDLE);
+
+ const result = await DocumentConverter.convert(
+ content,
+ "My Note.textpack",
+ "application/octet-stream"
+ );
+
+ expect(result.title).toEqual("My Note");
+ expect(result.text).toContain("Hello world!");
+ expect(result.text).toContain("data:image/png;base64,");
+ expect(result.text).not.toContain("assets/image.png");
+ });
+
+ it("should lift an emoji leading the title into the icon", async () => {
+ const content = await buildZip({
+ "Note.textbundle/text.markdown": "# 🚀 My Note\n\nHello world!\n",
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "My Note.textpack",
+ "application/octet-stream"
+ );
+
+ expect(result.icon).toEqual("🚀");
+ expect(result.title).toEqual("My Note");
+ expect(result.text).toContain("Hello world!");
+ });
+
+ it("should convert a flat TextBundle (no wrapper folder, text.md) to markdown", async () => {
+ const content = await buildZip({
+ "info.json": JSON.stringify({
+ version: 1,
+ type: "net.daringfireball.markdown",
+ }),
+ "text.md": "# Flat Note\n\nSee  here.\n",
+ "assets/photo.jpg": PNG_PIXEL,
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Flat.textpack",
+ "application/octet-stream"
+ );
+
+ expect(result.title).toEqual("Flat Note");
+ expect(result.text).toContain("data:image/jpeg;base64,");
+ expect(result.text).not.toContain("assets/photo.jpg");
+ });
+
+ it("should tolerate a bundle with no info.json", async () => {
+ const content = await buildZip({
+ "Plain.textbundle/text.txt":
+ "Just plain text content, no heading here.\n",
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Plain.textpack",
+ "application/octet-stream"
+ );
+
+ expect(result.text).toContain(
+ "Just plain text content, no heading here."
+ );
+ });
+
+ it("should fall back to extension-based routing when mimetype is unrecognized", async () => {
+ const content = await buildZip(BASIC_BUNDLE);
+
+ const result = await DocumentConverter.convert(
+ content,
+ "My Note.textpack",
+ "application/zip"
+ );
+
+ expect(result.title).toEqual("My Note");
+ });
+
+ it("should skip path-traversal entries and still import the legitimate text file", async () => {
+ const content = await fixture("textbundle-traversal.textpack");
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ "application/octet-stream"
+ );
+
+ expect(result.title).toEqual("Safe");
+ expect(result.text).toContain("Content here.");
+ });
+
+ it("should reject a bundle with no recognizable text file", async () => {
+ const content = await buildZip({
+ "Empty.textbundle/info.json": JSON.stringify({
+ version: 2,
+ type: "net.daringfireball.markdown",
+ }),
+ "Empty.textbundle/assets/image.png": PNG_PIXEL,
+ });
+
+ await expect(
+ DocumentConverter.convert(
+ content,
+ "Empty.textpack",
+ "application/octet-stream"
+ )
+ ).rejects.toThrow(
+ "TextPack file does not contain a recognizable text file"
+ );
+ });
+
+ it("should reject an asset larger than the maximum, however well it compresses", async () => {
+ const content = await buildZip({
+ "text.markdown": "# Note\n\n",
+ // Compresses to almost nothing, so the archive stays small while the
+ // asset is far past what may be held in memory as a data URI.
+ "assets/big.png": Buffer.alloc(20 * 1024 * 1024),
+ });
+
+ await expect(
+ DocumentConverter.convert(content, "Note.textpack", "")
+ ).rejects.toThrow("too large");
+ });
+
+ it("should reject a bundle with an absurd number of entries", async () => {
+ const files: Record = {
+ "text.markdown": "# Note",
+ };
+ for (let i = 0; i < 2001; i++) {
+ files[`assets/file-${i}.txt`] = "x";
+ }
+ const content = await buildZip(files);
+
+ await expect(
+ DocumentConverter.convert(content, "Note.textpack", "")
+ ).rejects.toThrow("TextPack file contains too many entries");
+ });
+
+ it("should read a text file with an extension the spec leaves open", async () => {
+ const content = await buildZip({
+ "Note.textbundle/info.json": JSON.stringify({
+ version: 2,
+ type: "net.daringfireball.markdown",
+ }),
+ "Note.textbundle/text.fountain": "# Screenplay\n\nFade in.",
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ ""
+ );
+
+ expect(result.title).toEqual("Screenplay");
+ expect(result.text).toContain("Fade in.");
+ });
+
+ it("should reject a bundle whose info.json declares a format we cannot read", async () => {
+ const content = await buildZip({
+ "Note.textbundle/info.json": JSON.stringify({
+ version: 2,
+ type: "public.rtf",
+ }),
+ "Note.textbundle/text.rtf": "{\\rtf1 hello}",
+ });
+
+ await expect(
+ DocumentConverter.convert(content, "Note.textpack", "")
+ ).rejects.toThrow("public.rtf");
+ });
+
+ it("should reject an unreadable text extension when info.json is absent", async () => {
+ const content = await buildZip({
+ "text.rtf": "{\\rtf1 hello}",
+ });
+
+ await expect(
+ DocumentConverter.convert(content, "Note.textpack", "")
+ ).rejects.toThrow(".rtf");
+ });
+
+ it("should embed an asset referenced by a link carrying a title", async () => {
+ const content = await buildZip({
+ "text.markdown": '# Note\n\n',
+ "assets/img.png": PNG_PIXEL,
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ ""
+ );
+
+ expect(result.text).toContain("data:image/png;base64,");
+ expect(result.text).not.toContain("assets/img.png");
+ });
+
+ it("should embed an asset whose destination is wrapped in angle brackets", async () => {
+ const content = await buildZip({
+ "text.markdown": "# Note\n\n![a photo]()",
+ "assets/my photo.png": PNG_PIXEL,
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ ""
+ );
+
+ expect(result.text).toContain("data:image/png;base64,");
+ expect(result.text).not.toContain("my photo.png");
+ });
+
+ it("should embed an asset when the assets folder is not lowercased", async () => {
+ const content = await buildZip({
+ "text.markdown": "# Note\n\n",
+ "Assets/photo.png": PNG_PIXEL,
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ ""
+ );
+
+ expect(result.text).toContain("data:image/png;base64,");
+ });
+
+ it("should leave an asset markdown-it cannot accept as a data URI alone", async () => {
+ const content = await buildZip({
+ "text.markdown":
+ "# Note\n\n[the spec](assets/spec.pdf)\n\n",
+ "assets/spec.pdf": Buffer.from("%PDF-1.4\n"),
+ "assets/logo.svg": Buffer.from(""),
+ });
+
+ const result = await DocumentConverter.convert(
+ content,
+ "Note.textpack",
+ ""
+ );
+
+ // Inlining these would leave the base64 in the document as literal
+ // text, since markdown-it rejects the destination.
+ expect(result.text).not.toContain("base64");
+ expect(result.text).toContain("assets/spec.pdf");
+ expect(result.text).toContain("assets/logo.svg");
+ });
+ });
});
describe("htmlToProsemirror", () => {
diff --git a/server/converters/DocumentConverter.ts b/server/converters/DocumentConverter.ts
new file mode 100644
index 0000000000..e21ee0a45e
--- /dev/null
+++ b/server/converters/DocumentConverter.ts
@@ -0,0 +1,268 @@
+import type { Node } from "prosemirror-model";
+import { DOMParser as ProsemirrorDOMParser } from "prosemirror-model";
+import { splitLeadingEmoji } from "@shared/utils/parseTitle";
+import { schema, serializer } from "@server/editor";
+import { FileImportError } from "@server/errors";
+import { trace } from "@server/logging/tracing";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { BaseConverter } from "./BaseConverter";
+import { HtmlPreprocessor } from "./HtmlPreprocessor";
+
+export interface ConvertResult {
+ /** The document content as markdown text. */
+ text: string;
+ /** The document content as Prosemirror. */
+ doc: Node;
+ /** The extracted title (from H1 heading if present). */
+ title: string;
+ /** The extracted emoji/icon from start of document. */
+ icon?: string;
+}
+
+/**
+ * Converts incoming files of various formats to structured documents. Each
+ * format's implementation lives in its own converter alongside this file; this
+ * class owns the pipeline they feed into — route, parse, lift the title and
+ * icon, serialize.
+ */
+@trace()
+export class DocumentConverter extends BaseConverter {
+ /**
+ * Convert an incoming file to a structured document result.
+ *
+ * @param content The content of the file.
+ * @param fileName The name of the file, including extension.
+ * @param mimeType The mime type of the file.
+ * @param options Conversion options.
+ * @param options.extractTitle Whether a leading H1 heading should be lifted
+ * out as the document title and removed from the body. Defaults to true;
+ * set false for sources where the filename is authoritative and the first
+ * heading must remain part of the content (e.g. Slab).
+ * @returns The converted document with text, data, title, and icon.
+ */
+ public static async convert(
+ content: Buffer | string,
+ fileName: string,
+ mimeType: string,
+ options: { extractTitle?: boolean } = {}
+ ): Promise {
+ const { extractTitle = true } = options;
+ let doc: Node;
+
+ // Route to appropriate conversion method
+ const html = await this.convertToHtml(content, fileName, mimeType);
+ if (html !== undefined) {
+ doc = await this.htmlToProsemirror(html);
+ } else {
+ const markdown = await this.convertToMarkdown(
+ content,
+ fileName,
+ mimeType
+ );
+ doc = ProsemirrorHelper.toProsemirror(markdown);
+ }
+
+ // Extract title from first H1 heading
+ let title = "";
+ let icon: string | undefined;
+ if (extractTitle) {
+ const headings = ProsemirrorHelper.getHeadings(doc);
+ if (headings.length > 0 && headings[0].level === 1) {
+ // An emoji leading the title becomes the document's icon, matching how
+ // a title is written back out on export.
+ const { emoji, rest } = splitLeadingEmoji(headings[0].title);
+ title = rest;
+ icon = emoji;
+ doc = ProsemirrorHelper.removeFirstHeading(doc);
+ }
+ }
+
+ // Only when the title supplied no icon is the body's leading emoji taken,
+ // so that a document without a heading can still lead with one.
+ if (!icon) {
+ const { emoji, doc: docWithoutEmoji } =
+ ProsemirrorHelper.extractEmojiFromStart(doc);
+ icon = emoji;
+ doc = docWithoutEmoji;
+ }
+
+ // Serialize to markdown and trim whitespace
+ const text = serializer.serialize(doc).trim();
+
+ return {
+ text,
+ doc,
+ title,
+ icon,
+ };
+ }
+
+ /**
+ * Convert HTML content directly to a Prosemirror document node.
+ *
+ * @param content The HTML content as a string or Buffer.
+ * @returns A Prosemirror Node representing the document.
+ */
+ public static async htmlToProsemirror(
+ content: Buffer | string
+ ): Promise {
+ content = this.bufferToString(content);
+
+ // Loaded lazily to keep jsdom off the startup path — only HTML imports need it.
+ const { JSDOM } = await import("jsdom");
+ const dom = new JSDOM(content);
+ const document = dom.window.document;
+
+ // Remove problematic elements before parsing
+ const elementsToRemove = document.querySelectorAll(
+ "script, style, title, head, meta, link"
+ );
+ elementsToRemove.forEach((el) => el.remove());
+
+ // Preprocess the DOM to handle edge cases
+ HtmlPreprocessor.preprocess(document);
+
+ // Patch global environment for Prosemirror DOMParser
+ const cleanup = ProsemirrorHelper.patchGlobalEnv(dom.window);
+
+ try {
+ const domParser = ProsemirrorDOMParser.fromSchema(schema);
+ return domParser.parse(document.body);
+ } finally {
+ cleanup();
+ try {
+ dom.window.close();
+ } catch (_err) {
+ // Best effort, closing the window releases its timers and resources.
+ }
+ }
+ }
+
+ /**
+ * Attempts to convert content to HTML for formats that support it.
+ * Returns undefined for formats that should be parsed as markdown directly.
+ *
+ * @param content The content of the file.
+ * @param fileName The name of the file, including extension.
+ * @param mimeType The mime type of the file.
+ * @returns HTML string if convertible, undefined otherwise.
+ */
+ private static async convertToHtml(
+ content: Buffer | string,
+ fileName: string,
+ mimeType: string
+ ): Promise {
+ const extension = fileName.split(".").pop()?.toLowerCase();
+
+ // First try to convert based on the mime type
+ switch (mimeType) {
+ case "text/html":
+ return this.bufferToString(content);
+ case "application/msword":
+ return (await this.mimeArchive()).confluenceToHtml(content);
+ case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+ return (await this.docx()).toHtml(content);
+ // Browsers report MHTML ("Save page as → Webpage, Single File") and .eml
+ // inconsistently across these three mime types, so the extension is
+ // used as a tie-breaker between the two archive kinds below.
+ case "multipart/related":
+ case "application/x-mimearchive": {
+ const converter = await this.mimeArchive();
+ return extension === "eml"
+ ? converter.emlToHtml(content)
+ : converter.mhtmlToHtml(content);
+ }
+ case "message/rfc822": {
+ const converter = await this.mimeArchive();
+ return extension === "mhtml" || extension === "mht"
+ ? converter.mhtmlToHtml(content)
+ : converter.emlToHtml(content);
+ }
+ default:
+ break;
+ }
+
+ // Try to convert based on the file extension
+ switch (extension) {
+ case "htm":
+ case "html":
+ return this.bufferToString(content);
+ case "docx":
+ return (await this.docx()).toHtml(content);
+ case "mhtml":
+ case "mht":
+ return (await this.mimeArchive()).mhtmlToHtml(content);
+ case "eml":
+ return (await this.mimeArchive()).emlToHtml(content);
+ default:
+ return undefined;
+ }
+ }
+
+ /**
+ * Converts content to markdown for text-based formats.
+ *
+ * @param content The content of the file.
+ * @param fileName The name of the file, including extension.
+ * @param mimeType The mime type of the file.
+ * @returns Markdown string.
+ */
+ private static async convertToMarkdown(
+ content: Buffer | string,
+ fileName: string,
+ mimeType: string
+ ): Promise {
+ let markdown: string;
+
+ switch (mimeType) {
+ case "text/plain":
+ case "text/markdown":
+ markdown = this.bufferToString(content);
+ break;
+ case "text/csv":
+ case "text/tab-separated-values":
+ return (await this.csv()).toMarkdown(content);
+ default: {
+ const extension = fileName.split(".").pop()?.toLowerCase();
+ switch (extension) {
+ case "md":
+ case "markdown":
+ case "txt":
+ markdown = this.bufferToString(content);
+ break;
+ case "csv":
+ case "tsv":
+ return (await this.csv()).toMarkdown(content);
+ case "textpack":
+ return (await this.textPack()).toMarkdown(content);
+ default:
+ throw FileImportError(`File type ${mimeType} not supported`);
+ }
+ }
+ }
+
+ // Process frontmatter and convert it to a YAML codeblock
+ return this.processFrontmatter(markdown);
+ }
+
+ /**
+ * Converters are loaded on demand so that the dependencies of a format —
+ * jsdom, mammoth, mailparser, fast-csv, yauzl — stay off the server startup
+ * path, and are only paid for by an import that actually needs them.
+ */
+ private static async docx() {
+ return (await import("./DocxConverter")).DocxConverter;
+ }
+
+ private static async mimeArchive() {
+ return (await import("./MimeArchiveConverter")).MimeArchiveConverter;
+ }
+
+ private static async csv() {
+ return (await import("./CsvConverter")).CsvConverter;
+ }
+
+ private static async textPack() {
+ return (await import("./TextPackConverter")).TextPackConverter;
+ }
+}
diff --git a/server/converters/DocxConverter.ts b/server/converters/DocxConverter.ts
new file mode 100644
index 0000000000..d125709cd9
--- /dev/null
+++ b/server/converters/DocxConverter.ts
@@ -0,0 +1,27 @@
+import mammoth from "mammoth";
+import { FileImportError } from "@server/errors";
+import { traceFunction } from "@server/logging/tracing";
+import { BaseConverter } from "./BaseConverter";
+
+/**
+ * Converts Word (.docx) files to HTML.
+ */
+export class DocxConverter extends BaseConverter {
+ /**
+ * Convert a docx file to HTML using mammoth.
+ *
+ * @param content The docx file content as a Buffer.
+ * @returns The HTML representation of the document.
+ */
+ public static async toHtml(content: Buffer | string): Promise {
+ if (content instanceof Buffer) {
+ const { value } = await traceFunction({ spanName: "convertToHtml" })(
+ mammoth.convertToHtml
+ )({
+ buffer: content,
+ });
+ return value;
+ }
+ throw FileImportError("Unsupported Word file");
+ }
+}
diff --git a/server/converters/HtmlPreprocessor.ts b/server/converters/HtmlPreprocessor.ts
new file mode 100644
index 0000000000..85e53d8fc0
--- /dev/null
+++ b/server/converters/HtmlPreprocessor.ts
@@ -0,0 +1,162 @@
+/**
+ * Cleans up an HTML DOM before it is parsed into a Prosemirror document,
+ * handling the quirks of the tools that produce imported HTML.
+ */
+export class HtmlPreprocessor {
+ /**
+ * Preprocesses HTML DOM before Prosemirror parsing to cleanup
+ * images and other elements.
+ *
+ * @param document The DOM document to preprocess.
+ */
+ public static preprocess(document: Document): void {
+ // Handle images: filter emoticons, remove Jira icons, apply Confluence sizing
+ const images = document.querySelectorAll("img");
+ images.forEach((img) => {
+ const className = img.className || "";
+
+ // Skip emoticon images (they'll be dropped)
+ if (className.includes("emoticon")) {
+ img.remove();
+ return;
+ }
+
+ // Remove Jira icon images
+ if (
+ className === "icon" &&
+ img.parentElement?.className.includes("jira-issue-key")
+ ) {
+ img.remove();
+ return;
+ }
+
+ // Handle Confluence image sizing: data-width/data-height → width/height
+ const dataWidth = img.getAttribute("data-width");
+ const dataHeight = img.getAttribute("data-height");
+ const width = img.getAttribute("width");
+
+ if (dataWidth && dataHeight && width) {
+ const ratio = parseInt(dataWidth) / parseInt(width);
+ const calculatedHeight = Math.round(parseInt(dataHeight) / ratio);
+ img.setAttribute("height", String(calculatedHeight));
+ }
+
+ // Extract dimensions from data URI images that lack width/height
+ // (e.g. images embedded by mammoth during docx import).
+ // Only decode a small prefix of the base64 data — headers for all
+ // supported formats live within the first 64 KB of the file.
+ if (!img.getAttribute("width") && !img.getAttribute("height")) {
+ const src = img.getAttribute("src") || "";
+ if (src.startsWith("data:") && src.includes(";base64,")) {
+ const base64Start = src.indexOf(";base64,") + 8;
+ // 4 base64 chars → 3 bytes; decode at most ~64 KB of image data.
+ const maxBase64Chars = Math.ceil(65536 / 3) * 4;
+ const base64Prefix = src.slice(
+ base64Start,
+ base64Start + maxBase64Chars
+ );
+ const dimensions = this.getImageDimensions(
+ Buffer.from(base64Prefix, "base64")
+ );
+ if (dimensions) {
+ img.setAttribute("width", String(dimensions.width));
+ img.setAttribute("height", String(dimensions.height));
+ }
+ }
+ }
+ });
+ }
+ /**
+ * Parse image dimensions from a binary buffer. Supports PNG, JPEG, and GIF.
+ *
+ * @param buffer The image data.
+ * @returns The width and height if parseable, otherwise undefined.
+ */
+ private static getImageDimensions(
+ buffer: Buffer
+ ): { width: number; height: number } | undefined {
+ try {
+ // PNG: signature + IHDR chunk
+ if (
+ buffer.length >= 24 &&
+ buffer[0] === 0x89 &&
+ buffer[1] === 0x50 &&
+ buffer[2] === 0x4e &&
+ buffer[3] === 0x47
+ ) {
+ return {
+ width: buffer.readUInt32BE(16),
+ height: buffer.readUInt32BE(20),
+ };
+ }
+
+ // GIF: signature + logical screen descriptor
+ if (
+ buffer.length >= 10 &&
+ buffer[0] === 0x47 &&
+ buffer[1] === 0x49 &&
+ buffer[2] === 0x46
+ ) {
+ return {
+ width: buffer.readUInt16LE(6),
+ height: buffer.readUInt16LE(8),
+ };
+ }
+
+ // JPEG: scan for SOF marker (cap at 64 KB to bound work)
+ if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xd8) {
+ const scanLimit = Math.min(buffer.length, 65536);
+ let offset = 2;
+ while (offset + 1 < scanLimit) {
+ if (buffer[offset] !== 0xff) {
+ offset++;
+ continue;
+ }
+ const marker = buffer[offset + 1];
+ offset += 2;
+
+ // Standalone markers without a payload
+ if (
+ marker === 0x00 ||
+ marker === 0x01 ||
+ (marker >= 0xd0 && marker <= 0xd9)
+ ) {
+ continue;
+ }
+
+ if (offset + 2 > scanLimit) {
+ break;
+ }
+ const segmentLength = buffer.readUInt16BE(offset);
+
+ // SOF markers contain the frame dimensions — check before
+ // the advance guard since this returns immediately.
+ if (
+ (marker >= 0xc0 && marker <= 0xc3) ||
+ (marker >= 0xc5 && marker <= 0xc7) ||
+ (marker >= 0xc9 && marker <= 0xcb) ||
+ (marker >= 0xcd && marker <= 0xcf)
+ ) {
+ if (offset + 7 <= buffer.length) {
+ return {
+ height: buffer.readUInt16BE(offset + 3),
+ width: buffer.readUInt16BE(offset + 5),
+ };
+ }
+ break;
+ }
+
+ // Length includes itself and must be >= 2; bail on malformed data.
+ if (segmentLength < 2 || offset + segmentLength > buffer.length) {
+ break;
+ }
+
+ offset += segmentLength;
+ }
+ }
+ } catch {
+ // Return undefined if parsing fails
+ }
+ return undefined;
+ }
+}
diff --git a/server/converters/MimeArchiveConverter.ts b/server/converters/MimeArchiveConverter.ts
new file mode 100644
index 0000000000..2d5d406ed2
--- /dev/null
+++ b/server/converters/MimeArchiveConverter.ts
@@ -0,0 +1,201 @@
+import { escape } from "es-toolkit/compat";
+import type { Attachment as MailAttachment, ParsedMail } from "mailparser";
+import { simpleParser } from "mailparser";
+import { FileImportError } from "@server/errors";
+import { BaseConverter } from "./BaseConverter";
+
+/**
+ * Converts MIME archives to HTML. Confluence's "Word" export, MHTML pages
+ * (Chrome/Edge "Save page as → Webpage, Single File") and .eml messages are
+ * all, structurally, multi-part email messages, so one parser serves them all.
+ */
+export class MimeArchiveConverter extends BaseConverter {
+ /** Maximum number of MIME archive parts that will be inlined as data URIs. */
+ private static readonly MIME_ARCHIVE_MAX_INLINE_PARTS = 100;
+
+ /** Maximum size of a single MIME archive part that will be inlined as a data URI. */
+ private static readonly MIME_ARCHIVE_MAX_PART_BYTES = 15 * 1024 * 1024;
+
+ /** Maximum combined size of MIME archive parts that will be inlined as data URIs. */
+ private static readonly MIME_ARCHIVE_MAX_TOTAL_BYTES = 50 * 1024 * 1024;
+
+ /** Matches a well-formed mime type, e.g. `image/png`. */
+ private static readonly MIME_TYPE_REGEX = /^[\w.+-]+\/[\w.+-]+$/;
+
+ /**
+ * Convert a Confluence Word export to HTML.
+ *
+ * @param content The Confluence Word export content.
+ * @returns The HTML representation of the document.
+ */
+ public static async confluenceToHtml(
+ content: Buffer | string
+ ): Promise {
+ const text = this.bufferToString(content);
+
+ // We're only supporting the output from Confluence here, regular Word documents should call
+ // into the docxToHtml importer. See: https://jira.atlassian.com/browse/CONFSERVER-38237
+ if (!text.includes("Content-Type: multipart/related")) {
+ throw FileImportError("Unsupported Word file");
+ }
+
+ const { html } = await this.parseMimeArchive(text, {
+ emptyMessage: "Unsupported Word file (No content found)",
+ });
+
+ return html;
+ }
+
+ /**
+ * Convert an MHTML file (e.g. Chrome/Edge "Save page as → Webpage, Single
+ * File") to HTML.
+ *
+ * @param content The MHTML file content.
+ * @returns The HTML representation of the document.
+ */
+ public static async mhtmlToHtml(content: Buffer | string): Promise {
+ const { html } = await this.parseMimeArchive(this.bufferToString(content), {
+ emptyMessage: "Unsupported MHTML file (No content found)",
+ });
+
+ return html;
+ }
+
+ /**
+ * Convert an .eml email message to HTML. The `Subject` header, if present,
+ * is inserted as a leading H1 so it is picked up as the document title by
+ * the same leading-heading extraction used for every other format.
+ *
+ * @param content The .eml file content.
+ * @returns The HTML representation of the message.
+ */
+ public static async emlToHtml(content: Buffer | string): Promise {
+ const { html, subject } = await this.parseMimeArchive(
+ this.bufferToString(content),
+ {
+ allowTextFallback: true,
+ emptyMessage: "Unsupported email file (No content found)",
+ }
+ );
+
+ return subject ? `
${escape(subject)}
\n${html}` : html;
+ }
+
+ /**
+ * Parse a MIME archive (`multipart/related`, `message/rfc822`, or similar)
+ * with mailparser and resolve its HTML body, inlining referenced parts as
+ * data URIs.
+ *
+ * Confluence's "Word" export and MHTML pages (Chrome/Edge "Save page as →
+ * Webpage, Single File") are both, structurally, multi-part email messages,
+ * so the same parser and inlining logic serves all of them.
+ *
+ * @param content The MIME archive content as a string.
+ * @param options.allowTextFallback Whether to fall back to the plain text
+ * body (rendered as HTML) when no HTML part is present. Used for .eml,
+ * where a text-only email is common; MHTML/Word exports always have HTML.
+ * @param options.emptyMessage The error message to throw when no usable
+ * body is found.
+ * @returns The resolved HTML body and, if present, the message subject.
+ */
+ private static async parseMimeArchive(
+ content: string,
+ options: { allowTextFallback?: boolean; emptyMessage: string }
+ ): Promise<{ html: string; subject?: string }> {
+ // Confluence "Word" documents, MHTML pages, and .eml files are all just multi-part
+ // email messages, so we can use mailparser to parse the content. `keepCidLinks` is
+ // set so we can apply our own bounds when inlining referenced parts below, rather
+ // than mailparser inlining every matching part unconditionally.
+ let parsed: ParsedMail;
+ try {
+ parsed = await simpleParser(content, { keepCidLinks: true });
+ } catch (_err) {
+ throw FileImportError(options.emptyMessage);
+ }
+
+ let html = parsed.html || undefined;
+ if (!html && options.allowTextFallback) {
+ html = parsed.textAsHtml || undefined;
+ }
+ if (!html) {
+ throw FileImportError(options.emptyMessage);
+ }
+
+ return {
+ html: this.inlineMimeArchiveParts(html, parsed.attachments),
+ subject: parsed.subject,
+ };
+ }
+
+ /**
+ * Replace references to MIME archive parts within HTML with data URIs, so
+ * the resulting document is self-contained. Parts are referenced either by
+ * `Content-Location` (used by MHTML and Confluence exports) or by
+ * `Content-ID` as a `cid:` URL (used by email). Inlining is bounded so a
+ * hostile or merely enormous archive cannot blow up memory or the
+ * resulting document.
+ *
+ * @param html The HTML body that may reference archive parts.
+ * @param attachments The parsed MIME archive parts.
+ * @returns The HTML with resolvable references replaced by data URIs.
+ */
+ private static inlineMimeArchiveParts(
+ html: string,
+ attachments: MailAttachment[]
+ ): string {
+ let inlinedParts = 0;
+ let inlinedBytes = 0;
+
+ for (const attachment of attachments) {
+ if (inlinedParts >= this.MIME_ARCHIVE_MAX_INLINE_PARTS) {
+ break;
+ }
+ if (
+ attachment.content.length > this.MIME_ARCHIVE_MAX_PART_BYTES ||
+ inlinedBytes + attachment.content.length >
+ this.MIME_ARCHIVE_MAX_TOTAL_BYTES
+ ) {
+ continue;
+ }
+
+ const references = new Set();
+ const contentLocation = attachment.headers.get("content-location") as
+ | string
+ | undefined;
+ if (contentLocation) {
+ references.add(contentLocation);
+ const basename = contentLocation.split("/").pop();
+ if (basename) {
+ references.add(basename);
+ }
+ }
+ if (attachment.cid) {
+ references.add(`cid:${attachment.cid}`);
+ }
+
+ // The content type comes from the archive's own headers, so anything that
+ // isn't a well-formed mime type is discarded rather than interpolated.
+ const contentType = this.MIME_TYPE_REGEX.test(attachment.contentType)
+ ? attachment.contentType
+ : "application/octet-stream";
+
+ let replaced = false;
+ for (const reference of references) {
+ if (html.includes(reference)) {
+ const dataUri = `data:${contentType};base64,${attachment.content.toString(
+ "base64"
+ )}`;
+ html = html.split(reference).join(dataUri);
+ replaced = true;
+ }
+ }
+
+ if (replaced) {
+ inlinedParts++;
+ inlinedBytes += attachment.content.length;
+ }
+ }
+
+ return html;
+ }
+}
diff --git a/server/converters/TextPackConverter.ts b/server/converters/TextPackConverter.ts
new file mode 100644
index 0000000000..b03511c8a8
--- /dev/null
+++ b/server/converters/TextPackConverter.ts
@@ -0,0 +1,371 @@
+import mime from "mime-types";
+import { AttachmentPreset } from "@shared/types";
+import { replaceMarkdownLinks } from "@shared/utils/markdown";
+import { DocumentValidation } from "@shared/validations";
+import { FileImportError } from "@server/errors";
+import Logger from "@server/logging/Logger";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
+import TextBundleHelper from "@server/models/helpers/TextBundleHelper";
+import ZipHelper from "@server/utils/ZipHelper";
+import { BaseConverter } from "./BaseConverter";
+
+/** Matches a TextBundle's text entry, whose extension the spec leaves open. */
+const TEXTBUNDLE_TEXT_REGEX = /^text\.[^.]+$/;
+
+/**
+ * Extensions of a TextBundle text entry that can be read as markdown. The spec
+ * allows any extension and declares the real content type in info.json, so this
+ * is only consulted for bundles that omit a type.
+ */
+const TEXTBUNDLE_TEXT_EXTENSIONS = [
+ "markdown",
+ "markdn",
+ "mdown",
+ "mmd",
+ "md",
+ "text",
+ "txt",
+];
+
+/** UTIs from info.json whose content can be read as markdown or plain text. */
+const TEXTBUNDLE_TEXT_TYPES = [
+ "net.daringfireball.markdown",
+ "public.plain-text",
+ "public.utf8-plain-text",
+ "public.text",
+];
+
+/**
+ * Media types that can be inlined as a data URI. Markdown-it rejects a `data:`
+ * destination outside this set, and an asset it rejects would be left in the
+ * document as literal base64 text, so anything else keeps its original
+ * reference instead.
+ */
+const TEXTBUNDLE_EMBEDDABLE_MIME_TYPES = [
+ "image/gif",
+ "image/jpeg",
+ "image/png",
+ "image/webp",
+];
+
+/** Maximum number of entries a TextPack archive may contain. */
+const TEXTBUNDLE_MAX_ENTRIES = 2000;
+
+/** Maximum size of the bundle's info.json, in bytes. */
+const TEXTBUNDLE_MAX_INFO_SIZE = 1024 * 1024;
+
+/** Maximum size of the bundle's text file, in bytes. */
+const TEXTBUNDLE_MAX_TEXT_SIZE = DocumentValidation.maxStateLength;
+
+/**
+ * Ceiling on the size of a single asset inlined as a data URI, in bytes. The
+ * configured attachment limit bounds this too, but on its own it is not enough:
+ * it can be set arbitrarily high, whereas inlining holds the asset in memory
+ * several times over.
+ */
+const TEXTBUNDLE_MAX_ASSET_SIZE = 15 * 1024 * 1024;
+
+/** Maximum combined size of all assets, in bytes. */
+const TEXTBUNDLE_MAX_TOTAL_ASSETS_SIZE = 50 * 1024 * 1024;
+
+interface TextBundleEntry {
+ fileName: string;
+ isDirectory: boolean;
+}
+
+interface TextBundleRoot {
+ /** Path prefix of the bundle's contents within the zip, "" if not nested in a wrapper folder. */
+ root: string;
+ /** Full path of the text entry within the zip. */
+ textPath: string;
+ /** Full path of the bundle's info.json, if it has one. */
+ infoPath?: string;
+}
+
+/**
+ * Converts a TextPack — the zipped form of a TextBundle — to markdown.
+ *
+ * @see https://textbundle.org/spec/
+ */
+export class TextPackConverter extends BaseConverter {
+ /**
+ * Convert a TextPack (a zipped TextBundle) to markdown, embedding referenced
+ * assets as base64 data URIs so they survive as ordinary markdown images
+ * until {@link ProsemirrorHelper.replaceImagesWithAttachments} turns them
+ * into real attachments. Assets of a type that cannot be inlined keep their
+ * original reference.
+ *
+ * @param content The TextPack file content as a Buffer.
+ * @returns The bundle's text content as markdown, with assets embedded.
+ * @throws {FileImportError} if the archive is malformed, oversized, holds no
+ * text file, or declares a format that cannot be read as markdown.
+ */
+ public static async toMarkdown(content: Buffer | string): Promise {
+ if (!(content instanceof Buffer)) {
+ throw FileImportError("Unsupported TextPack file");
+ }
+
+ const entries: TextBundleEntry[] = [];
+
+ await ZipHelper.walk(content, (entry) => {
+ entries.push({
+ fileName: entry.fileName,
+ isDirectory: entry.isDirectory,
+ });
+ if (entries.length > TEXTBUNDLE_MAX_ENTRIES) {
+ throw FileImportError("TextPack file contains too many entries");
+ }
+ });
+
+ const bundle = this.findTextBundleRoot(entries);
+ if (!bundle) {
+ throw FileImportError(
+ "TextPack file does not contain a recognizable text file"
+ );
+ }
+
+ const rootPrefix = bundle.root ? `${bundle.root}/` : "";
+ // Compared case-insensitively: the spec only recommends lowercase names.
+ const assetPrefix =
+ `${rootPrefix}${TextBundleHelper.assetsDirectory}/`.toLowerCase();
+ // Keyed on the lowercased path for the same reason; TextBundle is designed
+ // for case-insensitive filesystems, so two assets differing only in case
+ // cannot coexist in a valid bundle.
+ const assets = new Map();
+ // Each asset becomes an attachment downstream, so anything the attachment
+ // pipeline would reject is not worth inlining as a data URI first.
+ const maxAssetSize = Math.min(
+ TEXTBUNDLE_MAX_ASSET_SIZE,
+ AttachmentHelper.presetToMaxUploadSize(
+ AttachmentPreset.DocumentAttachment
+ )
+ );
+ let markdown: string | undefined;
+ let info: string | undefined;
+ let totalAssetBytes = 0;
+
+ await ZipHelper.walk(content, async (entry) => {
+ if (entry.isDirectory) {
+ return;
+ }
+
+ if (entry.fileName === bundle.textPath) {
+ const buffer = await entry.readBuffer(TEXTBUNDLE_MAX_TEXT_SIZE);
+ markdown = buffer.toString("utf8");
+ return;
+ }
+
+ if (entry.fileName === bundle.infoPath) {
+ const buffer = await entry.readBuffer(TEXTBUNDLE_MAX_INFO_SIZE);
+ info = buffer.toString("utf8");
+ return;
+ }
+
+ if (!entry.fileName.toLowerCase().startsWith(assetPrefix)) {
+ return;
+ }
+
+ const relativePath = entry.fileName.slice(rootPrefix.length);
+ const assetMimeType =
+ mime.lookup(relativePath) || "application/octet-stream";
+
+ if (!TEXTBUNDLE_EMBEDDABLE_MIME_TYPES.includes(assetMimeType)) {
+ Logger.info(
+ "utils",
+ `Skipping TextPack asset of unembeddable type ${assetMimeType}`
+ );
+ return;
+ }
+
+ const buffer = await entry.readBuffer(maxAssetSize);
+
+ // Checked against the actual decompressed size, not the archive's
+ // (attacker-controlled) declared size, so this also bounds the total
+ // work done for a bundle of many small-but-lying entries.
+ totalAssetBytes += buffer.length;
+ if (totalAssetBytes > TEXTBUNDLE_MAX_TOTAL_ASSETS_SIZE) {
+ throw FileImportError(
+ "TextPack file assets exceed the maximum combined size"
+ );
+ }
+
+ assets.set(
+ relativePath.toLowerCase(),
+ `data:${assetMimeType};base64,${buffer.toString("base64")}`
+ );
+ });
+
+ if (markdown === undefined) {
+ throw FileImportError(
+ "TextPack file does not contain a recognizable text file"
+ );
+ }
+
+ this.assertTextBundleIsText(bundle.textPath, info);
+
+ markdown = this.embedTextBundleAssets(markdown, assets);
+
+ return this.processFrontmatter(markdown);
+ }
+
+ /**
+ * Locate a TextBundle's text file within a zip's entries, tolerant of the
+ * bundle either being wrapped in a `*.textbundle` directory or zipped flat.
+ * The spec leaves the text entry's extension open, so any `text.*` matches.
+ *
+ * @param entries The zip's entries.
+ * @returns The bundle's root path prefix, text entry path and info.json path,
+ * or null if no recognizable text file was found.
+ */
+ private static findTextBundleRoot(
+ entries: TextBundleEntry[]
+ ): TextBundleRoot | null {
+ const isVisible = (segments: string[]) =>
+ !segments.some(
+ (segment) => segment === "__MACOSX" || segment.startsWith(".")
+ );
+
+ let best:
+ | (TextBundleRoot & { depth: number; knownExtension: boolean })
+ | null = null;
+ let candidates = 0;
+
+ for (const entry of entries) {
+ if (entry.isDirectory) {
+ continue;
+ }
+
+ const segments = entry.fileName.split("/").filter(Boolean);
+ if (!isVisible(segments)) {
+ continue;
+ }
+
+ const name = segments[segments.length - 1].toLowerCase();
+ if (!TEXTBUNDLE_TEXT_REGEX.test(name)) {
+ continue;
+ }
+
+ candidates++;
+
+ // Prefer the shallowest bundle, and among equals the one whose extension
+ // we already know how to read, so a `text.md` beside a `text.fountain`
+ // wins without needing info.json.
+ const depth = segments.length;
+ const knownExtension = TEXTBUNDLE_TEXT_EXTENSIONS.includes(
+ name.slice("text.".length)
+ );
+
+ if (
+ !best ||
+ depth < best.depth ||
+ (depth === best.depth && knownExtension && !best.knownExtension)
+ ) {
+ best = {
+ root: segments.slice(0, -1).join("/"),
+ textPath: entry.fileName,
+ depth,
+ knownExtension,
+ };
+ }
+ }
+
+ if (!best) {
+ return null;
+ }
+
+ if (candidates > 1) {
+ Logger.warn(
+ "TextPack contains more than one text file, importing the first bundle only",
+ { candidates }
+ );
+ }
+
+ const rootPrefix = best.root ? `${best.root}/` : "";
+ const infoPath = entries.find(
+ (entry) =>
+ !entry.isDirectory &&
+ entry.fileName.toLowerCase() ===
+ `${rootPrefix}${TextBundleHelper.infoFileName}`.toLowerCase()
+ )?.fileName;
+
+ return { root: best.root, textPath: best.textPath, infoPath };
+ }
+
+ /**
+ * Assert that a TextBundle holds markdown or plain text rather than another
+ * format such as RTF. The bundle's declared `type` is authoritative; when it
+ * is absent the text entry's extension is used instead.
+ *
+ * @param textPath Path of the bundle's text entry within the zip.
+ * @param info Raw contents of the bundle's info.json, if it has one.
+ * @throws {FileImportError} if the bundle declares a format we cannot read.
+ */
+ private static assertTextBundleIsText(
+ textPath: string,
+ info: string | undefined
+ ): void {
+ let type: string | undefined;
+
+ if (info) {
+ try {
+ const parsed: unknown = JSON.parse(info);
+ if (
+ parsed &&
+ typeof parsed === "object" &&
+ "type" in parsed &&
+ typeof parsed.type === "string"
+ ) {
+ type = parsed.type.toLowerCase();
+ }
+ } catch (_err) {
+ // A malformed info.json falls back to the extension check below.
+ }
+ }
+
+ if (type) {
+ if (
+ !TEXTBUNDLE_TEXT_TYPES.includes(type) &&
+ !type.includes("markdown") &&
+ !type.includes("plain-text")
+ ) {
+ throw FileImportError(
+ `TextPack file contains ${type} content, which cannot be imported`
+ );
+ }
+ return;
+ }
+
+ const extension = textPath.split(".").pop()?.toLowerCase() ?? "";
+ if (!TEXTBUNDLE_TEXT_EXTENSIONS.includes(extension)) {
+ throw FileImportError(
+ `TextPack file contains a .${extension} text file, which cannot be imported`
+ );
+ }
+ }
+
+ /**
+ * Replace TextBundle asset references (e.g. ``) in
+ * markdown with their base64 data URI, resolving relative and
+ * percent-encoded links.
+ *
+ * @param markdown The bundle's markdown text.
+ * @param assets Map of lowercased asset path (relative to the bundle root)
+ * to data URI.
+ * @returns The markdown with recognized asset links replaced.
+ */
+ private static embedTextBundleAssets(
+ markdown: string,
+ assets: Map
+ ): string {
+ return replaceMarkdownLinks(markdown, (href) => {
+ let key = href.replace(/^\.\//, "");
+ try {
+ key = decodeURIComponent(key);
+ } catch {
+ // Leave as-is if not validly percent-encoded.
+ }
+
+ return assets.get(key.toLowerCase());
+ });
+ }
+}
diff --git a/server/models/helpers/ProseMirrorHelper.test.ts b/server/models/helpers/ProseMirrorHelper.test.ts
index ce09060463..d3c6b2fae2 100644
--- a/server/models/helpers/ProseMirrorHelper.test.ts
+++ b/server/models/helpers/ProseMirrorHelper.test.ts
@@ -8,6 +8,7 @@ import { MentionType } from "@shared/types";
import { ProsemirrorHelper as SharedProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
import { createContext } from "@server/context";
import { schema } from "@server/editor";
+import { Attachment } from "@server/models";
import { buildProseMirrorDoc, buildUser } from "@server/test/factories";
import type { MentionAttrs } from "./ProsemirrorHelper";
import { ProsemirrorHelper } from "./ProsemirrorHelper";
@@ -1332,6 +1333,78 @@ describe("ProsemirrorHelper", () => {
expect(result.toJSON()).toEqual(doc.toJSON());
});
+
+ it("should turn a link pointing at a data URI into an attachment", async () => {
+ const user = await buildUser();
+ const ctx = createContext({ user });
+
+ const doc = buildProseMirrorDoc([
+ {
+ type: "paragraph",
+ content: [
+ {
+ type: "text",
+ text: "the spec",
+ marks: [
+ {
+ type: "link",
+ attrs: {
+ href: "data:application/pdf;base64,JVBERi0xLjQK",
+ },
+ },
+ ],
+ },
+ ],
+ },
+ ]);
+
+ const result = await ProsemirrorHelper.replaceImagesWithAttachments(
+ ctx,
+ doc,
+ user
+ );
+
+ const link = result.content
+ .child(0)
+ .content.child(0)
+ .marks.find((mark) => mark.type.name === "link");
+
+ expect(link?.attrs.href).toMatch(/^\/api\/attachments\.redirect\?id=/);
+ expect(await Attachment.count({ where: { teamId: user.teamId } })).toBe(
+ 1
+ );
+ });
+
+ it("should leave an ordinary external link untouched", async () => {
+ const user = await buildUser();
+ const ctx = createContext({ user });
+
+ const doc = buildProseMirrorDoc([
+ {
+ type: "paragraph",
+ content: [
+ {
+ type: "text",
+ text: "example",
+ marks: [
+ { type: "link", attrs: { href: "https://example.com/page" } },
+ ],
+ },
+ ],
+ },
+ ]);
+
+ const result = await ProsemirrorHelper.replaceImagesWithAttachments(
+ ctx,
+ doc,
+ user
+ );
+
+ expect(result.toJSON()).toEqual(doc.toJSON());
+ expect(await Attachment.count({ where: { teamId: user.teamId } })).toBe(
+ 0
+ );
+ });
});
describe("#applyCommentMarkByText", () => {
diff --git a/server/models/helpers/ProsemirrorHelper.tsx b/server/models/helpers/ProsemirrorHelper.tsx
index d28c7c9aaa..e3b9d748fc 100644
--- a/server/models/helpers/ProsemirrorHelper.tsx
+++ b/server/models/helpers/ProsemirrorHelper.tsx
@@ -975,12 +975,15 @@ export class ProsemirrorHelper extends SharedProsemirrorHelper {
/**
* Replaces remote and base64 encoded images in the given Prosemirror node
- * with attachment urls and uploads the images to the storage provider.
+ * with attachment urls and uploads the images to the storage provider. Links
+ * pointing at a data URI are uploaded too, so that files which are not
+ * images — a PDF bundled alongside a document, for example — do not remain
+ * embedded in the document as base64.
*
* @param ctx The API context.
* @param doc The Prosemirror node to process.
* @param user The user context.
- * @returns A new Prosemirror node with images replaced.
+ * @returns A new Prosemirror node with images and embedded files replaced.
*/
static async replaceImagesWithAttachments(
ctx: APIContext,
@@ -991,21 +994,41 @@ export class ProsemirrorHelper extends SharedProsemirrorHelper {
const videos = ProsemirrorHelper.getVideos(doc);
const nodes = [...images, ...videos];
- if (!nodes.length) {
+ // Only data URIs are collected from links: an ordinary external link is a
+ // reference to a page, not a file the document should take a copy of.
+ const links: { href: string; name: string }[] = [];
+ doc.descendants((node) => {
+ const link = node.marks.find((mark) => mark.type.name === "link");
+ const href = String(link?.attrs.href ?? "");
+ if (href.startsWith("data:")) {
+ links.push({ href, name: node.textContent || "file" });
+ }
+ return true;
+ });
+
+ if (!nodes.length && !links.length) {
return doc;
}
+ const sources = [
+ ...nodes.map((node) => ({
+ href: String(node.attrs.src ?? ""),
+ name: String(node.attrs.alt ?? node.type.name),
+ })),
+ ...links,
+ ];
+
const timeoutPerImage = Math.floor(
- Math.min(env.REQUEST_TIMEOUT / nodes.length, 10000)
+ Math.min(env.REQUEST_TIMEOUT / sources.length, 10000)
);
const urlToAttachment: Map = new Map();
- const chunks = chunk(nodes, 10);
+ const chunks = chunk(sources, 10);
- for (const nodeChunk of chunks) {
+ for (const sourceChunk of chunks) {
await Promise.all(
- nodeChunk.map(async (node) => {
- const src = String(node.attrs.src ?? "");
+ sourceChunk.map(async (source) => {
+ const src = source.href;
// Skip invalid URLs
try {
@@ -1026,7 +1049,7 @@ export class ProsemirrorHelper extends SharedProsemirrorHelper {
try {
const attachment = await attachmentCreator({
- name: String(node.attrs.alt ?? node.type.name),
+ name: source.name,
url: src,
preset: AttachmentPreset.DocumentAttachment,
user,
@@ -1049,7 +1072,8 @@ export class ProsemirrorHelper extends SharedProsemirrorHelper {
);
}
- // Transform the document to replace image/video src attributes
+ // Transform the document to replace image/video src attributes and the
+ // href of any link that was uploaded above.
const transformFragment = (fragment: Fragment): Fragment => {
const transformedNodes: Node[] = [];
@@ -1065,10 +1089,27 @@ export class ProsemirrorHelper extends SharedProsemirrorHelper {
} else {
transformedNodes.push(node);
}
- } else if (node.content.size > 0) {
- transformedNodes.push(node.copy(transformFragment(node.content)));
} else {
- transformedNodes.push(node);
+ const marks = node.marks.map((mark) => {
+ const attachment =
+ mark.type.name === "link"
+ ? urlToAttachment.get(String(mark.attrs.href ?? ""))
+ : undefined;
+
+ return attachment
+ ? mark.type.create({
+ ...mark.attrs,
+ href: attachment.redirectUrl,
+ })
+ : mark;
+ });
+
+ const content =
+ node.content.size > 0 ? transformFragment(node.content) : undefined;
+
+ transformedNodes.push(
+ (content ? node.copy(content) : node).mark(marks)
+ );
}
});
diff --git a/server/models/helpers/TextBundleHelper.ts b/server/models/helpers/TextBundleHelper.ts
new file mode 100644
index 0000000000..9d4ea97088
--- /dev/null
+++ b/server/models/helpers/TextBundleHelper.ts
@@ -0,0 +1,74 @@
+import path from "node:path";
+import env from "@server/env";
+import type Document from "@server/models/Document";
+import { serializeFilename } from "@server/utils/fs";
+
+/**
+ * Helpers for writing documents in the TextBundle format, a directory holding
+ * a document's text alongside the files it references.
+ *
+ * @see https://textbundle.org/spec/
+ */
+export default class TextBundleHelper {
+ /**
+ * Name of a bundle's text entry. Markdown is the format's default type, so
+ * the extension is the conventional one rather than a choice.
+ */
+ public static readonly textFileName = "text.markdown";
+
+ /** Name of a bundle's metadata entry. */
+ public static readonly infoFileName = "info.json";
+
+ /** Directory a bundle's referenced files live in. */
+ public static readonly assetsDirectory = "assets";
+
+ /** Extension of a bundle directory, without a leading dot. */
+ public static readonly bundleExtension = "textbundle";
+
+ /** Extension of a zipped bundle, without a leading dot. */
+ public static readonly packExtension = "textpack";
+
+ /**
+ * Builds the metadata entry that identifies a directory as a TextBundle.
+ *
+ * @param document The document the bundle was built from.
+ * @returns The contents of the bundle's info.json.
+ */
+ public static info(document: Document): string {
+ return JSON.stringify(
+ {
+ version: 2,
+ type: "net.daringfireball.markdown",
+ transient: false,
+ creatorIdentifier: "com.getoutline.outline",
+ creatorURL: env.URL,
+ sourceURL: `${env.URL}${document.url}`,
+ },
+ null,
+ 2
+ );
+ }
+
+ /**
+ * Resolves a name for an asset that is safe as a path component and unique
+ * within a single bundle, since two attachments on one document may share an
+ * original file name.
+ *
+ * @param name The attachment's original file name.
+ * @param used Names already taken within this bundle, added to in place.
+ * @returns The path of the asset relative to the bundle's text file.
+ */
+ public static assetPath(name: string, used: Set): string {
+ const serialized = serializeFilename(path.basename(name));
+ const { name: base, ext } = path.parse(serialized);
+
+ let candidate = base ? serialized : "file";
+ let i = 0;
+ while (used.has(candidate)) {
+ candidate = `${base || "file"} (${++i})${ext}`;
+ }
+
+ used.add(candidate);
+ return path.join(this.assetsDirectory, candidate);
+ }
+}
diff --git a/server/queues/processors/FileOperationCreatedProcessor.ts b/server/queues/processors/FileOperationCreatedProcessor.ts
index 0ef5161dbb..598b352b7e 100644
--- a/server/queues/processors/FileOperationCreatedProcessor.ts
+++ b/server/queues/processors/FileOperationCreatedProcessor.ts
@@ -4,6 +4,7 @@ import type { Event as TEvent, FileOperationEvent } from "@server/types";
import ExportHTMLZipTask from "../tasks/ExportHTMLZipTask";
import ExportJSONTask from "../tasks/ExportJSONTask";
import ExportMarkdownZipTask from "../tasks/ExportMarkdownZipTask";
+import ExportTextBundleZipTask from "../tasks/ExportTextBundleZipTask";
import BaseProcessor from "./BaseProcessor";
export default class FileOperationCreatedProcessor extends BaseProcessor {
@@ -32,6 +33,11 @@ export default class FileOperationCreatedProcessor extends BaseProcessor {
fileOperationId: event.modelId,
});
break;
+ case FileOperationFormat.TextBundleZip:
+ await new ExportTextBundleZipTask().schedule({
+ fileOperationId: event.modelId,
+ });
+ break;
case FileOperationFormat.JSON:
await new ExportJSONTask().schedule({
fileOperationId: event.modelId,
diff --git a/server/queues/tasks/ExportDocumentTreeTask.ts b/server/queues/tasks/ExportDocumentTreeTask.ts
index 9f6867904d..2b111d931f 100644
--- a/server/queues/tasks/ExportDocumentTreeTask.ts
+++ b/server/queues/tasks/ExportDocumentTreeTask.ts
@@ -11,17 +11,37 @@ import Document from "@server/models/Document";
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import TextBundleHelper from "@server/models/helpers/TextBundleHelper";
import ZipHelper from "@server/utils/ZipHelper";
import { serializeFilename } from "@server/utils/fs";
import ExportTask from "./ExportTask";
export default abstract class ExportDocumentTreeTask extends ExportTask {
+ /**
+ * The extension given to each document's entry in the archive. For TextBundle
+ * this names a directory rather than a file.
+ *
+ * @param format The format being exported.
+ * @returns The extension, without a leading dot.
+ */
+ private static extensionForFormat(format: FileOperationFormat): string {
+ switch (format) {
+ case FileOperationFormat.HTMLZip:
+ return "html";
+ case FileOperationFormat.TextBundleZip:
+ return TextBundleHelper.bundleExtension;
+ default:
+ return "md";
+ }
+ }
+
/**
* Exports the document tree to the given zip instance.
*
* @param zip The yazl ZipFile to add files to
* @param documentId The document ID to export
- * @param pathInZip The path in the zip to add the document to
+ * @param pathInZip The path in the zip to add the document to. For TextBundle
+ * this is the bundle directory rather than a file.
* @param format The format to export in
*/
protected async processDocument({
@@ -50,6 +70,15 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
? await DocumentHelper.toHTML(document, { centered: true })
: await DocumentHelper.toMarkdown(document);
+ const isTextBundle = format === FileOperationFormat.TextBundleZip;
+
+ // A TextBundle is a directory, so its text and assets sit inside the path
+ // reserved for the document rather than beside it.
+ const textPathInZip = isTextBundle
+ ? path.join(pathInZip, TextBundleHelper.textFileName)
+ : pathInZip;
+ const usedAssetNames = new Set();
+
const attachmentIds = includeAttachments
? ProsemirrorHelper.parseAttachmentIds(
DocumentHelper.toProsemirror(document)
@@ -123,15 +152,23 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
}
}
+ // TextBundle requires assets to live in the bundle's own assets folder,
+ // referenced relative to the text file, rather than at their storage key.
+ const reference = isTextBundle
+ ? TextBundleHelper.assetPath(attachment.name, usedAssetNames)
+ : attachment.key;
+
this.addAttachmentToArchive(
zip,
attachment,
- path.join(dir, attachment.key)
+ isTextBundle
+ ? path.join(pathInZip, reference)
+ : path.join(dir, reference)
);
text = text.replace(
new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
- encodeURI(attachment.key)
+ encodeURI(reference)
);
}
@@ -144,7 +181,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
const matchedDocPath = pathMap.get(matchedLink);
if (matchedDocPath) {
- const relativePath = path.relative(pathInZip, matchedDocPath);
+ const relativePath = path.relative(textPathInZip, matchedDocPath);
if (relativePath.startsWith(".")) {
text = text.replace(
matchedLink,
@@ -154,8 +191,16 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
}
});
+ if (isTextBundle) {
+ zip.addBuffer(
+ Buffer.from(TextBundleHelper.info(document)),
+ path.join(pathInZip, TextBundleHelper.infoFileName),
+ { mtime: document.updatedAt }
+ );
+ }
+
// Finally, add the document to the zip file
- zip.addBuffer(Buffer.from(text), pathInZip, {
+ zip.addBuffer(Buffer.from(text), textPathInZip, {
mtime: document.updatedAt,
fileComment: JSON.stringify({
createdAt: document.createdAt,
@@ -204,11 +249,13 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
}) {
const pathMap = new Map();
- const extension = format === FileOperationFormat.HTMLZip ? "html" : "md";
const rootFolderName = serializeFilename(document.titleWithDefault);
// entry for root document
- pathMap.set(document.path, `${rootFolderName}.${extension}`);
+ pathMap.set(
+ document.path,
+ `${rootFolderName}.${ExportDocumentTreeTask.extensionForFormat(format)}`
+ );
this.addDocumentTreeToPathMap(
pathMap,
@@ -313,7 +360,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask {
) {
for (const node of nodes) {
const title = serializeFilename(node.title) || "Untitled";
- const extension = format === FileOperationFormat.HTMLZip ? "html" : "md";
+ const extension = ExportDocumentTreeTask.extensionForFormat(format);
// Ensure the document is given a unique path in zip, even if it has
// the same title as another document in the same collection.
diff --git a/server/queues/tasks/ExportTextBundleZipTask.test.ts b/server/queues/tasks/ExportTextBundleZipTask.test.ts
new file mode 100644
index 0000000000..139709ce3f
--- /dev/null
+++ b/server/queues/tasks/ExportTextBundleZipTask.test.ts
@@ -0,0 +1,253 @@
+import { Readable } from "node:stream";
+import fs from "fs-extra";
+import { vi } from "vitest";
+import FileStorage from "@server/storage/files";
+import { DocumentConverter } from "@server/converters/DocumentConverter";
+import ZipHelper from "@server/utils/ZipHelper";
+import {
+ buildAttachment,
+ buildCollection,
+ buildDocument,
+ buildDocumentWithAttachment,
+ buildFileOperation,
+ buildTeam,
+ buildUser,
+} from "@server/test/factories";
+import { buildZip } from "@server/test/support";
+import ExportTextBundleZipTask from "./ExportTextBundleZipTask";
+
+/** A 1x1 transparent PNG, used as attachment contents in the round trip. */
+const PNG_PIXEL = Buffer.from(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==",
+ "base64"
+);
+
+describe("ExportTextBundleZipTask", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("should write each document as a TextBundle directory", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const collection = await buildCollection({
+ teamId: team.id,
+ createdById: user.id,
+ });
+ const parent = await buildDocument({
+ teamId: team.id,
+ userId: user.id,
+ collectionId: collection.id,
+ title: "Parent",
+ });
+ await collection.addDocumentToStructure(parent);
+ const child = await buildDocument({
+ teamId: team.id,
+ userId: user.id,
+ collectionId: collection.id,
+ parentDocumentId: parent.id,
+ title: "Child",
+ });
+ await collection.addDocumentToStructure(child);
+
+ const fileOperation = await buildFileOperation({
+ teamId: team.id,
+ userId: user.id,
+ });
+
+ const task = new ExportTextBundleZipTask();
+ const filePath = await task.exportCollections([collection], fileOperation);
+
+ try {
+ const contents = await readZipContents(filePath);
+
+ // Nested documents keep the folder layout the markdown export uses, with
+ // each document itself becoming a bundle directory.
+ expect(Object.keys(contents).sort()).toEqual([
+ `${collection.name}/Parent.textbundle/info.json`,
+ `${collection.name}/Parent.textbundle/text.markdown`,
+ `${collection.name}/Parent/Child.textbundle/info.json`,
+ `${collection.name}/Parent/Child.textbundle/text.markdown`,
+ ]);
+ } finally {
+ await fs.remove(filePath);
+ }
+ });
+
+ it("should declare the bundle as markdown in info.json", async () => {
+ const { collection, document, fileOperation } =
+ await buildDocumentWithAttachment();
+
+ vi.spyOn(FileStorage, "getFileStream").mockResolvedValue(
+ Readable.from(["bytes"])
+ );
+
+ const task = new ExportTextBundleZipTask();
+ const filePath = await task.exportCollections([collection], fileOperation);
+
+ try {
+ const contents = await readZipContents(filePath);
+ const info = JSON.parse(
+ contents[`${collection.name}/${document.title}.textbundle/info.json`]
+ );
+
+ expect(info.version).toBe(2);
+ expect(info.type).toBe("net.daringfireball.markdown");
+ expect(info.transient).toBe(false);
+ } finally {
+ await fs.remove(filePath);
+ }
+ });
+
+ it("should write attachments into the bundle's assets folder", async () => {
+ const { collection, document, attachment, fileOperation } =
+ await buildDocumentWithAttachment();
+
+ const getFileStream = vi
+ .spyOn(FileStorage, "getFileStream")
+ .mockResolvedValue(Readable.from(["image-", "bytes"]));
+
+ const task = new ExportTextBundleZipTask();
+ const filePath = await task.exportCollections([collection], fileOperation);
+
+ try {
+ const contents = await readZipContents(filePath);
+ const bundle = `${collection.name}/${document.title}.textbundle`;
+
+ expect(contents[`${bundle}/assets/${attachment.name}`]).toBe(
+ "image-bytes"
+ );
+ // The spec requires assets to be referenced relative to the text file.
+ expect(contents[`${bundle}/text.markdown`]).toContain(
+ `assets/${attachment.name}`
+ );
+ expect(contents[`${bundle}/text.markdown`]).not.toContain(
+ attachment.redirectUrl
+ );
+ expect(getFileStream).toHaveBeenCalledWith(attachment.key);
+ } finally {
+ await fs.remove(filePath);
+ }
+ });
+
+ it("should link between documents relative to the text file", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const collection = await buildCollection({
+ teamId: team.id,
+ createdById: user.id,
+ });
+ const target = await buildDocument({
+ teamId: team.id,
+ userId: user.id,
+ collectionId: collection.id,
+ title: "Target",
+ });
+ await collection.addDocumentToStructure(target);
+ const source = await buildDocument({
+ teamId: team.id,
+ userId: user.id,
+ collectionId: collection.id,
+ title: "Source",
+ text: `See [Target](${target.url})`,
+ });
+ await collection.addDocumentToStructure(source);
+
+ const fileOperation = await buildFileOperation({
+ teamId: team.id,
+ userId: user.id,
+ });
+
+ const task = new ExportTextBundleZipTask();
+ const filePath = await task.exportCollections([collection], fileOperation);
+
+ try {
+ const contents = await readZipContents(filePath);
+ const text =
+ contents[`${collection.name}/Source.textbundle/text.markdown`];
+
+ // The text file sits one level inside the bundle, so a sibling bundle is
+ // reached by stepping out of it first.
+ expect(text).toContain("../Target.textbundle");
+ } finally {
+ await fs.remove(filePath);
+ }
+ });
+
+ it("should produce a bundle the TextPack importer can read back", async () => {
+ const team = await buildTeam();
+ const user = await buildUser({ teamId: team.id });
+ const collection = await buildCollection({
+ teamId: team.id,
+ createdById: user.id,
+ });
+ const attachment = await buildAttachment(
+ { teamId: team.id, userId: user.id, contentType: "image/png" },
+ "photo.png"
+ );
+ const document = await buildDocument({
+ teamId: team.id,
+ userId: user.id,
+ collectionId: collection.id,
+ title: "Round Trip",
+ icon: "🚀",
+ text: ``,
+ });
+ await collection.addDocumentToStructure(document);
+ const fileOperation = await buildFileOperation({
+ teamId: team.id,
+ userId: user.id,
+ });
+
+ vi.spyOn(FileStorage, "getFileStream").mockResolvedValue(
+ Readable.from([PNG_PIXEL])
+ );
+
+ const task = new ExportTextBundleZipTask();
+ const filePath = await task.exportCollections([collection], fileOperation);
+
+ try {
+ // Repackage the exported bundle on its own, which is what a TextPack is.
+ const bundle = `${collection.name}/${document.title}.textbundle/`;
+ const files: Record = {};
+ await ZipHelper.walk(filePath, async (entry) => {
+ if (!entry.isDirectory && entry.fileName.startsWith(bundle)) {
+ files[entry.fileName.slice(bundle.length)] = await entry.readBuffer(
+ 1024 * 1024
+ );
+ }
+ });
+
+ const result = await DocumentConverter.convert(
+ await buildZip(files),
+ `${document.title}.textpack`,
+ "application/octet-stream"
+ );
+
+ expect(result.title).toEqual(document.title);
+ // The icon is written into the title on export, so it has to come back
+ // out of it rather than staying glued to the title.
+ expect(result.icon).toEqual("🚀");
+ // The asset resolves through the round trip rather than being left as a
+ // dangling relative reference.
+ expect(result.text).toContain("data:image/png;base64,");
+ expect(result.text).not.toContain(`assets/${attachment.name}`);
+ } finally {
+ await fs.remove(filePath);
+ }
+ });
+});
+
+async function readZipContents(
+ filePath: string
+): Promise> {
+ const contents: Record = {};
+ await ZipHelper.walk(filePath, async (entry) => {
+ if (!entry.isDirectory) {
+ contents[entry.fileName] = (await entry.readBuffer(1024 * 1024)).toString(
+ "utf8"
+ );
+ }
+ });
+ return contents;
+}
diff --git a/server/queues/tasks/ExportTextBundleZipTask.ts b/server/queues/tasks/ExportTextBundleZipTask.ts
new file mode 100644
index 0000000000..afff6661c2
--- /dev/null
+++ b/server/queues/tasks/ExportTextBundleZipTask.ts
@@ -0,0 +1,31 @@
+import type { NavigationNode } from "@shared/types";
+import { FileOperationFormat } from "@shared/types";
+import type { Collection, FileOperation } from "@server/models";
+import type { Document } from "@server/models";
+import ExportDocumentTreeTask from "./ExportDocumentTreeTask";
+
+export default class ExportTextBundleZipTask extends ExportDocumentTreeTask {
+ public async exportCollections(
+ collections: Collection[],
+ fileOperation: FileOperation
+ ) {
+ return await this.addCollectionsToArchive(
+ collections,
+ FileOperationFormat.TextBundleZip,
+ fileOperation.options?.includeAttachments
+ );
+ }
+
+ public async exportDocument(
+ document: Document,
+ documentStructure: NavigationNode[],
+ includeAttachments: boolean
+ ): Promise {
+ return await this.addDocumentToArchive({
+ document,
+ documentStructure,
+ format: FileOperationFormat.TextBundleZip,
+ includeAttachments,
+ });
+ }
+}
diff --git a/server/queues/tasks/MarkdownAPIImportTask.test.ts b/server/queues/tasks/MarkdownAPIImportTask.test.ts
index ad832dc50b..a57fa6c8ae 100644
--- a/server/queues/tasks/MarkdownAPIImportTask.test.ts
+++ b/server/queues/tasks/MarkdownAPIImportTask.test.ts
@@ -125,4 +125,22 @@ describe("rewriteInternalLinks", () => {
);
expect(out).toBe("see [other](<>)");
});
+
+ it("rewrites an angle bracketed link to a document with spaces", () => {
+ const out = rewriteInternalLinks(
+ "see [other](<./My Doc.md>)",
+ "Collection/parent.md",
+ { "Collection/My Doc.md": "doc-4" }
+ );
+ expect(out).toBe("see [other](<>)");
+ });
+
+ it("rewrites a link carrying a title, keeping the title", () => {
+ const out = rewriteInternalLinks(
+ 'see [other](./other.md "The other one")',
+ "Collection/parent.md",
+ { "Collection/other.md": "doc-5" }
+ );
+ expect(out).toBe('see [other](<> "The other one")');
+ });
});
diff --git a/server/queues/tasks/MarkdownAPIImportTask.ts b/server/queues/tasks/MarkdownAPIImportTask.ts
index 83815051f2..5f3864583f 100644
--- a/server/queues/tasks/MarkdownAPIImportTask.ts
+++ b/server/queues/tasks/MarkdownAPIImportTask.ts
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
import { escapeRegExp, truncate } from "es-toolkit/compat";
import mime from "mime-types";
import { UniqueConstraintError } from "sequelize";
+import { replaceMarkdownLinks } from "@shared/utils/markdown";
import { CollectionValidation, DocumentValidation } from "@shared/validations";
import type {
ImportTaskInput,
@@ -28,7 +29,7 @@ import type { ZipTreeNode } from "@server/utils/ZipHelper";
import ZipHelper from "@server/utils/ZipHelper";
import type { ProcessOutput } from "./APIImportTask";
import APIImportTask from "./APIImportTask";
-import { DocumentConverter } from "@server/utils/DocumentConverter";
+import { DocumentConverter } from "@server/converters/DocumentConverter";
type Markdown = IntegrationService.Markdown;
@@ -133,22 +134,18 @@ export function rewriteInternalLinks(
docMap: Record
): string {
const basePath = path.dirname(documentPath);
- const internalLinks = [...markdown.matchAll(/\[[^\]]+\]\(([^)]+\.md)\)/g)];
- let text = markdown;
- for (const match of internalLinks) {
- const referredDocPath = match[1];
- const normalizedDocPath = decodeURI(
- path.normalize(`${basePath}/${referredDocPath}`)
- );
+ return replaceMarkdownLinks(markdown, (href) => {
+ let normalizedDocPath: string;
+ try {
+ normalizedDocPath = decodeURI(path.normalize(`${basePath}/${href}`));
+ } catch {
+ return undefined;
+ }
const referredDocId = docMap[normalizedDocPath];
- if (referredDocId) {
- text = text.replace(referredDocPath, `<<${referredDocId}>>`);
- }
- }
-
- return text;
+ return referredDocId ? `<<${referredDocId}>>` : undefined;
+ });
}
export default class MarkdownAPIImportTask extends APIImportTask {
diff --git a/server/routes/api/documents/documents.test.ts b/server/routes/api/documents/documents.test.ts
index 2bf1d128cf..c283b7bbbc 100644
--- a/server/routes/api/documents/documents.test.ts
+++ b/server/routes/api/documents/documents.test.ts
@@ -6,6 +6,7 @@ import FormData from "form-data";
import {
CollectionPermission,
DocumentPermission,
+ ExportContentType,
StatusFilter,
UserRole,
} from "@shared/types";
@@ -736,6 +737,78 @@ describe("#documents.export", () => {
expect(entries["export-test.md"]).not.toContain(attachment.redirectUrl);
});
+ it("should stream a textpack when TextBundle is requested", async () => {
+ const user = await buildUser();
+ const attachment = await buildAttachment(
+ { teamId: user.teamId, userId: user.id, contentType: "image/png" },
+ "photo.png"
+ );
+ const document = await buildDocument({
+ title: "Export Test",
+ userId: user.id,
+ teamId: user.teamId,
+ text: ``,
+ });
+ vi.spyOn(FileStorage, "getFileBuffer").mockResolvedValue(
+ Buffer.from("image-data")
+ );
+
+ const res = await server.post("/api/documents.export", user, {
+ body: {
+ id: document.id,
+ },
+ headers: {
+ accept: ExportContentType.TextBundle,
+ },
+ });
+
+ expect(res.status).toEqual(200);
+ expect(res.headers.get("content-disposition")).toContain(
+ `filename="export-test.textpack"`
+ );
+
+ const bundle = "export-test.textbundle";
+ const entries = await readZipResponse(res);
+ expect(Object.keys(entries).sort()).toEqual([
+ `${bundle}/assets/photo.png`,
+ `${bundle}/info.json`,
+ `${bundle}/text.markdown`,
+ ]);
+ expect(entries[`${bundle}/assets/photo.png`]).toEqual("image-data");
+ expect(entries[`${bundle}/text.markdown`]).toContain("assets/photo.png");
+ expect(JSON.parse(entries[`${bundle}/info.json`]).type).toEqual(
+ "net.daringfireball.markdown"
+ );
+ });
+
+ it("should stream a textpack when TextBundle is requested for a document with no attachments", async () => {
+ const user = await buildUser();
+ const document = await buildDocument({
+ title: "Plain Export",
+ userId: user.id,
+ teamId: user.teamId,
+ });
+
+ const res = await server.post("/api/documents.export", user, {
+ body: {
+ id: document.id,
+ },
+ headers: {
+ accept: ExportContentType.TextBundle,
+ },
+ });
+
+ expect(res.status).toEqual(200);
+
+ // A bundle is a directory, so there is no self-contained single file form
+ // to fall back to when nothing is referenced.
+ const entries = await readZipResponse(res);
+ expect(Object.keys(entries).sort()).toEqual([
+ "plain-export.textbundle/info.json",
+ "plain-export.textbundle/text.markdown",
+ ]);
+ });
+
it("should require authorization without token", async () => {
const document = await buildDocument();
const res = await server.post("/api/documents.export", {
diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts
index 2925ae6a43..779d6456eb 100644
--- a/server/routes/api/documents/documents.ts
+++ b/server/routes/api/documents/documents.ts
@@ -13,6 +13,7 @@ import { errToString } from "@shared/utils/error";
import type { DirectionFilter, SortFilter } from "@shared/types";
import { type NavigationNode } from "@shared/types";
import {
+ ExportContentType,
FileOperationFormat,
FileOperationState,
FileOperationType,
@@ -68,6 +69,7 @@ import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
import HTMLHelper from "@server/models/helpers/HTMLHelper";
import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import TextBundleHelper from "@server/models/helpers/TextBundleHelper";
import SearchProviderManager from "@server/utils/SearchProviderManager";
import { TextHelper } from "@server/models/helpers/TextHelper";
import { authorize, cannot } from "@server/policies";
@@ -812,8 +814,12 @@ router.post(
const document = await documentLoader({
id,
user,
- // We need the collaborative state to generate HTML.
- includeState: !accept?.includes("text/markdown"),
+ // We need the collaborative state to generate HTML, but not for the
+ // formats that are written from markdown.
+ includeState: !(
+ accept?.includes("text/markdown") ||
+ accept?.includes(ExportContentType.TextBundle)
+ ),
});
authorize(user, "download", document);
@@ -822,9 +828,11 @@ router.post(
? FileOperationFormat.HTMLZip
: accept?.includes("text/markdown")
? FileOperationFormat.MarkdownZip
- : accept?.includes("application/pdf")
- ? FileOperationFormat.PDF
- : null;
+ : accept?.includes(ExportContentType.TextBundle)
+ ? FileOperationFormat.TextBundleZip
+ : accept?.includes("application/pdf")
+ ? FileOperationFormat.PDF
+ : null;
if (format === FileOperationFormat.PDF) {
throw IncorrectEditionError(
@@ -876,13 +884,17 @@ router.post(
teamId: user.teamId,
});
+ // A TextBundle is a directory of files, so unlike the other formats it has
+ // no self-contained single-file form to fall back to.
+ const isTextBundle = format === FileOperationFormat.TextBundleZip;
+
if (format === FileOperationFormat.HTMLZip) {
contentType = "text/html";
content = await DocumentHelper.toHTML(document, {
centered: true,
includeMermaid: true,
});
- } else if (format === FileOperationFormat.MarkdownZip) {
+ } else if (isTextBundle || format === FileOperationFormat.MarkdownZip) {
contentType = "text/markdown";
content = await toMarkdown();
} else {
@@ -944,6 +956,44 @@ router.post(
externalAttachments.push({ attachment, buffer });
}
+ if (isTextBundle) {
+ const root = `${fileName}.${TextBundleHelper.bundleExtension}`;
+ const usedAssetNames = new Set();
+
+ streamZipResponse(
+ ctx,
+ `${fileName}.${TextBundleHelper.packExtension}`,
+ (zip) => {
+ for (const { attachment, buffer } of externalAttachments) {
+ const reference = TextBundleHelper.assetPath(
+ attachment.name,
+ usedAssetNames
+ );
+ zip.addBuffer(buffer, path.join(root, reference), {
+ mtime: attachment.updatedAt,
+ });
+
+ content = content.replace(
+ new RegExp(escapeRegExp(attachment.redirectUrl), "g"),
+ encodeURI(reference)
+ );
+ }
+
+ zip.addBuffer(
+ Buffer.from(TextBundleHelper.info(document)),
+ path.join(root, TextBundleHelper.infoFileName),
+ { mtime: document.updatedAt }
+ );
+ zip.addBuffer(
+ Buffer.from(content),
+ path.join(root, TextBundleHelper.textFileName),
+ { mtime: document.updatedAt }
+ );
+ }
+ );
+ return;
+ }
+
// When there are no external attachments the document is self-contained and
// can be served directly rather than bundled in a zip.
if (externalAttachments.length === 0) {
diff --git a/server/test/fixtures/textbundle-traversal.textpack b/server/test/fixtures/textbundle-traversal.textpack
new file mode 100644
index 0000000000..42b6d9c2ea
Binary files /dev/null and b/server/test/fixtures/textbundle-traversal.textpack differ
diff --git a/server/test/support.ts b/server/test/support.ts
index d0d2e3e3b6..66cf7143d7 100644
--- a/server/test/support.ts
+++ b/server/test/support.ts
@@ -4,6 +4,7 @@ import path from "node:path";
import { randomUUID } from "node:crypto";
import { faker } from "@faker-js/faker";
import type { Transaction } from "sequelize";
+import { ZipFile } from "yazl";
import { afterEach, beforeEach, vi } from "vitest";
import sharedEnv from "@shared/env";
import { createContext } from "@server/context";
@@ -108,6 +109,35 @@ export async function readZipResponse(
}
}
+/**
+ * Build a zip archive in memory from a map of entry path to contents. Useful
+ * for describing an archive's structure inline in a test rather than carrying
+ * an opaque binary fixture.
+ *
+ * @param files Map of path within the archive to its contents.
+ * @returns The zip archive as a Buffer.
+ */
+export function buildZip(
+ files: Record
+): Promise {
+ return new Promise((resolve, reject) => {
+ const zip = new ZipFile();
+ const chunks: Buffer[] = [];
+
+ zip.outputStream.on("data", (chunk: Buffer) => chunks.push(chunk));
+ zip.outputStream.on("end", () => resolve(Buffer.concat(chunks)));
+ zip.outputStream.on("error", reject);
+
+ for (const [name, contents] of Object.entries(files)) {
+ zip.addBuffer(
+ Buffer.isBuffer(contents) ? contents : Buffer.from(contents, "utf8"),
+ name
+ );
+ }
+ zip.end();
+ });
+}
+
/**
* Helper function to convert an object to form-urlencoded string.
* Useful for testing OAuth endpoints that expect application/x-www-form-urlencoded content type.
diff --git a/server/utils/DocumentConverter.ts b/server/utils/DocumentConverter.ts
deleted file mode 100644
index a308a20558..0000000000
--- a/server/utils/DocumentConverter.ts
+++ /dev/null
@@ -1,757 +0,0 @@
-import { escape, escapeRegExp } from "es-toolkit/compat";
-import type { Attachment as MailAttachment, ParsedMail } from "mailparser";
-import type { Node } from "prosemirror-model";
-import { DOMParser as ProsemirrorDOMParser } from "prosemirror-model";
-import yaml from "js-yaml";
-import { schema, serializer } from "@server/editor";
-import { FileImportError } from "@server/errors";
-import { trace, traceFunction } from "@server/logging/tracing";
-import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
-
-export interface ConvertResult {
- /** The document content as markdown text. */
- text: string;
- /** The document content as Prosemirror. */
- doc: Node;
- /** The extracted title (from H1 heading if present). */
- title: string;
- /** The extracted emoji/icon from start of document. */
- icon?: string;
-}
-
-/**
- * Converts incoming files of various formats to structured documents.
- */
-@trace()
-export class DocumentConverter {
- /**
- * Convert an incoming file to a structured document result.
- *
- * @param content The content of the file.
- * @param fileName The name of the file, including extension.
- * @param mimeType The mime type of the file.
- * @param options Conversion options.
- * @param options.extractTitle Whether a leading H1 heading should be lifted
- * out as the document title and removed from the body. Defaults to true;
- * set false for sources where the filename is authoritative and the first
- * heading must remain part of the content (e.g. Slab).
- * @returns The converted document with text, data, title, and icon.
- */
- public static async convert(
- content: Buffer | string,
- fileName: string,
- mimeType: string,
- options: { extractTitle?: boolean } = {}
- ): Promise {
- const { extractTitle = true } = options;
- let doc: Node;
-
- // Route to appropriate conversion method
- const html = await this.convertToHtml(content, fileName, mimeType);
- if (html !== undefined) {
- doc = await this.htmlToProsemirror(html);
- } else {
- const markdown = await this.convertToMarkdown(
- content,
- fileName,
- mimeType
- );
- doc = ProsemirrorHelper.toProsemirror(markdown);
- }
-
- // Extract title from first H1 heading
- let title = "";
- if (extractTitle) {
- const headings = ProsemirrorHelper.getHeadings(doc);
- if (headings.length > 0 && headings[0].level === 1) {
- title = headings[0].title;
- doc = ProsemirrorHelper.removeFirstHeading(doc);
- }
- }
-
- // Extract emoji from start of document
- const { emoji: icon, doc: docWithoutEmoji } =
- ProsemirrorHelper.extractEmojiFromStart(doc);
- doc = docWithoutEmoji;
-
- // Serialize to markdown and trim whitespace
- const text = serializer.serialize(doc).trim();
-
- return {
- text,
- doc,
- title,
- icon,
- };
- }
-
- /**
- * Convert HTML content directly to a Prosemirror document node.
- *
- * @param content The HTML content as a string or Buffer.
- * @returns A Prosemirror Node representing the document.
- */
- public static async htmlToProsemirror(
- content: Buffer | string
- ): Promise {
- if (typeof content !== "string") {
- content = content.toString("utf8");
- }
-
- // Loaded lazily to keep jsdom off the startup path — only HTML imports need it.
- const { JSDOM } = await import("jsdom");
- const dom = new JSDOM(content);
- const document = dom.window.document;
-
- // Remove problematic elements before parsing
- const elementsToRemove = document.querySelectorAll(
- "script, style, title, head, meta, link"
- );
- elementsToRemove.forEach((el) => el.remove());
-
- // Preprocess the DOM to handle edge cases
- this.preprocessHtmlForImport(document);
-
- // Patch global environment for Prosemirror DOMParser
- const cleanup = ProsemirrorHelper.patchGlobalEnv(dom.window);
-
- try {
- const domParser = ProsemirrorDOMParser.fromSchema(schema);
- return domParser.parse(document.body);
- } finally {
- cleanup();
- try {
- dom.window.close();
- } catch (_err) {
- // Best effort, closing the window releases its timers and resources.
- }
- }
- }
-
- /**
- * Preprocesses HTML DOM before Prosemirror parsing to cleanup
- * images and other elements.
- *
- * @param document The DOM document to preprocess.
- */
- private static preprocessHtmlForImport(document: Document): void {
- // Handle images: filter emoticons, remove Jira icons, apply Confluence sizing
- const images = document.querySelectorAll("img");
- images.forEach((img) => {
- const className = img.className || "";
-
- // Skip emoticon images (they'll be dropped)
- if (className.includes("emoticon")) {
- img.remove();
- return;
- }
-
- // Remove Jira icon images
- if (
- className === "icon" &&
- img.parentElement?.className.includes("jira-issue-key")
- ) {
- img.remove();
- return;
- }
-
- // Handle Confluence image sizing: data-width/data-height → width/height
- const dataWidth = img.getAttribute("data-width");
- const dataHeight = img.getAttribute("data-height");
- const width = img.getAttribute("width");
-
- if (dataWidth && dataHeight && width) {
- const ratio = parseInt(dataWidth) / parseInt(width);
- const calculatedHeight = Math.round(parseInt(dataHeight) / ratio);
- img.setAttribute("height", String(calculatedHeight));
- }
-
- // Extract dimensions from data URI images that lack width/height
- // (e.g. images embedded by mammoth during docx import).
- // Only decode a small prefix of the base64 data — headers for all
- // supported formats live within the first 64 KB of the file.
- if (!img.getAttribute("width") && !img.getAttribute("height")) {
- const src = img.getAttribute("src") || "";
- if (src.startsWith("data:") && src.includes(";base64,")) {
- const base64Start = src.indexOf(";base64,") + 8;
- // 4 base64 chars → 3 bytes; decode at most ~64 KB of image data.
- const maxBase64Chars = Math.ceil(65536 / 3) * 4;
- const base64Prefix = src.slice(
- base64Start,
- base64Start + maxBase64Chars
- );
- const dimensions = this.getImageDimensionsFromBuffer(
- Buffer.from(base64Prefix, "base64")
- );
- if (dimensions) {
- img.setAttribute("width", String(dimensions.width));
- img.setAttribute("height", String(dimensions.height));
- }
- }
- }
- });
- }
-
- /**
- * Attempts to convert content to HTML for formats that support it.
- * Returns undefined for formats that should be parsed as markdown directly.
- *
- * @param content The content of the file.
- * @param fileName The name of the file, including extension.
- * @param mimeType The mime type of the file.
- * @returns HTML string if convertible, undefined otherwise.
- */
- private static async convertToHtml(
- content: Buffer | string,
- fileName: string,
- mimeType: string
- ): Promise {
- const extension = fileName.split(".").pop()?.toLowerCase();
-
- // First try to convert based on the mime type
- switch (mimeType) {
- case "text/html":
- return typeof content === "string" ? content : content.toString("utf8");
- case "application/msword":
- return this.confluenceToHtml(content);
- case "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
- return this.docxToHtml(content);
- // Browsers report MHTML ("Save page as → Webpage, Single File") and .eml
- // inconsistently across these three mime types, so the extension is
- // used as a tie-breaker between the two archive kinds below.
- case "multipart/related":
- case "application/x-mimearchive":
- return extension === "eml"
- ? this.emlToHtml(content)
- : this.mhtmlToHtml(content);
- case "message/rfc822":
- return extension === "mhtml" || extension === "mht"
- ? this.mhtmlToHtml(content)
- : this.emlToHtml(content);
- default:
- break;
- }
-
- // Try to convert based on the file extension
- switch (extension) {
- case "htm":
- case "html":
- return typeof content === "string" ? content : content.toString("utf8");
- case "docx":
- return this.docxToHtml(content);
- case "mhtml":
- case "mht":
- return this.mhtmlToHtml(content);
- case "eml":
- return this.emlToHtml(content);
- default:
- return undefined;
- }
- }
-
- /**
- * Converts content to markdown for text-based formats.
- *
- * @param content The content of the file.
- * @param fileName The name of the file, including extension.
- * @param mimeType The mime type of the file.
- * @returns Markdown string.
- */
- private static async convertToMarkdown(
- content: Buffer | string,
- fileName: string,
- mimeType: string
- ): Promise {
- let markdown: string;
-
- switch (mimeType) {
- case "text/plain":
- case "text/markdown":
- markdown = this.bufferToString(content);
- break;
- case "text/csv":
- case "text/tab-separated-values":
- return this.csvToMarkdown(content);
- default: {
- const extension = fileName.split(".").pop()?.toLowerCase();
- switch (extension) {
- case "md":
- case "markdown":
- case "txt":
- markdown = this.bufferToString(content);
- break;
- case "csv":
- case "tsv":
- return this.csvToMarkdown(content);
- default:
- throw FileImportError(`File type ${mimeType} not supported`);
- }
- }
- }
-
- // Process frontmatter and convert it to a YAML codeblock
- return this.processFrontmatter(markdown);
- }
-
- /**
- * Convert a docx file to HTML using mammoth.
- *
- * @param content The docx file content as a Buffer.
- * @returns The HTML representation of the document.
- */
- private static async docxToHtml(content: Buffer | string): Promise {
- if (content instanceof Buffer) {
- // Loaded lazily to keep mammoth off the startup path — only docx imports need it.
- const mammoth = (await import("mammoth")).default;
- const { value } = await traceFunction({ spanName: "convertToHtml" })(
- mammoth.convertToHtml
- )({
- buffer: content,
- });
- return value;
- }
- throw FileImportError("Unsupported Word file");
- }
-
- /** Maximum number of MIME archive parts that will be inlined as data URIs. */
- private static readonly MIME_ARCHIVE_MAX_INLINE_PARTS = 100;
-
- /** Maximum size of a single MIME archive part that will be inlined as a data URI. */
- private static readonly MIME_ARCHIVE_MAX_PART_BYTES = 15 * 1024 * 1024;
-
- /** Maximum combined size of MIME archive parts that will be inlined as data URIs. */
- private static readonly MIME_ARCHIVE_MAX_TOTAL_BYTES = 50 * 1024 * 1024;
-
- /** Matches a well-formed mime type, e.g. `image/png`. */
- private static readonly MIME_TYPE_REGEX = /^[\w.+-]+\/[\w.+-]+$/;
-
- /**
- * Convert a Confluence Word export to HTML.
- *
- * @param content The Confluence Word export content.
- * @returns The HTML representation of the document.
- */
- private static async confluenceToHtml(
- content: Buffer | string
- ): Promise {
- const text = this.bufferToString(content);
-
- // We're only supporting the output from Confluence here, regular Word documents should call
- // into the docxToHtml importer. See: https://jira.atlassian.com/browse/CONFSERVER-38237
- if (!text.includes("Content-Type: multipart/related")) {
- throw FileImportError("Unsupported Word file");
- }
-
- const { html } = await this.parseMimeArchive(text, {
- emptyMessage: "Unsupported Word file (No content found)",
- });
-
- return html;
- }
-
- /**
- * Convert an MHTML file (e.g. Chrome/Edge "Save page as → Webpage, Single
- * File") to HTML.
- *
- * @param content The MHTML file content.
- * @returns The HTML representation of the document.
- */
- private static async mhtmlToHtml(content: Buffer | string): Promise {
- const { html } = await this.parseMimeArchive(this.bufferToString(content), {
- emptyMessage: "Unsupported MHTML file (No content found)",
- });
-
- return html;
- }
-
- /**
- * Convert an .eml email message to HTML. The `Subject` header, if present,
- * is inserted as a leading H1 so it is picked up as the document title by
- * the same leading-heading extraction used for every other format.
- *
- * @param content The .eml file content.
- * @returns The HTML representation of the message.
- */
- private static async emlToHtml(content: Buffer | string): Promise {
- const { html, subject } = await this.parseMimeArchive(
- this.bufferToString(content),
- {
- allowTextFallback: true,
- emptyMessage: "Unsupported email file (No content found)",
- }
- );
-
- return subject ? `
${escape(subject)}
\n${html}` : html;
- }
-
- /**
- * Parse a MIME archive (`multipart/related`, `message/rfc822`, or similar)
- * with mailparser and resolve its HTML body, inlining referenced parts as
- * data URIs.
- *
- * Confluence's "Word" export and MHTML pages (Chrome/Edge "Save page as →
- * Webpage, Single File") are both, structurally, multi-part email messages,
- * so the same parser and inlining logic serves all of them.
- *
- * @param content The MIME archive content as a string.
- * @param options.allowTextFallback Whether to fall back to the plain text
- * body (rendered as HTML) when no HTML part is present. Used for .eml,
- * where a text-only email is common; MHTML/Word exports always have HTML.
- * @param options.emptyMessage The error message to throw when no usable
- * body is found.
- * @returns The resolved HTML body and, if present, the message subject.
- */
- private static async parseMimeArchive(
- content: string,
- options: { allowTextFallback?: boolean; emptyMessage: string }
- ): Promise<{ html: string; subject?: string }> {
- // Confluence "Word" documents, MHTML pages, and .eml files are all just multi-part
- // email messages, so we can use mailparser to parse the content. Loaded lazily to
- // keep mailparser off the startup path — only these formats need it. `keepCidLinks`
- // is set so we can apply our own bounds when inlining referenced parts below, rather
- // than mailparser inlining every matching part unconditionally.
- const { simpleParser } = await import("mailparser");
-
- let parsed: ParsedMail;
- try {
- parsed = await simpleParser(content, { keepCidLinks: true });
- } catch (_err) {
- throw FileImportError(options.emptyMessage);
- }
-
- let html = parsed.html || undefined;
- if (!html && options.allowTextFallback) {
- html = parsed.textAsHtml || undefined;
- }
- if (!html) {
- throw FileImportError(options.emptyMessage);
- }
-
- return {
- html: this.inlineMimeArchiveParts(html, parsed.attachments),
- subject: parsed.subject,
- };
- }
-
- /**
- * Replace references to MIME archive parts within HTML with data URIs, so
- * the resulting document is self-contained. Parts are referenced either by
- * `Content-Location` (used by MHTML and Confluence exports) or by
- * `Content-ID` as a `cid:` URL (used by email). Inlining is bounded so a
- * hostile or merely enormous archive cannot blow up memory or the
- * resulting document.
- *
- * @param html The HTML body that may reference archive parts.
- * @param attachments The parsed MIME archive parts.
- * @returns The HTML with resolvable references replaced by data URIs.
- */
- private static inlineMimeArchiveParts(
- html: string,
- attachments: MailAttachment[]
- ): string {
- let inlinedParts = 0;
- let inlinedBytes = 0;
-
- for (const attachment of attachments) {
- if (inlinedParts >= this.MIME_ARCHIVE_MAX_INLINE_PARTS) {
- break;
- }
- if (
- attachment.content.length > this.MIME_ARCHIVE_MAX_PART_BYTES ||
- inlinedBytes + attachment.content.length >
- this.MIME_ARCHIVE_MAX_TOTAL_BYTES
- ) {
- continue;
- }
-
- const references = new Set();
- const contentLocation = attachment.headers.get("content-location") as
- | string
- | undefined;
- if (contentLocation) {
- references.add(contentLocation);
- const basename = contentLocation.split("/").pop();
- if (basename) {
- references.add(basename);
- }
- }
- if (attachment.cid) {
- references.add(`cid:${attachment.cid}`);
- }
-
- // The content type comes from the archive's own headers, so anything that
- // isn't a well-formed mime type is discarded rather than interpolated.
- const contentType = this.MIME_TYPE_REGEX.test(attachment.contentType)
- ? attachment.contentType
- : "application/octet-stream";
-
- let replaced = false;
- for (const reference of references) {
- if (html.includes(reference)) {
- const dataUri = `data:${contentType};base64,${attachment.content.toString(
- "base64"
- )}`;
- html = html.split(reference).join(dataUri);
- replaced = true;
- }
- }
-
- if (replaced) {
- inlinedParts++;
- inlinedBytes += attachment.content.length;
- }
- }
-
- return html;
- }
-
- /**
- * Convert a CSV file to a markdown table.
- *
- * @param content The CSV file content.
- * @returns A markdown table representation.
- */
- private static async csvToMarkdown(
- content: Buffer | string
- ): Promise {
- // Loaded lazily to keep @fast-csv off the startup path — only CSV imports need it.
- const { parse } = await import("@fast-csv/parse");
-
- return new Promise((resolve, reject) => {
- const text = this.bufferToString(content).trim();
- const textLines = text.split("\n");
-
- // Find the first non-empty line to determine the delimiter
- const firstNonEmptyLine =
- textLines.find((line) => line.trim().length > 0) || "";
-
- // Determine the separator used in the CSV file based on number of occurrences of each separator on first line
- const delimiter = [";", ",", "\t"].reduce(
- (acc, separator) => {
- const count = (
- firstNonEmptyLine.match(new RegExp(escapeRegExp(separator), "g")) ||
- []
- ).length;
- return count > acc.count ? { count, separator } : acc;
- },
- { count: 0, separator: "," }
- ).separator;
-
- const lines: string[][] = [];
- const stream = parse({ delimiter })
- .on("error", (error) => {
- reject(
- FileImportError(`There was an error parsing the CSV file: ${error}`)
- );
- })
- .on("data", (row) => lines.push(row))
- .on("end", () => {
- // Filter out completely empty rows
- const nonEmptyLines = lines.filter((row) =>
- row.some((cell) => cell.trim() !== "")
- );
-
- if (nonEmptyLines.length === 0) {
- resolve("");
- return;
- }
-
- // Check if all rows have a trailing empty cell (trailing comma artifact)
- // Only trim if ALL non-empty rows end with an empty cell
- let trimmedLines = nonEmptyLines;
- while (
- trimmedLines.length > 0 &&
- trimmedLines.every(
- (row) => row.length > 0 && row[row.length - 1].trim() === ""
- )
- ) {
- trimmedLines = trimmedLines.map((row) => row.slice(0, -1));
- }
-
- // Find the most common column count
- const columnCounts = new Map();
- for (const row of trimmedLines) {
- if (row.length > 0) {
- columnCounts.set(
- row.length,
- (columnCounts.get(row.length) || 0) + 1
- );
- }
- }
-
- // Get the column count that appears most frequently
- let expectedColumns = 0;
- let maxFrequency = 0;
- for (const [count, frequency] of columnCounts) {
- if (frequency > maxFrequency) {
- maxFrequency = frequency;
- expectedColumns = count;
- }
- }
-
- // Find the first row with the expected column count (this is the header)
- const headerIndex = trimmedLines.findIndex(
- (row) => row.length === expectedColumns
- );
- if (headerIndex === -1) {
- resolve("");
- return;
- }
-
- const headers = trimmedLines[headerIndex];
- const dataRows = trimmedLines
- .slice(headerIndex + 1)
- .filter((row) => row.length === expectedColumns);
-
- const table = dataRows
- .map((cells) => `| ${cells.join(" | ")} |`)
- .join("\n");
-
- const headerLine = `| ${headers.join(" | ")} |`;
- const separatorLine = `| ${headers.map(() => "---").join(" | ")} |`;
-
- resolve(`${headerLine}\n${separatorLine}\n${table}\n`);
- });
-
- stream.write(text);
- stream.end();
- });
- }
-
- /**
- * Convert a Buffer to a string.
- *
- * @param content The content as a Buffer or string.
- * @returns The content as a string.
- */
- private static bufferToString(content: Buffer | string): string {
- return typeof content === "string" ? content : content.toString("utf8");
- }
-
- /**
- * Parse and convert frontmatter to a YAML codeblock.
- *
- * @param content The markdown content that may contain frontmatter.
- * @returns The markdown content with frontmatter converted to a YAML codeblock.
- */
- private static processFrontmatter(content: string): string {
- // Frontmatter must start at the beginning of the document
- const frontmatterRegex = /^---\n([\s\S]*?)\n---(?:\n|$)/;
- const match = content.match(frontmatterRegex);
-
- if (!match) {
- return content;
- }
-
- const frontmatterContent = match[1];
- const remainingContent = content.slice(match[0].length);
-
- // Validate that the frontmatter is valid YAML
- try {
- yaml.load(frontmatterContent);
- } catch {
- // If it's not valid YAML, return content unchanged
- return content;
- }
-
- // Convert frontmatter to a YAML codeblock
- const codeBlockDelimiter = "```";
- const yamlCodeblock = `${codeBlockDelimiter}yaml\n${frontmatterContent}\n${codeBlockDelimiter}\n\n`;
-
- return yamlCodeblock + remainingContent;
- }
-
- /**
- * Parse image dimensions from a binary buffer. Supports PNG, JPEG, and GIF.
- *
- * @param buffer The image data.
- * @returns The width and height if parseable, otherwise undefined.
- */
- private static getImageDimensionsFromBuffer(
- buffer: Buffer
- ): { width: number; height: number } | undefined {
- try {
- // PNG: signature + IHDR chunk
- if (
- buffer.length >= 24 &&
- buffer[0] === 0x89 &&
- buffer[1] === 0x50 &&
- buffer[2] === 0x4e &&
- buffer[3] === 0x47
- ) {
- return {
- width: buffer.readUInt32BE(16),
- height: buffer.readUInt32BE(20),
- };
- }
-
- // GIF: signature + logical screen descriptor
- if (
- buffer.length >= 10 &&
- buffer[0] === 0x47 &&
- buffer[1] === 0x49 &&
- buffer[2] === 0x46
- ) {
- return {
- width: buffer.readUInt16LE(6),
- height: buffer.readUInt16LE(8),
- };
- }
-
- // JPEG: scan for SOF marker (cap at 64 KB to bound work)
- if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xd8) {
- const scanLimit = Math.min(buffer.length, 65536);
- let offset = 2;
- while (offset + 1 < scanLimit) {
- if (buffer[offset] !== 0xff) {
- offset++;
- continue;
- }
- const marker = buffer[offset + 1];
- offset += 2;
-
- // Standalone markers without a payload
- if (
- marker === 0x00 ||
- marker === 0x01 ||
- (marker >= 0xd0 && marker <= 0xd9)
- ) {
- continue;
- }
-
- if (offset + 2 > scanLimit) {
- break;
- }
- const segmentLength = buffer.readUInt16BE(offset);
-
- // SOF markers contain the frame dimensions — check before
- // the advance guard since this returns immediately.
- if (
- (marker >= 0xc0 && marker <= 0xc3) ||
- (marker >= 0xc5 && marker <= 0xc7) ||
- (marker >= 0xc9 && marker <= 0xcb) ||
- (marker >= 0xcd && marker <= 0xcf)
- ) {
- if (offset + 7 <= buffer.length) {
- return {
- height: buffer.readUInt16BE(offset + 3),
- width: buffer.readUInt16BE(offset + 5),
- };
- }
- break;
- }
-
- // Length includes itself and must be >= 2; bail on malformed data.
- if (segmentLength < 2 || offset + segmentLength > buffer.length) {
- break;
- }
-
- offset += segmentLength;
- }
- }
- } catch {
- // Return undefined if parsing fails
- }
- return undefined;
- }
-}
diff --git a/server/utils/ZipHelper.test.ts b/server/utils/ZipHelper.test.ts
index b1d3ced24e..9bfc5de58e 100644
--- a/server/utils/ZipHelper.test.ts
+++ b/server/utils/ZipHelper.test.ts
@@ -4,6 +4,7 @@ import fs from "fs-extra";
import tmp from "tmp";
import { vi } from "vitest";
import { ZipFile } from "yazl";
+import { buildZip } from "@server/test/support";
import ZipHelper from "./ZipHelper";
async function writeZip(
@@ -228,3 +229,36 @@ describe("ZipHelper.toFileTree", () => {
expect(sizes).toEqual({ "Collection/page.md": 11 });
});
});
+
+describe("ZipHelper.walk", () => {
+ it("walks an archive held in memory", async () => {
+ const buffer = await buildZip({
+ "Collection/page.md": "hello world",
+ "Collection/image.png": "binary",
+ });
+
+ const contents: Record = {};
+ await ZipHelper.walk(buffer, async (entry) => {
+ if (!entry.isDirectory) {
+ contents[entry.fileName] = (await entry.readBuffer(1024)).toString(
+ "utf8"
+ );
+ }
+ });
+
+ expect(contents).toEqual({
+ "Collection/page.md": "hello world",
+ "Collection/image.png": "binary",
+ });
+ });
+
+ it("rejects an oversized entry in an archive held in memory", async () => {
+ const buffer = await buildZip({ "Collection/page.md": "hello world" });
+
+ await expect(
+ ZipHelper.walk(buffer, async (entry) => {
+ await entry.readBuffer(4);
+ })
+ ).rejects.toThrow("Collection/page.md is too large");
+ });
+});
diff --git a/server/utils/ZipHelper.ts b/server/utils/ZipHelper.ts
index e39f727c26..91239f36d2 100644
--- a/server/utils/ZipHelper.ts
+++ b/server/utils/ZipHelper.ts
@@ -105,120 +105,139 @@ export default class ZipHelper {
* Entries are visited serially in archive order. `onEntry` may be async; the
* next entry is only read once the previous handler resolves.
*
- * @param filePath The file path where the zip is located.
+ * @param source The file path where the zip is located, or a Buffer holding
+ * the zip contents already in memory.
* @param onEntry Handler invoked for each entry. Skip an entry by returning
* without calling `entry.readBuffer(maxSize)`.
* @returns Promise that resolves once the archive has been fully walked.
*/
public static walk(
- filePath: string,
+ source: string | Buffer,
onEntry: (entry: ZipEntryHandle) => Promise | void
): Promise {
return new Promise((resolve, reject) => {
- yauzl.open(
- filePath,
- {
- lazyEntries: true,
- autoClose: true,
- decodeStrings: false,
- },
- function (err, zipfile) {
- if (err) {
- return reject(err);
+ const options = {
+ lazyEntries: true,
+ autoClose: true,
+ decodeStrings: false,
+ };
+
+ const onOpen = function (
+ err: Error | null,
+ zipfile: yauzl.ZipFile
+ ): void {
+ if (err) {
+ return reject(err);
+ }
+
+ let settled = false;
+ const fail = (error: Error) => {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ zipfile.close();
+ reject(error);
+ };
+
+ zipfile.on("entry", (entry: Entry) => {
+ const fileName = Buffer.from(entry.fileName).toString("utf8");
+
+ if (validateFileName(fileName)) {
+ Logger.warn("Invalid zip entry", { fileName });
+ zipfile.readEntry();
+ return;
}
- let settled = false;
- const fail = (error: Error) => {
- if (settled) {
- return;
- }
- settled = true;
- zipfile.close();
- reject(error);
+ const handle: ZipEntryHandle = {
+ fileName,
+ uncompressedSize: entry.uncompressedSize,
+ isDirectory: fileName.endsWith("/"),
+ readBuffer: (maxSize) =>
+ new Promise((res, rej) => {
+ if (entry.uncompressedSize > maxSize) {
+ return rej(ZipHelper.entryTooLargeError(fileName, maxSize));
+ }
+
+ zipfile.openReadStream(entry, (rErr, readStream) => {
+ if (rErr) {
+ return rej(rErr);
+ }
+ const chunks: Buffer[] = [];
+ let bytesRead = 0;
+ let settled = false;
+ readStream.on("data", (chunk: Buffer) => {
+ bytesRead += chunk.length;
+ if (bytesRead > maxSize) {
+ readStream.destroy(
+ ZipHelper.entryTooLargeError(fileName, maxSize)
+ );
+ return;
+ }
+ chunks.push(chunk);
+ });
+ readStream.on("end", () => {
+ if (!settled) {
+ settled = true;
+ res(Buffer.concat(chunks));
+ }
+ });
+ readStream.on("error", (err) => {
+ if (!settled) {
+ settled = true;
+ rej(err);
+ }
+ });
+ readStream.on("close", () => {
+ if (!settled) {
+ settled = true;
+ rej(
+ new Error(
+ `Stream closed before completing read of ${fileName}`
+ )
+ );
+ }
+ });
+ });
+ }),
};
- zipfile.on("entry", (entry: Entry) => {
- const fileName = Buffer.from(entry.fileName).toString("utf8");
+ Promise.resolve()
+ .then(() => onEntry(handle))
+ .then(() => {
+ if (!settled) {
+ zipfile.readEntry();
+ }
+ })
+ .catch(fail);
+ });
- if (validateFileName(fileName)) {
- Logger.warn("Invalid zip entry", { fileName });
- zipfile.readEntry();
- return;
- }
+ const done = () => {
+ if (!settled) {
+ settled = true;
+ resolve();
+ }
+ };
- const handle: ZipEntryHandle = {
- fileName,
- uncompressedSize: entry.uncompressedSize,
- isDirectory: fileName.endsWith("/"),
- readBuffer: (maxSize) =>
- new Promise((res, rej) => {
- if (entry.uncompressedSize > maxSize) {
- return rej(ZipHelper.entryTooLargeError(fileName, maxSize));
- }
-
- zipfile.openReadStream(entry, (rErr, readStream) => {
- if (rErr) {
- return rej(rErr);
- }
- const chunks: Buffer[] = [];
- let bytesRead = 0;
- let settled = false;
- readStream.on("data", (chunk: Buffer) => {
- bytesRead += chunk.length;
- if (bytesRead > maxSize) {
- readStream.destroy(
- ZipHelper.entryTooLargeError(fileName, maxSize)
- );
- return;
- }
- chunks.push(chunk);
- });
- readStream.on("end", () => {
- if (!settled) {
- settled = true;
- res(Buffer.concat(chunks));
- }
- });
- readStream.on("error", (err) => {
- if (!settled) {
- settled = true;
- rej(err);
- }
- });
- readStream.on("close", () => {
- if (!settled) {
- settled = true;
- rej(
- new Error(
- `Stream closed before completing read of ${fileName}`
- )
- );
- }
- });
- });
- }),
- };
-
- Promise.resolve()
- .then(() => onEntry(handle))
- .then(() => {
- if (!settled) {
- zipfile.readEntry();
- }
- })
- .catch(fail);
- });
-
- zipfile.on("close", () => {
- if (!settled) {
- settled = true;
- resolve();
- }
- });
- zipfile.on("error", (error) => fail(error));
- zipfile.readEntry();
+ // A file-backed archive resolves on "close", so that its descriptor is
+ // released before the caller gets control back and can delete the file.
+ // A buffer-backed one has no descriptor and yauzl's BufferSlicer never
+ // emits "close", so "end" — every entry read — is the only signal.
+ if (typeof source === "string") {
+ zipfile.on("close", done);
+ } else {
+ zipfile.on("end", done);
}
- );
+
+ zipfile.on("error", (error) => fail(error));
+ zipfile.readEntry();
+ };
+
+ if (typeof source === "string") {
+ yauzl.open(source, options, onOpen);
+ } else {
+ yauzl.fromBuffer(source, options, onOpen);
+ }
});
}
diff --git a/shared/i18n/locales/en_US/translation.json b/shared/i18n/locales/en_US/translation.json
index 2217424f08..1cd7b11c0a 100644
--- a/shared/i18n/locales/en_US/translation.json
+++ b/shared/i18n/locales/en_US/translation.json
@@ -81,6 +81,7 @@
"Download document": "Download document",
"Download as Markdown": "Download as Markdown",
"Download as HTML": "Download as HTML",
+ "Download as TextBundle": "Download as TextBundle",
"Download as PDF": "Download as PDF",
"Copy as Markdown": "Copy as Markdown",
"Markdown copied to clipboard": "Markdown copied to clipboard",
@@ -300,6 +301,7 @@
"Preparing your download": "Preparing your download",
"A file containing the selected documents in Markdown format.": "A file containing the selected documents in Markdown format.",
"A file containing the selected documents in HTML format.": "A file containing the selected documents in HTML format.",
+ "A file containing the selected documents and their images in TextBundle format.": "A file containing the selected documents and their images in TextBundle format.",
"A file containing the selected documents in PDF format.": "A file containing the selected documents in PDF format.",
"Include child documents": "Include child documents",
"When selected, exporting the document {{documentName}} may take some time.": "When selected, exporting the document {{documentName}} may take some time.",
@@ -379,6 +381,7 @@
"Show detail": "Show detail",
"A ZIP file containing the images, and documents in the Markdown format.": "A ZIP file containing the images, and documents in the Markdown format.",
"A ZIP file containing the images, and documents as HTML files.": "A ZIP file containing the images, and documents as HTML files.",
+ "A ZIP file containing each document and its images as a TextBundle.": "A ZIP file containing each document and its images as a TextBundle.",
"Structured data that can be used to transfer data to another compatible {{ appName }} instance.": "Structured data that can be used to transfer data to another compatible {{ appName }} instance.",
"Exporting the collection {{collectionName}} may take some time.": "Exporting the collection {{collectionName}} may take some time.",
"Include attachments": "Include attachments",
diff --git a/shared/types.ts b/shared/types.ts
index b8e37e439e..2be0c35459 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -56,6 +56,7 @@ export enum Client {
export enum ExportContentType {
Markdown = "text/markdown",
Html = "text/html",
+ TextBundle = "application/x-textbundle",
Pdf = "application/pdf",
}
@@ -63,6 +64,7 @@ export enum FileOperationFormat {
JSON = "json",
MarkdownZip = "outline-markdown",
HTMLZip = "html",
+ TextBundleZip = "textbundle",
PDF = "pdf",
Notion = "notion",
}
diff --git a/shared/utils/markdown.test.ts b/shared/utils/markdown.test.ts
new file mode 100644
index 0000000000..977308f054
--- /dev/null
+++ b/shared/utils/markdown.test.ts
@@ -0,0 +1,53 @@
+import { replaceMarkdownLinks } from "./markdown";
+
+describe("replaceMarkdownLinks", () => {
+ const toUpper = (href: string) => href.toUpperCase();
+
+ it("replaces the destination of a link, leaving its label", () => {
+ expect(replaceMarkdownLinks("see [other](./other.md)", toUpper)).toBe(
+ "see [other](./OTHER.MD)"
+ );
+ });
+
+ it("replaces the destination of an image", () => {
+ expect(replaceMarkdownLinks("", toUpper)).toBe(
+ ""
+ );
+ });
+
+ it("preserves a link title", () => {
+ expect(
+ replaceMarkdownLinks('', toUpper)
+ ).toBe('');
+ });
+
+ it("understands an angle bracketed destination", () => {
+ expect(replaceMarkdownLinks("![alt]()", toUpper)).toBe(
+ ""
+ );
+ });
+
+ it("preserves a title after an angle bracketed destination", () => {
+ expect(replaceMarkdownLinks('[x]( "Title")', toUpper)).toBe(
+ '[x](MY DOC.MD "Title")'
+ );
+ });
+
+ it("leaves a link alone when the replacer returns undefined", () => {
+ expect(replaceMarkdownLinks("[x](./y.md)", () => undefined)).toBe(
+ "[x](./y.md)"
+ );
+ });
+
+ it("leaves an unterminated angle bracket alone", () => {
+ expect(replaceMarkdownLinks("[x]( {
+ expect(replaceMarkdownLinks("[a](one.md) and [b](two.md)", toUpper)).toBe(
+ "[a](ONE.MD) and [b](TWO.MD)"
+ );
+ });
+});
diff --git a/shared/utils/markdown.ts b/shared/utils/markdown.ts
index 9565fdfaa0..07566f7010 100644
--- a/shared/utils/markdown.ts
+++ b/shared/utils/markdown.ts
@@ -38,3 +38,57 @@ export const escape = function (text: string) {
export const unescape = function (text: string) {
return text.replace(/\\([\\*+-\d.])/g, "$1");
};
+
+/**
+ * Matches a markdown link or image, capturing the text between its
+ * parentheses. A destination containing an unescaped closing parenthesis is
+ * not matched.
+ */
+const linkRegex = /(!?\[[^\]]*\]\()([^)]*)(\))/g;
+
+/**
+ * Replaces the destination of every markdown link and image in a string.
+ *
+ * Understands the angle-bracket form used when a destination contains spaces,
+ * and preserves any link title.
+ *
+ * @param text The markdown text to rewrite.
+ * @param replace Called with each destination; return the replacement, or
+ * undefined to leave the link as it was written.
+ * @returns The markdown with replaced destinations.
+ */
+export function replaceMarkdownLinks(
+ text: string,
+ replace: (href: string) => string | undefined
+): string {
+ return text.replace(
+ linkRegex,
+ (match, prefix: string, target: string, suffix: string) => {
+ const leading = target.length - target.trimStart().length;
+ const trimmed = target.trim();
+
+ let href: string;
+ let title: string;
+
+ if (trimmed.startsWith("<")) {
+ const end = trimmed.indexOf(">");
+ if (end === -1) {
+ return match;
+ }
+ href = trimmed.slice(1, end);
+ title = target.slice(leading + end + 1);
+ } else {
+ [href] = trimmed.split(/\s/, 1);
+ if (!href) {
+ return match;
+ }
+ title = target.slice(leading + href.length);
+ }
+
+ const replacement = replace(href);
+ return replacement === undefined
+ ? match
+ : `${prefix}${replacement}${title}${suffix}`;
+ }
+ );
+}
diff --git a/shared/utils/parseTitle.ts b/shared/utils/parseTitle.ts
index 759345bb13..e6a9adebd3 100644
--- a/shared/utils/parseTitle.ts
+++ b/shared/utils/parseTitle.ts
@@ -1,23 +1,36 @@
import emojiRegex from "emoji-regex";
import { unescape } from "./markdown";
-export default function parseTitle(text = "") {
- const regex = emojiRegex();
+/**
+ * Splits a leading emoji from the start of a string.
+ *
+ * @param text The text to split.
+ * @returns The leading emoji, if there is one, and the remaining text.
+ */
+export function splitLeadingEmoji(text: string): {
+ emoji?: string;
+ rest: string;
+} {
+ const matches = emojiRegex().exec(text);
+ const firstEmoji = matches ? matches[0] : null;
+ if (!firstEmoji || !text.startsWith(firstEmoji)) {
+ return { rest: text };
+ }
+
+ return {
+ emoji: firstEmoji,
+ rest: text.slice(firstEmoji.length).trim(),
+ };
+}
+
+export default function parseTitle(text = "") {
// find and extract title
const firstLine = text.trim().split(/\r?\n/)[0];
const title = unescape(firstLine.replace(/^#/, "").trim());
// find and extract first emoji
- const matches = regex.exec(title);
- const firstEmoji = matches ? matches[0] : null;
- const startsWithEmoji = firstEmoji && title.startsWith(firstEmoji);
- const emoji = startsWithEmoji ? firstEmoji : undefined;
-
- // title with first leading emoji stripped
- const strippedTitle = startsWithEmoji
- ? title.replace(firstEmoji, "").trim()
- : title;
+ const { emoji, rest: strippedTitle } = splitLeadingEmoji(title);
return {
title,