fix: OOM on large markdown imports (#13086)

fix: Markdown import does no truncate titles

server/queues/tasks/MarkdownAPIImportTask.ts

@@ -1,9 +1,9 @@
 import path from "node:path";
 import { randomUUID } from "node:crypto";
-import { escapeRegExp } from "es-toolkit/compat";
+import { escapeRegExp, truncate } from "es-toolkit/compat";
 import mime from "mime-types";
 import { UniqueConstraintError } from "sequelize";
-import { DocumentValidation } from "@shared/validations";
+import { CollectionValidation, DocumentValidation } from "@shared/validations";
 import type {
   ImportTaskInput,
   ImportTaskOutput,
@@ -289,12 +289,23 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
 
       const collections: DiscoveredCollection[] = [];
       const manifest: MarkdownAttachmentManifestItem[] = [];
+      // Loose markdown files that sit at the archive root rather than inside a
+      // collection folder are gathered into a fallback collection below.
+      const looseDocuments: ZipTreeNode[] = [];
 
       for (const node of rootNodes) {
         if (node.children.length === 0) {
-          Logger.debug("task", `Unhandled file in zip: ${node.pathInZip}`, {
-            importTaskId: importTask.id,
-          });
+          const ext = path.extname(node.name).toLowerCase();
+          if (ext === ".md" || ext === ".markdown") {
+            looseDocuments.push(node);
+          } else {
+            manifest.push({
+              id: randomUUID(),
+              name: node.name,
+              pathInZip: node.pathInZip,
+              mimeType: mime.lookup(node.name) || "application/octet-stream",
+            });
+          }
           continue;
         }
 
@@ -319,6 +330,23 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
         });
       }
 
+      if (looseDocuments.length > 0) {
+        const collection: DiscoveredCollection = {
+          id: randomUUID(),
+          title: this.fallbackCollectionTitle(),
+          children: [],
+        };
+        collections.push(collection);
+
+        this.collectDocumentsAndAttachments({
+          children: looseDocuments,
+          collectionId: collection.id,
+          out: collection.children,
+          manifest,
+          markdownByNode,
+        });
+      }
+
       // Build docMap (pathInZip -> externalId) for internal-link resolution.
       // Walk the full document tree to collect every doc id, since internal
       // markdown links can target any document regardless of depth.
@@ -347,7 +375,8 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
           permission: placeholder.permission,
         })),
       ];
-      associatedImport.scratch = { storageKey, manifest };
+
+      associatedImport.scratch = { storageKey, manifest, docMap };
       await associatedImport.save();
 
       // Append collection placeholder items so ImportsProcessor iterates
@@ -361,14 +390,15 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
           path: c.title,
           markdownText: "",
           attachmentMap: [],
-          docMap: {},
         }));
 
       importTask.input = [importTask.input[0], ...collectionInputItems];
 
       const collectionOutputs: ImportTaskOutput = collections.map((c) => ({
         externalId: c.id,
-        title: c.title,
+        title: truncate(c.title, {
+          length: CollectionValidation.maxNameLength,
+        }),
         content: ProsemirrorDataHelper.getEmpty() as ProsemirrorDoc,
       }));
 
@@ -378,7 +408,7 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
       // depth-ordered cascade of ImportTask rows so parent FKs are always
       // satisfied at child-doc creation time.
       const childTasksInput: ImportTaskInput<Markdown> = collections.flatMap(
-        (c) => c.children.map((d) => this.toPageInput(d, manifest, docMap))
+        (c) => c.children.map((d) => this.toPageInput(d, manifest))
       );
 
       return { taskOutput: collectionOutputs, childTasksInput };
@@ -394,13 +424,11 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
    *
    * @param doc The discovered document, including its descendants.
    * @param manifest The full attachment manifest (used for per-page refs).
-   * @param docMap Path → externalId map for internal link rewriting.
    * @returns A self-contained per-page task input.
    */
   private toPageInput(
     doc: DiscoveredDocument,
-    manifest: MarkdownAttachmentManifestItem[],
-    docMap: Record<string, string>
+    manifest: MarkdownAttachmentManifestItem[]
   ): MarkdownPageImportTaskInputItem {
     return {
       externalId: doc.id,
@@ -410,9 +438,8 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
       path: doc.pathInZip,
       markdownText: doc.markdownText,
       attachmentMap: this.attachmentsReferencedBy(doc.markdownText, manifest),
-      docMap,
       children: doc.children.length
-        ? doc.children.map((c) => this.toPageInput(c, manifest, docMap))
+        ? doc.children.map((c) => this.toPageInput(c, manifest))
         : undefined,
     };
   }
@@ -423,6 +450,10 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
     const taskOutput: ImportTaskOutput = [];
     const childTasksInput: MarkdownPageImportTaskInputItem[] = [];
 
+    // docMap is stored once on scratch rather than embedded in each page's
+    // input (see processBootstrap) — read it back here for link rewriting.
+    const docMap = importTask.import.scratch?.docMap ?? {};
+
     const items = importTask.input as MarkdownPageImportTaskInputItem[];
     for (const item of items) {
       // Empty markdown short-circuits — used by collection placeholders so
@@ -433,11 +464,13 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
       if (!item.markdownText) {
         taskOutput.push({
           externalId: item.externalId,
-          title: item.title,
+          title: truncate(item.title, {
+            length: DocumentValidation.maxTitleLength,
+          }),
           content: ProsemirrorDataHelper.getEmpty() as ProsemirrorDoc,
         });
       } else {
-        const transformedMarkdown = this.rewriteMarkdown(item);
+        const transformedMarkdown = this.rewriteMarkdown(item, docMap);
         const { doc, title, icon } = await DocumentConverter.convert(
           transformedMarkdown,
           path.basename(item.path),
@@ -447,7 +480,9 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
 
         taskOutput.push({
           externalId: item.externalId,
-          title: title || item.title,
+          title: truncate(title || item.title, {
+            length: DocumentValidation.maxTitleLength,
+          }),
           icon,
           content: doc.toJSON() as ProsemirrorDoc,
         });
@@ -475,10 +510,15 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
    * an angle-bracket-wrapped URL (which produced broken image src attrs).
    *
    * @param page The per-page task input.
+   * @param docMap Path → externalId map for internal link rewriting, shared
+   *               across all pages via `Import.scratch`.
    * @returns Rewritten markdown text ready for DocumentConverter.
    */
-  private rewriteMarkdown(page: MarkdownPageImportTaskInputItem): string {
-    let text = rewriteInternalLinks(page.markdownText, page.path, page.docMap);
+  private rewriteMarkdown(
+    page: MarkdownPageImportTaskInputItem,
+    docMap: Record<string, string>
+  ): string {
+    let text = rewriteInternalLinks(page.markdownText, page.path, docMap);
 
     // Convert `[label](<<id>>)` links from rewriteInternalLinks into mention
     // markdown the editor recognises: `@[label](mention://<uuid>/document/<id>)`.
@@ -534,6 +574,17 @@ export default class MarkdownAPIImportTask extends APIImportTask<Markdown> {
     return nodes;
   }
 
+  /**
+   * Title for the fallback collection that holds markdown files found loose at
+   * the archive root (outside any collection folder). Subclasses can override
+   * to give the collection a source-appropriate name.
+   *
+   * @returns The fallback collection title.
+   */
+  protected fallbackCollectionTitle(): string {
+    return "Documents";
+  }
+
   /**
    * Detects folders containing only attachments (no markdown documents).
    * Recursively considers nested folders; mirrors the legacy heuristic.

shared/schema.ts

@@ -92,6 +92,7 @@ export type MarkdownAttachmentManifestItem = z.infer<
 export interface MarkdownImportScratch {
   storageKey: string;
   manifest?: MarkdownAttachmentManifestItem[];
+  docMap?: Record<string, string>;
 }
 
 /**
@@ -157,7 +158,6 @@ export interface MarkdownPageImportTaskInputItem {
   path: string;
   markdownText: string;
   attachmentMap: MarkdownAttachmentManifestItem[];
-  docMap: Record<string, string>;
   children?: MarkdownPageImportTaskInputItem[];
 }