fix: only highlight the closest list in the formatting toolbar

When the selection sits inside a nested list, the toolbar previously marked
both the inner list and any ancestor list of a different type as active,
because each list button walked the full ancestor chain. Highlight only the
list closest to the selection.

https://claude.ai/code/session_01BGH191WyuL9SgQyjfJ3YsD

app/editor/menus/formatting.tsx

@@ -29,6 +29,7 @@ import HighlightColorPicker from "../components/HighlightColorPicker";
 import { getDocumentHighlightColors } from "@shared/editor/queries/getDocumentHighlightColors";
 import { getMarksBetween } from "@shared/editor/queries/getMarksBetween";
 import { isInList } from "@shared/editor/queries/isInList";
+import { isListActive } from "@shared/editor/queries/isListActive";
 import { isMarkActive } from "@shared/editor/queries/isMarkActive";
 import { isNodeActive } from "@shared/editor/queries/isNodeActive";
 import type { MenuItem, SelectionContext } from "@shared/editor/types";
@@ -384,7 +385,7 @@ export default function formattingMenuItems(ctx: SelectionContext): MenuItem[] {
       shortcut: `⇧+Ctrl+7`,
       icon: <TodoListIcon />,
       keywords: "checklist checkbox task",
-      active: isNodeActive(schema.nodes.checkbox_list),
+      active: isListActive(schema.nodes.checkbox_list),
       visible: !isInCodeBlock && !isTableCell && (!isList || !isTouch),
     },
     {
@@ -392,7 +393,7 @@ export default function formattingMenuItems(ctx: SelectionContext): MenuItem[] {
       tooltip: t("Bulleted list"),
       shortcut: `⇧+Ctrl+8`,
       icon: <BulletedListIcon />,
-      active: isNodeActive(schema.nodes.bullet_list),
+      active: isListActive(schema.nodes.bullet_list),
       visible: !isInCodeBlock && !isTableCell && (!isList || !isTouch),
     },
     {
@@ -400,7 +401,7 @@ export default function formattingMenuItems(ctx: SelectionContext): MenuItem[] {
       tooltip: t("Ordered list"),
       shortcut: `⇧+Ctrl+9`,
       icon: <OrderedListIcon />,
-      active: isNodeActive(schema.nodes.ordered_list),
+      active: isListActive(schema.nodes.ordered_list),
       visible: !isInCodeBlock && !isTableCell && (!isList || !isTouch),
     },
     {

shared/editor/queries/isListActive.test.ts

@@ -0,0 +1,106 @@
+import type { Node } from "prosemirror-model";
+import {
+  createEditorStateWithSelection,
+  doc,
+  p,
+  schema,
+} from "@shared/test/editor";
+import { isListActive } from "./isListActive";
+
+const { bullet_list, ordered_list, list_item, checkbox_list, checkbox_item } =
+  schema.nodes;
+
+/**
+ * Creates a list item node with the given block content.
+ */
+function li(content: Node[]) {
+  return list_item.create(null, content);
+}
+
+/**
+ * Creates a checkbox item node with the given block content.
+ */
+function cli(content: Node[]) {
+  return checkbox_item.create(null, content);
+}
+
+/**
+ * Returns a position inside the first text node matching the given text.
+ *
+ * @throws if no matching text node exists in the document.
+ */
+function posOfText(node: Node, text: string) {
+  let found = -1;
+  node.descendants((child, pos) => {
+    if (found === -1 && child.isText && child.text === text) {
+      found = pos;
+    }
+    return found === -1;
+  });
+  if (found === -1) {
+    throw new Error(`Text "${text}" not found in document`);
+  }
+  return found + 1;
+}
+
+/**
+ * Builds an editor state with the selection placed inside the given text.
+ */
+function stateAt(testDoc: Node, selectionText: string) {
+  return createEditorStateWithSelection(
+    testDoc,
+    posOfText(testDoc, selectionText)
+  );
+}
+
+describe("isListActive", () => {
+  it("matches the closest list type", () => {
+    const testDoc = doc([
+      bullet_list.create(null, [li([p("one")]), li([p("two")])]),
+    ]);
+    const state = stateAt(testDoc, "one");
+
+    expect(isListActive(bullet_list)(state)).toBe(true);
+    expect(isListActive(ordered_list)(state)).toBe(false);
+    expect(isListActive(checkbox_list)(state)).toBe(false);
+  });
+
+  it("does not mark an ancestor list of a different type as active", () => {
+    // An ordered list nested inside a checkbox list, selection in the
+    // ordered list item.
+    const testDoc = doc([
+      checkbox_list.create(null, [
+        cli([
+          p("moo"),
+          ordered_list.create(null, [li([p("dfsdf")]), li([p("sd")])]),
+        ]),
+      ]),
+    ]);
+    const state = stateAt(testDoc, "dfsdf");
+
+    expect(isListActive(ordered_list)(state)).toBe(true);
+    expect(isListActive(checkbox_list)(state)).toBe(false);
+    expect(isListActive(bullet_list)(state)).toBe(false);
+  });
+
+  it("matches the parent list when the selection is in the parent item", () => {
+    const testDoc = doc([
+      checkbox_list.create(null, [
+        cli([p("moo"), ordered_list.create(null, [li([p("dfsdf")])])]),
+      ]),
+    ]);
+    const state = stateAt(testDoc, "moo");
+
+    expect(isListActive(checkbox_list)(state)).toBe(true);
+    expect(isListActive(ordered_list)(state)).toBe(false);
+  });
+
+  it("returns false when the selection is not in a list", () => {
+    const testDoc = doc([p("hello")]);
+    const state = stateAt(testDoc, "hello");
+
+    expect(isListActive(bullet_list)(state)).toBe(false);
+    expect(isListActive(ordered_list)(state)).toBe(false);
+    expect(isListActive(checkbox_list)(state)).toBe(false);
+  });
+});

shared/editor/queries/isListActive.ts

@@ -0,0 +1,22 @@
+import type { NodeType } from "prosemirror-model";
+import type { EditorState } from "prosemirror-state";
+import { findParentNode } from "./findParentNode";
+import { isList } from "./isList";
+
+/**
+ * Checks whether the list closest to the current selection is of the given
+ * type. Unlike isNodeActive, this only matches the innermost list so that a
+ * nested list does not also mark an ancestor list of a different type as
+ * active in the toolbar.
+ *
+ * @param type the list node type to check for.
+ * @returns a function that returns true when the closest list is of the type.
+ */
+export const isListActive =
+  (type: NodeType) =>
+  (state: EditorState): boolean => {
+    const closestList = findParentNode((node) => isList(node, state.schema))(
+      state.selection
+    );
+    return closestList?.node.type === type;
+  };