perf: Refactor to run boot migration check as separate process (#13049)

* perf: Refactor to run boot migration check as separate process

* Apply suggestions from code review

server/index.ts

@@ -1,41 +1,16 @@
 /* oxlint-disable @typescript-eslint/no-misused-promises */
 /* oxlint-disable import/order */
-import { toError } from "@shared/utils/error";
-import env from "./env";
-
 import "./logging/tracer"; // must come before importing any instrumented module
 
-import http from "node:http";
-import https from "node:https";
 import os from "node:os";
-import type { Context } from "koa";
-import Koa from "koa";
-import helmet from "koa-helmet";
-import logger from "koa-logger";
-import Router from "koa-router";
-import type { AddressInfo } from "node:net";
-import stoppable from "stoppable";
 import throng from "throng";
-import { escape } from "es-toolkit/compat";
+import env from "./env";
 import Logger from "./logging/Logger";
-import services from "./services";
-import { getArg } from "./utils/args";
-import { getSSLOptions } from "./utils/ssl";
-import { defaultRateLimiter } from "@server/middlewares/rateLimiter";
 import {
   printEnv,
   checkPendingMigrations,
   configureChildHeapLimit,
 } from "./utils/startup";
-import { checkUpdates } from "./utils/updates";
-import onerror from "./onerror";
-import ShutdownHelper, { ShutdownOrder } from "./utils/ShutdownHelper";
-import { checkConnection, sequelize } from "./storage/database";
-import Redis from "@server/storage/redis";
-import Metrics from "@server/logging/Metrics";
-import { CacheHelper } from "./utils/CacheHelper";
-import { RedisPrefixHelper } from "./utils/RedisPrefixHelper";
-import { PluginManager } from "./utils/PluginManager";
 
 // The number of processes to run, defaults to the number of CPU's available
 // for the web service, and 1 for collaboration unless REDIS_COLLABORATION_URL is set.
@@ -52,211 +27,12 @@ if (env.SERVICES.includes("collaboration") && !env.REDIS_COLLABORATION_URL) {
   webProcessCount = 1;
 }
 
-// This function will only be called once in the original process
+// This function will only be called once in the original process. The database
+// connection and migration checks run in a short-lived child process so that
+// the long-lived master never loads the (heavy) Sequelize models graph.
 async function master() {
-  await checkConnection(sequelize);
   await checkPendingMigrations();
   await printEnv();
-
-  if (env.TELEMETRY && env.isProduction) {
-    void checkUpdates();
-    setInterval(checkUpdates, 24 * 3600 * 1000).unref();
-  }
-}
-
-// This function will only be called in each forked process
-async function start(_id: number, disconnect: () => void) {
-  // Ensure plugins are loaded
-  PluginManager.loadPlugins();
-
-  // Clear unfurl cache in development so code changes take effect immediately
-  if (env.isDevelopment) {
-    void CacheHelper.clearData(RedisPrefixHelper.getUnfurlKey(""));
-  }
-
-  // Find if SSL certs are available
-  const ssl = getSSLOptions();
-  const useHTTPS = !!ssl.key && !!ssl.cert;
-
-  // If a --port flag is passed then it takes priority over the env variable
-  const normalizedPort = getArg("port", "p") || env.PORT;
-  const app = new Koa();
-  const server = stoppable(
-    useHTTPS
-      ? https.createServer(ssl, app.callback())
-      : http.createServer(app.callback()),
-    ShutdownHelper.connectionGraceTimeout
-  );
-  const router = new Router();
-
-  // install basic middleware shared by all services
-  if (env.DEBUG.includes("http")) {
-    app.use(logger((str) => Logger.info("http", str)));
-  }
-
-  app.use(helmet());
-
-  // catch errors in one place, automatically set status and response headers
-  onerror(app);
-
-  // Apply default rate limit to all routes
-  app.use(defaultRateLimiter());
-
-  /** Perform a redirect on the browser so that the user's auth cookies are included in the request. */
-  app.context.redirectOnClient = function (
-    this: Context,
-    /** The URL to redirect to */
-    url: string,
-    /**
-     * The HTTP method to use for the redirect. Use POST when preventing links in emails from being
-     * clicked by bots. Otherwise, use GET.
-     */
-    method: "GET" | "POST" = "GET"
-  ) {
-    this.type = "text/html";
-
-    if (method === "POST") {
-      // For POST method, create a form that auto-submits
-      const urlObj = new URL(url);
-      const formAction = `${urlObj.origin}${urlObj.pathname}`;
-      const searchParams = urlObj.searchParams;
-
-      let formFields = "";
-      searchParams.forEach((value, key) => {
-        formFields += `<input type="hidden" name="${escape(
-          key
-        )}" value="${escape(value)}" />`;
-      });
-
-      if (this.userAgent.isBot) {
-        formFields += `
-          <p>If you are not redirected automatically, please click the button below.</p>
-          <input type="submit" value="Continue" />
-        `;
-      }
-
-      this.body = `
-<html lang="en">
-<head>
-  <title>Redirecting…</title>
-</head>
-<body>
-  <form id="redirect-form" method="POST" action="${formAction}">
-    ${formFields}
-  </form>
-  <script nonce="${this.state.cspNonce}">
-    ${!this.userAgent.isBot} && document.getElementById('redirect-form').submit();
-  </script>
-</body>
-</html>`;
-    } else {
-      // Default GET method using meta refresh
-      this.body = `
-<html lang="en">
-<head>
-<meta http-equiv="refresh" content="0;URL='${escape(url)}'" />
-</head>
-</html>`;
-    }
-  };
-
-  // Add a health check endpoint to all services
-  router.get("/_health", async (ctx) => {
-    try {
-      await sequelize.query("SELECT 1");
-    } catch (err) {
-      Logger.error("Database connection failed", toError(err));
-      ctx.status = 500;
-      return;
-    }
-
-    try {
-      await Redis.defaultClient.ping();
-    } catch (err) {
-      Logger.error("Redis ping failed", toError(err));
-      ctx.status = 500;
-      return;
-    }
-
-    ctx.body = "OK";
-  });
-
-  app.use(router.routes());
-
-  // loop through requested services at startup
-  for (const name of env.SERVICES) {
-    if (!Object.keys(services).includes(name)) {
-      throw new Error(`Unknown service ${name}`);
-    }
-
-    Logger.info("lifecycle", `Starting ${name} service`);
-    const { default: init } = await services[name as keyof typeof services]();
-    await Promise.resolve(init(app, server as https.Server, env.SERVICES));
-  }
-
-  server.on("error", (err) => {
-    if ("code" in err && err.code === "EADDRINUSE") {
-      Logger.error(`Port ${normalizedPort} is already in use. Exiting…`, err);
-      process.exit(0);
-    }
-
-    if ("code" in err && err.code === "EACCES") {
-      Logger.error(
-        `Port ${normalizedPort} requires elevated privileges. Exiting…`,
-        err
-      );
-      process.exit(0);
-    }
-
-    throw err;
-  });
-  server.on("listening", () => {
-    const address = server.address();
-    const port = (address as AddressInfo).port;
-
-    Logger.info(
-      "lifecycle",
-      `Listening on ${useHTTPS ? "https" : "http"}://localhost:${port} / ${
-        env.URL
-      }`
-    );
-  });
-
-  server.listen(normalizedPort);
-  server.setTimeout(env.REQUEST_TIMEOUT);
-
-  ShutdownHelper.add(
-    "server",
-    ShutdownOrder.last,
-    () =>
-      new Promise((resolve, reject) => {
-        // Calling stop prevents new connections from being accepted and waits for
-        // existing connections to close for the grace period before forcefully
-        // closing them.
-        server.stop((err, gracefully) => {
-          disconnect();
-
-          if (err) {
-            reject(err);
-          } else {
-            resolve(gracefully);
-          }
-        });
-      })
-  );
-
-  ShutdownHelper.add("metrics", ShutdownOrder.last, () => Metrics.flush());
-
-  // Handle uncaught promise rejections
-  process.on("unhandledRejection", (error: Error) => {
-    Logger.error("Unhandled promise rejection", error, {
-      stack: error.stack,
-    });
-  });
-
-  // Handle shutdown signals
-  process.once("SIGTERM", () => ShutdownHelper.execute());
-  process.once("SIGINT", () => ShutdownHelper.execute());
 }
 
 const isWebProcess =
@@ -279,6 +55,9 @@ configureChildHeapLimit(processCount);
 
 void throng({
   master,
-  worker: start,
+  // The worker's heavy dependency graph is loaded lazily so it never enters the
+  // master process, which only supervises the forked workers.
+  worker: (id, disconnect) =>
+    import("./main").then(({ start }) => start(id, disconnect)),
   count: isWorkerProcess ? 1 : isWebProcess ? webProcessCount : undefined,
 });

server/main.ts

@@ -0,0 +1,181 @@
+/* oxlint-disable @typescript-eslint/no-misused-promises */
+import http from "node:http";
+import https from "node:https";
+import type { AddressInfo } from "node:net";
+import Koa from "koa";
+import helmet from "koa-helmet";
+import logger from "koa-logger";
+import Router from "koa-router";
+import stoppable from "stoppable";
+import { toError } from "@shared/utils/error";
+import env from "./env";
+import Metrics from "@server/logging/Metrics";
+import Redis from "@server/storage/redis";
+import Logger from "./logging/Logger";
+import { defaultRateLimiter } from "@server/middlewares/rateLimiter";
+import onerror from "./onerror";
+import services from "./services";
+import { sequelize } from "./storage/database";
+import { getArg } from "./utils/args";
+import { CacheHelper } from "./utils/CacheHelper";
+import { PluginManager } from "./utils/PluginManager";
+import { redirectOnClient } from "./utils/redirectOnClient";
+import { RedisPrefixHelper } from "./utils/RedisPrefixHelper";
+import ShutdownHelper, { ShutdownOrder } from "./utils/ShutdownHelper";
+import { getSSLOptions } from "./utils/ssl";
+import { checkUpdates } from "./utils/updates";
+
+/**
+ * Starts a single forked service process. This is where the heavy dependency
+ * graph (Sequelize models, services, middleware) is loaded, so it is imported
+ * dynamically from the entry point to keep the supervising master process lean.
+ *
+ * @param id the cluster worker id assigned by throng.
+ * @param disconnect callback used to signal the worker has stopped.
+ */
+export async function start(id: number, disconnect: () => void) {
+  // Ensure plugins are loaded
+  PluginManager.loadPlugins();
+
+  // Clear unfurl cache in development so code changes take effect immediately
+  if (env.isDevelopment) {
+    void CacheHelper.clearData(RedisPrefixHelper.getUnfurlKey(""));
+  }
+
+  // Find if SSL certs are available
+  const ssl = getSSLOptions();
+  const useHTTPS = !!ssl.key && !!ssl.cert;
+
+  // If a --port flag is passed then it takes priority over the env variable
+  const normalizedPort = getArg("port", "p") || env.PORT;
+  const app = new Koa();
+  const server = stoppable(
+    useHTTPS
+      ? https.createServer(ssl, app.callback())
+      : http.createServer(app.callback()),
+    ShutdownHelper.connectionGraceTimeout
+  );
+  const router = new Router();
+
+  // install basic middleware shared by all services
+  if (env.DEBUG.includes("http")) {
+    app.use(logger((str) => Logger.info("http", str)));
+  }
+
+  app.use(helmet());
+
+  // catch errors in one place, automatically set status and response headers
+  onerror(app);
+
+  // Apply default rate limit to all routes
+  app.use(defaultRateLimiter());
+
+  // Allow browser-side redirects that include the user's auth cookies.
+  app.context.redirectOnClient = redirectOnClient;
+
+  // Add a health check endpoint to all services
+  router.get("/_health", async (ctx) => {
+    try {
+      await sequelize.query("SELECT 1");
+    } catch (err) {
+      Logger.error("Database connection failed", toError(err));
+      ctx.status = 500;
+      return;
+    }
+
+    try {
+      await Redis.defaultClient.ping();
+    } catch (err) {
+      Logger.error("Redis ping failed", toError(err));
+      ctx.status = 500;
+      return;
+    }
+
+    ctx.body = "OK";
+  });
+
+  app.use(router.routes());
+
+  // loop through requested services at startup
+  for (const name of env.SERVICES) {
+    if (!Object.keys(services).includes(name)) {
+      throw new Error(`Unknown service ${name}`);
+    }
+
+    Logger.info("lifecycle", `Starting ${name} service`);
+    const { default: init } = await services[name as keyof typeof services]();
+    await Promise.resolve(init(app, server as https.Server, env.SERVICES));
+  }
+
+  server.on("error", (err) => {
+    if ("code" in err && err.code === "EADDRINUSE") {
+      Logger.error(`Port ${normalizedPort} is already in use. Exiting…`, err);
+      process.exit(1);
+    }
+
+    if ("code" in err && err.code === "EACCES") {
+      Logger.error(
+        `Port ${normalizedPort} requires elevated privileges. Exiting…`,
+        err
+      );
+      process.exit(1);
+    }
+
+    throw err;
+  });
+  server.on("listening", () => {
+    const address = server.address();
+    const port = (address as AddressInfo).port;
+
+    Logger.info(
+      "lifecycle",
+      `Listening on ${useHTTPS ? "https" : "http"}://localhost:${port} / ${
+        env.URL
+      }`
+    );
+  });
+
+  server.listen(normalizedPort);
+  server.setTimeout(env.REQUEST_TIMEOUT);
+
+  // Run telemetry from a single worker only. This keeps the supervising master
+  // process free of the models graph that the update check requires, while
+  // avoiding duplicate reporting when multiple workers are running.
+  if (id === 1 && env.TELEMETRY && env.isProduction) {
+    void checkUpdates();
+    setInterval(checkUpdates, 24 * 3600 * 1000).unref();
+  }
+
+  ShutdownHelper.add(
+    "server",
+    ShutdownOrder.last,
+    () =>
+      new Promise((resolve, reject) => {
+        // Calling stop prevents new connections from being accepted and waits for
+        // existing connections to close for the grace period before forcefully
+        // closing them.
+        server.stop((err, gracefully) => {
+          disconnect();
+
+          if (err) {
+            reject(err);
+          } else {
+            resolve(gracefully);
+          }
+        });
+      })
+  );
+
+  ShutdownHelper.add("metrics", ShutdownOrder.last, () => Metrics.flush());
+
+  // Handle uncaught promise rejections
+  process.on("unhandledRejection", (error: Error) => {
+    Logger.error("Unhandled promise rejection", error, {
+      stack: error.stack,
+    });
+  });
+
+  // Handle shutdown signals
+  process.once("SIGTERM", () => ShutdownHelper.execute());
+  process.once("SIGINT", () => ShutdownHelper.execute());
+}

server/scripts/checkMigrations.ts

@@ -0,0 +1,107 @@
+import "./bootstrap";
+import { styleText } from "node:util";
+import { isEmpty } from "es-toolkit/compat";
+import { toError, errToString } from "@shared/utils/error";
+import { Minute } from "@shared/utils/time";
+import env from "@server/env";
+import Logger from "@server/logging/Logger";
+import AuthenticationProvider from "@server/models/AuthenticationProvider";
+import Team from "@server/models/Team";
+import {
+  checkConnection,
+  migrations,
+  sequelize,
+} from "@server/storage/database";
+import { getArg } from "@server/utils/args";
+import { MutexLock } from "@server/utils/MutexLock";
+import ShutdownHelper from "@server/utils/ShutdownHelper";
+
+/**
+ * Checks for pending database migrations and runs them, unless the
+ * --no-migrate flag was passed in which case the process exits with an error.
+ */
+async function checkPendingMigrations() {
+  let lock;
+  try {
+    lock = await MutexLock.acquire("migrations", 10 * Minute.ms, {
+      releaseOnShutdown: true,
+    });
+
+    const pending = await migrations.pending();
+    if (!isEmpty(pending)) {
+      if (getArg("no-migrate")) {
+        Logger.fatal(
+          styleText(
+            "red",
+            `Database migrations are pending and were not run because the --no-migrate flag was passed.\nRun the migrations with "yarn db:migrate".`
+          ),
+          new Error("Migrations pending")
+        );
+      } else {
+        Logger.info("database", "Running migrations…");
+        await migrations.up();
+      }
+    }
+    await checkDataMigrations();
+  } catch (err) {
+    const message = errToString(err);
+    const error = toError(err);
+    if (message.includes("ECONNREFUSED")) {
+      Logger.fatal(
+        styleText(
+          "red",
+          `Could not connect to the database. Please check your connection settings.`
+        ),
+        error
+      );
+    } else {
+      Logger.fatal(styleText("red", message), error);
+    }
+  } finally {
+    if (lock) {
+      await MutexLock.release(lock);
+    }
+  }
+}
+
+/**
+ * Checks whether a required data migration has been completed for self-hosted
+ * installations, exiting the process with instructions if it has not.
+ */
+async function checkDataMigrations() {
+  if (env.isCloudHosted) {
+    return;
+  }
+
+  const team = await Team.findOne();
+  const provider = await AuthenticationProvider.findOne();
+
+  if (
+    env.isProduction &&
+    team &&
+    team.createdAt < new Date("2024-01-01") &&
+    !provider
+  ) {
+    Logger.fatal(
+      `
+This version of Outline cannot start until a data migration is complete.
+Backup your database, run the database migrations and the following script:
+(Note: script run needed only when upgrading to any version between 0.54.0 and 0.61.1, including both)
+
+$ node ./build/server/scripts/20210226232041-migrate-authentication.js
+`,
+      new Error("Data migration required")
+    );
+  }
+}
+
+void (async () => {
+  await checkConnection(sequelize);
+  await checkPendingMigrations();
+
+  // Skip the success exit when a Logger.fatal above is already shutting the
+  // process down with a non-zero code.
+  if (!ShutdownHelper.isShuttingDown) {
+    process.exit(0);
+  }
+})();

server/utils/ShutdownHelper.ts

@@ -36,11 +36,14 @@ export default class ShutdownHelper {
    */
   public static readonly forceQuitTimeout = 60 * 1000;
 
-  /** Whether the server is currently shutting down */
-  private static isShuttingDown = false;
-
-  /** List of shutdown handlers to execute */
-  private static handlers: Handler[] = [];
+  /**
+   * Whether the process is currently shutting down.
+   *
+   * @returns true once `execute` has been called.
+   */
+  public static get isShuttingDown() {
+    return this.shuttingDown;
+  }
 
   /**
    * Add a shutdown handler to be executed when the process is exiting
@@ -71,10 +74,10 @@ export default class ShutdownHelper {
    * @param code The exit code to use
    */
   public static async execute(code = 0) {
-    if (this.isShuttingDown) {
+    if (this.shuttingDown) {
       return;
     }
-    this.isShuttingDown = true;
+    this.shuttingDown = true;
 
     // Start the shutdown timer
     void sleep(this.forceQuitTimeout).then(() => {
@@ -111,4 +114,10 @@ export default class ShutdownHelper {
     Logger.info("lifecycle", "Gracefully quitting");
     process.exit(code);
   }
+
+  /** Whether the server is currently shutting down */
+  private static shuttingDown = false;
+
+  /** List of shutdown handlers to execute */
+  private static handlers: Handler[] = [];
 }

server/utils/redirectOnClient.ts

@@ -0,0 +1,62 @@
+import { escape } from "es-toolkit/compat";
+import type { Context } from "koa";
+
+/**
+ * Performs a redirect on the browser so that the user's auth cookies are
+ * included in the request. Assigned to the Koa context as `redirectOnClient`.
+ *
+ * @param url the URL to redirect to.
+ * @param method the HTTP method to use for the redirect. Use POST when
+ * preventing links in emails from being clicked by bots. Otherwise, use GET.
+ */
+export function redirectOnClient(
+  this: Context,
+  url: string,
+  method: "GET" | "POST" = "GET"
+) {
+  this.type = "text/html";
+
+  if (method === "POST") {
+    // For POST method, create a form that auto-submits
+    const urlObj = new URL(url);
+    const formAction = `${urlObj.origin}${urlObj.pathname}`;
+    const searchParams = urlObj.searchParams;
+
+    let formFields = "";
+    searchParams.forEach((value, key) => {
+      formFields += `<input type="hidden" name="${escape(
+        key
+      )}" value="${escape(value)}" />`;
+    });
+
+    if (this.userAgent.isBot) {
+      formFields += `
+          <p>If you are not redirected automatically, please click the button below.</p>
+          <input type="submit" value="Continue" />
+        `;
+    }
+
+    this.body = `
+<html lang="en">
+<head>
+  <title>Redirecting…</title>
+</head>
+<body>
+  <form id="redirect-form" method="POST" action="${formAction}">
+    ${formFields}
+  </form>
+  <script nonce="${this.state.cspNonce}">
+    ${!this.userAgent.isBot} && document.getElementById('redirect-form').submit();
+  </script>
+</body>
+</html>`;
+  } else {
+    // Default GET method using meta refresh
+    this.body = `
+<html lang="en">
+<head>
+<meta http-equiv="refresh" content="0;URL='${escape(url)}'" />
+</head>
+</html>`;
+  }
+}

server/utils/startup.ts

@@ -1,16 +1,10 @@
+import { fork } from "node:child_process";
 import cluster from "node:cluster";
 import os from "node:os";
+import path from "node:path";
 import { styleText } from "node:util";
-import { isEmpty } from "es-toolkit/compat";
-import { toError, errToString } from "@shared/utils/error";
 import env from "@server/env";
 import Logger from "@server/logging/Logger";
-import AuthenticationProvider from "@server/models/AuthenticationProvider";
-import Team from "@server/models/Team";
-import { migrations } from "@server/storage/database";
-import { getArg } from "./args";
-import { MutexLock } from "./MutexLock";
-import { Minute } from "@shared/utils/time";
 
 /**
  * Applies a default V8 heap limit to forked service processes when the process
@@ -72,82 +66,34 @@ export function configureChildHeapLimit(processCount: number) {
 }
 
 /**
- * Checks for pending database migrations on startup and runs them, unless the
- * --no-migrate flag was passed in which case the process exits with an error.
+ * Runs the database connection and pending migration checks in a short-lived
+ * child process, resolving once it exits successfully. Running these checks out
+ * of process keeps the long-lived master free of the Sequelize models graph.
+ *
+ * @returns a promise that resolves when the checks have completed successfully.
  */
-export async function checkPendingMigrations() {
-  let lock;
-  try {
-    lock = await MutexLock.acquire("migrations", 10 * Minute.ms, {
-      releaseOnShutdown: true,
+export function checkPendingMigrations(): Promise<void> {
+  return new Promise((resolve) => {
+    const child = fork(
+      path.join(__dirname, "..", "scripts", "checkMigrations.js"),
+      process.argv.slice(2),
+      // Run without the parent's CLI flags so an inherited --inspect port does
+      // not clash; the heap limit is inherited via NODE_OPTIONS in the env.
+      { execArgv: [], stdio: ["inherit", "inherit", "inherit", "ipc"] }
+    );
+
+    child.once("error", (err) => {
+      Logger.fatal("Failed to run database migration checks", err);
     });
 
-    const pending = await migrations.pending();
-    if (!isEmpty(pending)) {
-      if (getArg("no-migrate")) {
-        Logger.fatal(
-          styleText(
-            "red",
-            `Database migrations are pending and were not ran because --no-migrate flag was passed.\nRun the migrations with "yarn db:migrate".`
-          ),
-          new Error("Migrations pending")
-        );
-      } else {
-        Logger.info("database", "Running migrations…");
-        await migrations.up();
+    child.once("exit", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
       }
-    }
-    await checkDataMigrations();
-  } catch (err) {
-    const message = errToString(err);
-    const error = toError(err);
-    if (message.includes("ECONNREFUSED")) {
-      Logger.fatal(
-        styleText(
-          "red",
-          `Could not connect to the database. Please check your connection settings.`
-        ),
-        error
-      );
-    } else {
-      Logger.fatal(styleText("red", message), error);
-    }
-  } finally {
-    if (lock) {
-      await MutexLock.release(lock);
-    }
-  }
-}
-
-/**
- * Checks whether a required data migration has been completed for self-hosted
- * installations, exiting the process with instructions if it has not.
- */
-export async function checkDataMigrations() {
-  if (env.isCloudHosted) {
-    return;
-  }
-
-  const team = await Team.findOne();
-  const provider = await AuthenticationProvider.findOne();
-
-  if (
-    env.isProduction &&
-    team &&
-    team.createdAt < new Date("2024-01-01") &&
-    !provider
-  ) {
-    Logger.fatal(
-      `
-This version of Outline cannot start until a data migration is complete.
-Backup your database, run the database migrations and the following script:
-(Note: script run needed only when upgrading to any version between 0.54.0 and 0.61.1, including both)
-
-$ node ./build/server/scripts/20210226232041-migrate-authentication.js
-`,
-      new Error("Data migration required")
-    );
-  }
+      process.exit(code ?? 1);
+    });
+  });
 }
 
 /**