Reimplementing Next.js's Output File Tracing with a custom server

copy-traced-files.ts

import { nodeFileTrace } from "@vercel/nft";
import { Sema } from "async-sema";
import { glob } from "glob";
import { NextConfigComplete } from "next/dist/server/config-shared";
import fs from "node:fs/promises";
import path from "node:path";

interface RequiredServerFilesManifest {
  version: number;
  config: NextConfigComplete;
  appDir: string;
  relativeAppDir: string;
  files: string[];
  ignore: string[];
}

const cwd = process.cwd();
const requiredServerFilesPath = path.join(
  cwd,
  ".next/required-server-files.json"
);
const requiredServerFiles = JSON.parse(
  await fs.readFile(requiredServerFilesPath, "utf-8")
) as RequiredServerFilesManifest;
const tracingRoot =
  requiredServerFiles.config.experimental.outputFileTracingRoot!;
const projectDirectory = requiredServerFiles.appDir;
const distDir = path.join(projectDirectory, requiredServerFiles.config.distDir);
const standaloneDirectory = path.join(cwd, "custom-standalone");
const hasInstrumentationHook = false;

const middlewareManifest = JSON.parse(
  await fs.readFile(
    path.join(distDir, "server/middleware-manifest.json"),
    "utf-8"
  )
);

const copiedFiles = new Set<string>();

interface TraceFile {
  files: string[];
}

await fs.rm(standaloneDirectory, { recursive: true, force: true });

async function handleTraceFiles(
  traceFileDir: string,
  traceDataFiles: string[]
) {
  const copySema = new Sema(10, { capacity: traceDataFiles.length });

  await Promise.all(
    traceDataFiles.map(async (relativeFile) => {
      await copySema.acquire();

      try {
        const tracedFilePath = path.join(traceFileDir, relativeFile);

        const fileOutputPath = path.join(
          standaloneDirectory,
          path.relative(tracingRoot, tracedFilePath)
        );

        if (!copiedFiles.has(fileOutputPath)) {
          copiedFiles.add(fileOutputPath);

          await fs.mkdir(path.dirname(fileOutputPath), { recursive: true });

          const symlink = await fs.readlink(tracedFilePath).catch(() => null);

          if (symlink) {
            try {
              await fs.symlink(symlink, fileOutputPath);
            } catch (e: any) {
              if (e.code !== "EEXIST") {
                throw e;
              }
            }
          } else {
            await fs.copyFile(tracedFilePath, fileOutputPath);
          }
        }
      } finally {
        copySema.release();
      }
    })
  );
}

await Promise.all([
  (async () => {
    const pageTraceFiles = await glob(["**/*.js.nft.json"], {
      cwd: path.join(distDir, "server/app"),
      dotRelative: false,
      nodir: true,
    });

    for (const pageTraceFile of pageTraceFiles) {
      const page = pageTraceFile
        .replace(/\\/, "/")
        .replace(/\.js\.nft\.json$/, "");
      if (middlewareManifest.functions.hasOwnProperty(page)) {
        continue;
      }
      const traceFilePath = path.join(distDir, "server/app", pageTraceFile);
      const traceFileDir = path.dirname(traceFilePath);
      const traceData = JSON.parse(
        await fs.readFile(traceFilePath, "utf8")
      ) as TraceFile;
      await handleTraceFiles(traceFileDir, traceData.files).catch((err) => {
        console.warn(`Failed to copy traced files from ${pageTraceFile}`, err);
      });
    }
  })(),
  (async () => {
    if (hasInstrumentationHook) {
      const traceFilePath = path.join(
        distDir,
        "server/instrumentation.js.nft.json"
      );
      const traceFileDir = path.dirname(traceFilePath);
      const traceData = JSON.parse(
        await fs.readFile(traceFilePath, "utf8")
      ) as TraceFile;
      await handleTraceFiles(traceFileDir, traceData.files);
    }
  })(),
  (async () => {
    const { fileList: customServerFileList } = await nodeFileTrace(
      ["server.js"],
      {
        base: tracingRoot,
        processCwd: projectDirectory,
        mixedModules: true,
        async readFile(p) {
          try {
            return await fs.readFile(p, "utf8");
          } catch (e) {
            if (
              e instanceof Error &&
              "code" in e &&
              (e.code === "ENOENT" || e.code === "EISDIR")
            ) {
              return "";
            }
            throw e;
          }
        },
        async readlink(p) {
          try {
            return await fs.readlink(p);
          } catch (e) {
            if (
              e instanceof Error &&
              "code" in e &&
              (e.code === "EINVAL" ||
                e.code === "ENOENT" ||
                e.code === "UNKNOWN")
            ) {
              return null;
            }
            throw e;
          }
        },
        async stat(p) {
          try {
            return await fs.stat(p);
          } catch (e) {
            if (
              e instanceof Error &&
              "code" in e &&
              (e.code === "ENOENT" || e.code === "ENOTDIR")
            ) {
              return null;
            }
            throw e;
          }
        },
      }
    );

    await handleTraceFiles(projectDirectory, [...customServerFileList]);

    const serverInput = await fs.readFile(
      path.join(projectDirectory, "server.js"),
      "utf-8"
    );
    const serverOutputPath = path.join(
      standaloneDirectory,
      path.relative(tracingRoot, projectDirectory),
      "server.js"
    );

    await fs.writeFile(
      serverOutputPath,
      serverInput.replace(
        "__NEXT_CONFIG__",
        JSON.stringify({
          ...requiredServerFiles.config,
          output: "standalone",
        })
      )
    );
  })(),
]);

const requiredServerFileList = [
  ...requiredServerFiles.files,
  path.relative(projectDirectory, requiredServerFilesPath),
  // .env files
];

const copySema = new Sema(10, { capacity: requiredServerFileList.length });

await Promise.all(
  requiredServerFileList.map(async (file) => {
    await copySema.acquire();
    try {
      const filePath = path.join(projectDirectory, file);
      const fileOutputPath = path.join(
        standaloneDirectory,
        path.relative(tracingRoot, filePath)
      );
      await fs.mkdir(path.dirname(fileOutputPath), {
        recursive: true,
      });
      await fs.copyFile(filePath, fileOutputPath);
    } finally {
      copySema.release();
    }
  })
);

await fs.cp(
  path.join(distDir, "server/app"),
  path.join(
    standaloneDirectory,
    path.relative(tracingRoot, distDir),
    "server/app"
  ),
  { recursive: true }
);

await fs.cp(
  path.join(projectDirectory, "public"),
  path.join(
    standaloneDirectory,
    path.relative(tracingRoot, projectDirectory),
    "public"
  ),
  { recursive: true }
);

await fs.cp(
  path.join(distDir, "static"),
  path.join(standaloneDirectory, path.relative(tracingRoot, distDir), "static"),
  { recursive: true }
);

server.js

import { fileURLToPath } from "node:url";
import path from "node:path";
import express from "express";
import next from "next";

const __dirname = fileURLToPath(new URL(".", import.meta.url));
const dir = path.join(__dirname);
const dev = process.env.NODE_ENV !== "production";
const hostname = process.env.HOSTNAME || "0.0.0.0";
const port = process.env.PORT || 3000;

let nextConfig = undefined;

if (!dev) {
  nextConfig = __NEXT_CONFIG__;
  process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(nextConfig);
}

const app = next({
  dir,
  dev,
  hostname,
  port,
  preloadedConfig: nextConfig,
});
const handle = app.getRequestHandler();

app.prepare().then(() => {
  const server = express();

  server.get("*", (req, res) => {
    return handle(req, res);
  });

  server.listen(port, hostname, (err) => {
    if (err) throw err;
    console.log(`> Ready on http://${hostname}:${port}`);
  });
});