Monaco with custom language support

editor.ts

import "monaco-editor/esm/vs/editor/editor.all.js";
import "monaco-editor/esm/vs/editor/standalone/browser/accessibilityHelp/accessibilityHelp.js";
import "monaco-editor/esm/vs/editor/standalone/browser/inspectTokens/inspectTokens.js";
import "monaco-editor/esm/vs/editor/standalone/browser/iPadShowKeyboard/iPadShowKeyboard.js";
import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneGotoLineQuickAccess.js";
import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneGotoSymbolQuickAccess.js";
import "monaco-editor/esm/vs/editor/standalone/browser/quickAccess/standaloneCommandsQuickAccess.js";
import "monaco-editor/esm/vs/editor/standalone/browser/quickInput/standaloneQuickInputService.js";
import "monaco-editor/esm/vs/editor/standalone/browser/referenceSearch/standaloneReferenceSearch.js";
import "monaco-editor/esm/vs/editor/standalone/browser/toggleHighContrast/toggleHighContrast.js";
import * as monaco from "monaco-editor/esm/vs/editor/editor.api";
import { generateTokensCSSForColorMap } from "monaco-editor/esm/vs/editor/common/languages/supports/tokenization";
import { TokenizationRegistry } from "monaco-editor/esm/vs/editor/common/languages";
import { Color } from "monaco-editor/esm/vs/base/common/color";
import styles from "monaco-editor/min/vs/editor/editor.main.css";
import EditorWorker from "monaco-editor/esm/vs/editor/editor.worker?worker";
import { INITIAL, Registry, parseRawGrammar, StateStack } from "vscode-textmate";
import { createOnigScanner, createOnigString, loadWASM, WebAssemblyInstantiator } from "vscode-oniguruma";
// This import is Vite specific...
import onigWASM from "vscode-oniguruma/release/onig.wasm?init";
import VsCodeDarkTheme from "./dark-theme";
import stateMLtmLanguage from "./StateML.tmLanguage.json";
import stateMLConfiguration from "./language-configuration.json";
import { MonacoLanguageClient, CloseAction, ErrorAction, MonacoServices } from "monaco-languageclient";
import { BrowserMessageReader, BrowserMessageWriter } from "vscode-languageserver-protocol/browser";
import { StandaloneServices } from "vscode/services";

const LANGUAGE_ID = "stateml";
const SCOPE_NAME = `source.${LANGUAGE_ID}`;

self.MonacoEnvironment = {
  getWorker: () => new EditorWorker(),
};

MonacoServices.install();
StandaloneServices.initialize({});

export function createMonacoEditor(root: HTMLElement, initialValue: string) {
  // Vite only returns the WASM instance, we can mock `module` since `vscode-oniguruma` doesn't use it
  // See https://github.com/microsoft/vscode-oniguruma/blob/main/src/index.ts#L370
  const instantiator: WebAssemblyInstantiator = async (options) => {
    return { instance: await onigWASM(options || {}) } as WebAssembly.WebAssemblyInstantiatedSource;
  };

  const onigLib = loadWASM({ instantiator }).then(() => ({ createOnigScanner, createOnigString }));

  const registry = new Registry({
    onigLib,
    loadGrammar: async (scopeName) => {
      if (scopeName === SCOPE_NAME) {
        return parseRawGrammar(JSON.stringify(stateMLtmLanguage), "stateml.json");
      }
      return null;
    },
    theme: VsCodeDarkTheme,
  });

  monaco.languages.register({ id: LANGUAGE_ID, extensions: [`.${LANGUAGE_ID}`] });

  monaco.languages.onLanguage(LANGUAGE_ID, async () => {
    const encodedLanguageId = monaco.languages.getEncodedLanguageId(LANGUAGE_ID);
    const grammar = await registry.loadGrammarWithConfiguration(SCOPE_NAME, encodedLanguageId, {});

    if (grammar != null) {
      monaco.languages.setTokensProvider(LANGUAGE_ID, {
        getInitialState: () => INITIAL,
        tokenizeEncoded(line, state) {
          const { tokens, ruleStack: endState } = grammar.tokenizeLine2(line, state as StateStack);
          return { tokens, endState };
        },
      });
    }

    monaco.languages.setLanguageConfiguration(LANGUAGE_ID, rehydrateRegexps(stateMLConfiguration));

    createLanguageClient();
  });

  const editor = monaco.editor.create(root, {
    value: initialValue,
    language: LANGUAGE_ID,
    theme: "vs-dark",
    tabSize: 2,
    minimap: {
      enabled: false,
    },
    automaticLayout: true,
  });

  const cssColors = registry.getColorMap();
  const colorMap = cssColors.map(Color.Format.CSS.parseHex);
  TokenizationRegistry.setColorMap(colorMap);
  // We expect that monaco styles are applied before the Textmate theming
  const css =
    styles +
    "\n\n" +
    generateTokensCSSForColorMap(colorMap);
  return { editor, css };
}

function createLanguageClient() {
  const worker = new Worker(new URL("./lsp-worker.ts", import.meta.url), { type: "module" });
  const reader = new BrowserMessageReader(worker);
  const writer = new BrowserMessageWriter(worker);

  const languageClient = new MonacoLanguageClient({
    name: "StateML Language Client",
    clientOptions: {
      documentSelector: [LANGUAGE_ID],
      errorHandler: {
        error: () => ({ action: ErrorAction.Continue }),
        closed: () => ({ action: CloseAction.DoNotRestart }),
      },
    },
    connectionProvider: {
      get: async () => ({ reader, writer }),
    },
  });
  languageClient.start();
  reader.onClose(() => languageClient.stop());
}

/**
 * Hydrate some configuration as Regex since Monaco only accepts regex for those
 */
function rehydrateRegexps(config: Record<string, any>): monaco.languages.LanguageConfiguration {
  const { indentationRules: ir, markers, wordPattern } = config;
  if (wordPattern) {
    config.wordPattern = new RegExp(wordPattern);
  }
  if (markers?.start) {
    markers.state = new RegExp(markers?.start);
  }
  if (markers?.end) {
    markers.end = new RegExp(markers?.end);
  }
  if (ir?.decreaseIndentPattern) {
    ir.decreaseIndentPattern = new RegExp(ir.decreaseIndentPattern);
  }
  if (ir?.increaseIndentPattern) {
    ir.increaseIndentPattern = new RegExp(ir);
  }
  if (ir?.indentNextLinePattern) {
    ir.indentNextLinePattern = new RegExp(ir.indentNextLinePattern);
  }
  if (ir?.unIndentedLinePattern) {
    ir.indentationRules.unIndentedLinePattern = new RegExp(ir.unIndentedLinePattern);
  }

  return config;
}

lsp-worker.ts

import { createConnection, BrowserMessageReader, BrowserMessageWriter } from "vscode-languageserver/browser.js";

const messageReader = new BrowserMessageReader(self);
const messageWriter = new BrowserMessageWriter(self);

// Use this connection to create a LSP server in this web worker
const connection = createConnection(messageReader, messageWriter);

pacakge.json

{
  "peerDependencies": {
    "monaco-editor": "^0.34.1",
    "monaco-languageclient": "^4.0.1",
    "vscode": "npm:@codingame/monaco-vscode-api@^1.69.7",
    "vscode-languageserver": "^7.0.0",
    "vscode-languageserver-protocol": "^3.17.2",
    "vscode-oniguruma": "^1.6.2",
    "vscode-textmate": "^7.0.3"
  }, 
}

Adding this comment for any who, like me, found this through google and were attempting to do something similar in monaco 0.52.0.
CloseAction and ErrorAction have been merged with the matching vscode enums, and thus are no longer exported from monaco, you get them from vscode-languageclient
import { CloseAction, ErrorAction } from "vscode-languageclient";
Also:

        connectionProvider: {
            get: async () => ({ reader, writer }),
        },

is now:

        messageTransports: {
            reader,
            writer
        }