Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@
"dist/**/*.js",
"dist/**/*.lua",
"dist/**/*.ts",
"dist/lualib/**/*.json"
"dist/lualib/**/*.json",
"src/lualib/**/*.ts",
"src/lualib/**/*.json",
"src/lualib-build/**/*.ts",
"language-extensions/index.d.ts"
],
"main": "dist/index.js",
"types": "dist/index.d.ts",
Expand Down
4 changes: 4 additions & 0 deletions src/CompilerOptions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,15 @@ export interface TransformerImport {
export interface LuaPluginImport {
name: string;
import?: string;
skipRecompileLuaLib?: boolean;

[option: string]: any;
}

export interface InMemoryLuaPlugin {
plugin: Plugin | ((options: Record<string, any>) => Plugin);
skipRecompileLuaLib?: boolean;

[option: string]: any;
}

Expand All @@ -46,6 +49,7 @@ export interface TypeScriptToLuaOptions {
tstlVerbose?: boolean;
lua51AllowTryCatchInAsyncAwait?: boolean;
measurePerformance?: boolean;
recompileLuaLib?: boolean;
}

export type CompilerOptions = OmitIndexSignature<ts.CompilerOptions> &
Expand Down
116 changes: 105 additions & 11 deletions src/LuaLib.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import * as path from "path";
import { EmitHost } from "./transpilation";
import { EmitHost, transpileProject } from "./transpilation";
import * as lua from "./LuaAST";
import { LuaTarget } from "./CompilerOptions";
import { LuaTarget, type CompilerOptions } from "./CompilerOptions";
import { getOrUpdate } from "./utils";
import { createEmitOutputCollector, type TranspiledFile } from "./transpilation/output-collector";
import { parseConfigFileWithSystem } from "./cli/tsconfig";
import { createDiagnosticReporter } from "./cli/report";

export enum LuaLibFeature {
ArrayAt = "ArrayAt",
Expand Down Expand Up @@ -139,12 +142,16 @@ export function resolveLuaLibDir(luaTarget: LuaTarget) {
export const luaLibModulesInfoFileName = "lualib_module_info.json";
const luaLibModulesInfo = new Map<LuaTarget, LuaLibModulesInfo>();

export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost): LuaLibModulesInfo {
if (!luaLibModulesInfo.has(luaTarget)) {
export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost, useCache = true): LuaLibModulesInfo {
if (!useCache || !luaLibModulesInfo.has(luaTarget)) {
const lualibPath = path.join(resolveLuaLibDir(luaTarget), luaLibModulesInfoFileName);
const result = emitHost.readFile(lualibPath);
if (result !== undefined) {
luaLibModulesInfo.set(luaTarget, JSON.parse(result) as LuaLibModulesInfo);
const info = JSON.parse(result) as LuaLibModulesInfo;
if (!useCache) {
return info;
}
luaLibModulesInfo.set(luaTarget, info);
} else {
throw new Error(`Could not load lualib dependencies from '${lualibPath}'`);
}
Expand Down Expand Up @@ -175,7 +182,21 @@ export function getLuaLibExportToFeatureMap(

const lualibFeatureCache = new Map<LuaTarget, Map<LuaLibFeature, string>>();

export function readLuaLibFeature(feature: LuaLibFeature, luaTarget: LuaTarget, emitHost: EmitHost): string {
export function readLuaLibFeature(
feature: LuaLibFeature,
luaTarget: LuaTarget,
emitHost: EmitHost,
useCache = true
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't get what the point of this getCache in these functions is, why would we no longer want the cache? In my mind either we read from disk or we recompile, but either way we want to cache the result there.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because every time we recompile, we need new fresh LuaLib code.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I still don't understand this: Either we read the file from disk and we want to cache the data we read, or we recompile and we want to cache the compilation result. Why would we ever want to skip the cache in either case? I don't see why the lualib would change after first compilation within a single tstl run

): string {
if (!useCache) {
const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`);
const luaLibFeature = emitHost.readFile(featurePath);
if (luaLibFeature === undefined) {
throw new Error(`Could not load lualib feature from '${featurePath}'`);
}
return luaLibFeature;
}

const featureMap = getOrUpdate(lualibFeatureCache, luaTarget, () => new Map());
if (!featureMap.has(feature)) {
const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`);
Expand Down Expand Up @@ -257,9 +278,69 @@ export function loadImportedLualibFeatures(
return statements;
}

const recompileLualibCache = new WeakMap<EmitHost, TranspiledFile[]>();

function recompileLuaLibFiles(sourceOptions: CompilerOptions, emitHost: EmitHost): TranspiledFile[] {
let transpiledFiles = recompileLualibCache.get(emitHost);
if (!transpiledFiles) {
const tsconfigPath =
sourceOptions.luaTarget === LuaTarget.Lua50
? path.join(__dirname, "../src/lualib/tsconfig.lua50.json")
: path.join(__dirname, "../src/lualib/tsconfig.json");
const config = parseConfigFileWithSystem(tsconfigPath);
const options = config.options;
const sourcePlugins = (sourceOptions.luaPlugins ?? []).filter(p => !p.skipRecompileLuaLib);
options.luaPlugins = [...(options.luaPlugins ?? []), ...sourcePlugins];

const collector = createEmitOutputCollector(options.extension);
const reportDiagnostic = createDiagnosticReporter(false);

const { diagnostics } = transpileProject(tsconfigPath, options, collector.writeFile);
diagnostics.forEach(reportDiagnostic);

transpiledFiles = collector.files;
recompileLualibCache.set(emitHost, transpiledFiles);
}

return transpiledFiles;
}

function recompileLuaLibBundle(sourceOptions: CompilerOptions, emitHost: EmitHost): string | undefined {
const transpiledFiles = recompileLuaLibFiles(sourceOptions, emitHost);
const lualibBundle = transpiledFiles.find(f => f.outPath.endsWith("lualib_bundle.lua"));
return lualibBundle?.lua;
}

export function recompileInlineLualibFeatures(
features: Iterable<LuaLibFeature>,
options: CompilerOptions,
emitHost: EmitHost
): string {
const luaTarget = options.luaTarget ?? LuaTarget.Universal;
const transpiledFiles = recompileLuaLibFiles(options, emitHost);
emitHost = {
readFile(filePath: string) {
const file = transpiledFiles.find(f => f.outPath === filePath);
return file ? file.text : undefined;
},
} as any as EmitHost;
const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, false);
return resolveRecursiveLualibFeatures(features, luaTarget, emitHost, moduleInfo)
.map(feature => readLuaLibFeature(feature, luaTarget, emitHost, false))
.join("\n");
}

const luaLibBundleContent = new Map<string, string>();

export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost): string {
export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost, options: CompilerOptions): string {
if (options.recompileLuaLib) {
const result = recompileLuaLibBundle(options, emitHost);
if (!result) {
throw new Error(`Failed to recompile lualib bundle`);
}
return result;
}

const lualibPath = path.join(resolveLuaLibDir(luaTarget), "lualib_bundle.lua");
if (!luaLibBundleContent.has(lualibPath)) {
const result = emitHost.readFile(lualibPath);
Expand All @@ -279,13 +360,26 @@ export function getLualibBundleReturn(exportedValues: string[]): string {

export function buildMinimalLualibBundle(
features: Iterable<LuaLibFeature>,
luaTarget: LuaTarget,
options: CompilerOptions,
emitHost: EmitHost
): string {
const code = loadInlineLualibFeatures(features, luaTarget, emitHost);
const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost);
const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports);
const luaTarget = options.luaTarget ?? LuaTarget.Universal;
let code;
if (options.recompileLuaLib) {
code = recompileInlineLualibFeatures(features, options, emitHost);
const transpiledFiles = recompileLuaLibFiles(options, emitHost);
emitHost = {
readFile(filePath: string) {
const file = transpiledFiles.find(f => f.outPath === filePath);
return file ? file.text : undefined;
},
} as any as EmitHost;
} else {
code = loadInlineLualibFeatures(features, luaTarget, emitHost);
}

const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, !options.recompileLuaLib);
const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports);
return code + getLualibBundleReturn(exports);
}

Expand Down
13 changes: 11 additions & 2 deletions src/LuaPrinter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@ import { Mapping, SourceMapGenerator, SourceNode } from "source-map";
import * as ts from "typescript";
import { CompilerOptions, isBundleEnabled, LuaLibImportKind, LuaTarget } from "./CompilerOptions";
import * as lua from "./LuaAST";
import { loadImportedLualibFeatures, loadInlineLualibFeatures, LuaLibFeature } from "./LuaLib";
import {
loadImportedLualibFeatures,
loadInlineLualibFeatures,
LuaLibFeature,
recompileInlineLualibFeatures,
} from "./LuaLib";
import { isValidLuaIdentifier, shouldAllowUnicode } from "./transformation/utils/safe-names";
import { EmitHost, getEmitPath } from "./transpilation";
import { intersperse, normalizeSlashes } from "./utils";
Expand Down Expand Up @@ -246,7 +251,11 @@ export class LuaPrinter {
} else if (luaLibImport === LuaLibImportKind.Inline && file.luaLibFeatures.size > 0) {
// Inline lualib features
sourceChunks.push("-- Lua Library inline imports\n");
sourceChunks.push(loadInlineLualibFeatures(file.luaLibFeatures, luaTarget, this.emitHost));
if (this.options.recompileLuaLib) {
sourceChunks.push(recompileInlineLualibFeatures(file.luaLibFeatures, this.options, this.emitHost));
} else {
sourceChunks.push(loadInlineLualibFeatures(file.luaLibFeatures, luaTarget, this.emitHost));
}
sourceChunks.push("-- End of Lua Library inline imports\n");
}

Expand Down
5 changes: 5 additions & 0 deletions src/cli/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ export const optionDeclarations: CommandLineOption[] = [
description: "Measure performance of the tstl compiler.",
type: "boolean",
},
{
name: "recompileLuaLib",
description: "Recompile the Lua standard library with custom plugins.",
type: "boolean",
},
];

export function updateParsedConfigFile(parsedConfigFile: ts.ParsedCommandLine): ParsedCommandLine {
Expand Down
24 changes: 23 additions & 1 deletion src/lualib-build/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ import {
isImport,
isRequire,
} from "./util";
import { createDiagnosticFactoryWithCode } from "../utils";
import { createDiagnosticFactoryWithCode, normalizeSlashes } from "../utils";
import type { CompilerOptions } from "..";

export const lualibDiagnostic = createDiagnosticFactoryWithCode(200000, (message: string, file?: ts.SourceFile) => ({
messageText: message,
Expand Down Expand Up @@ -168,6 +169,27 @@ class LuaLibPlugin implements tstl.Plugin {
}
return { result: result as LuaLibModulesInfo, diagnostics };
}

public moduleResolution(
moduleIdentifier: string,
requiringFile: string,
_options: CompilerOptions,
emitHost: EmitHost
): string | undefined {
const tstlRoot = path.resolve(__dirname, "../..");
const relativeRequiringFile = normalizeSlashes(path.relative(tstlRoot, requiringFile));
if (!relativeRequiringFile.startsWith("src/lualib/")) {
return;
}

const tryingPaths = ["./", "./universal/"];
for (const tryingPath of tryingPaths) {
const possiblePath = path.join(tstlRoot, "src", "lualib", tryingPath, `${moduleIdentifier}.ts`);
if (emitHost.fileExists(possiblePath)) {
return possiblePath;
}
}
}
}

class LuaLibPrinter extends tstl.LuaPrinter {
Expand Down
3 changes: 3 additions & 0 deletions src/transpilation/output-collector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ export interface TranspiledFile {
js?: string;
/** @internal */
jsSourceMap?: string;
/** @internal */
text?: string;
}

export function createEmitOutputCollector(luaExtension = ".lua") {
Expand All @@ -38,6 +40,7 @@ export function createEmitOutputCollector(luaExtension = ".lua") {
} else if (fileName.endsWith(".d.ts.map")) {
file.declarationMap = data;
}
file.text = data;
};

return { writeFile, files };
Expand Down
9 changes: 8 additions & 1 deletion src/transpilation/transpile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,16 @@ export function getProgramTranspileResult(
transpiledFiles = [];
}

const proxyEmitHost =
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand what this does or why it's needed

Copy link
Contributor Author

@pilaoda pilaoda Aug 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because we write lualib_module_info.json when we recompile lualib_bundle.lua, this proxy make sure we don't overwrite the exists one, but collect by our recompile instance.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Couldn't you solve this in lualib.ts where you start the new transpile (with this explanation as comment because code is very hard to understand), instead of here in the global transpile process?

writeFileResult !== emitHost.writeFile
? new Proxy(emitHost, {
get: (target, prop) => (prop === "writeFile" ? writeFileResult : target[prop as keyof EmitHost]),
})
: emitHost;

for (const plugin of plugins) {
if (plugin.afterPrint) {
const pluginDiagnostics = plugin.afterPrint(program, options, emitHost, transpiledFiles) ?? [];
const pluginDiagnostics = plugin.afterPrint(program, options, proxyEmitHost, transpiledFiles) ?? [];
diagnostics.push(...pluginDiagnostics);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/transpilation/transpiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,9 @@ export class Transpiler {
this.emitHost,
resolvedFiles.map(f => f.code)
);
return buildMinimalLualibBundle(usedFeatures, luaTarget, this.emitHost);
return buildMinimalLualibBundle(usedFeatures, options, this.emitHost);
} else {
return getLuaLibBundle(luaTarget, this.emitHost);
return getLuaLibBundle(luaTarget, this.emitHost, options);
}
}
}
Expand Down
58 changes: 57 additions & 1 deletion test/transpile/lualib.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import * as ts from "typescript";
import { LuaLibFeature, LuaTarget } from "../../src";
import { LuaLibFeature, LuaLibImportKind, LuaTarget } from "../../src";
import { readLuaLibFeature } from "../../src/LuaLib";
import * as util from "../util";
import path = require("path");

test.each(Object.entries(LuaLibFeature))("Lualib does not use ____exports (%p)", (_, feature) => {
const lualibCode = readLuaLibFeature(feature, LuaTarget.Lua54, ts.sys);
Expand Down Expand Up @@ -29,3 +30,58 @@ test("Lualib bundle does not assign globals", () => {
.withLanguageExtensions()
.expectNoExecutionError();
});

test("Lualib recompile with import kind require", () => {
const { transpiledFiles } = util.testExpression`Array.isArray({})`
.setOptions({
recompileLuaLib: true,
luaPlugins: [{ name: path.join(__dirname, "./plugins/beforeEmit.ts") }],
})
.expectToHaveNoDiagnostics()
.getLuaResult();
const lualibBundle = transpiledFiles.find(f => f.outPath === "lualib_bundle.lua");
expect(lualibBundle).toBeDefined();

// plugin apply twice: main compilation and lualib recompile
expect(lualibBundle?.lua).toContain(
"-- Comment added by beforeEmit plugin\n-- Comment added by beforeEmit plugin\n"
);
});

test("Lualib recompile with import kind require minimal", () => {
const builder = util.testExpression`Array.isArray(123)`
.setOptions({
luaLibImport: LuaLibImportKind.RequireMinimal,
recompileLuaLib: true,
luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }],
})
.expectToHaveNoDiagnostics()
.expectToEqual(true);
const transpiledFiles = builder.getLuaResult().transpiledFiles;
const lualibBundle = transpiledFiles.find(f => f.outPath === "lualib_bundle.lua");
expect(lualibBundle).toBeDefined();
});

test("Lualib recompile with import kind inline", () => {
util.testExpression`Array.isArray(123)`
.setOptions({
luaLibImport: LuaLibImportKind.Inline,
recompileLuaLib: true,
luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }],
})
.expectToHaveNoDiagnostics()
.expectToEqual(true);
});

test("Lualib recompile with bundling", () => {
util.testExpression`Array.isArray(123)`
.setOptions({
luaBundle: "bundle.lua",
luaBundleEntry: "main.ts",
luaLibImport: LuaLibImportKind.RequireMinimal,
recompileLuaLib: true,
luaPlugins: [{ name: path.join(__dirname, "./plugins/visitor.ts") }],
})
.expectToHaveNoDiagnostics()
.expectToEqual(true);
});
4 changes: 4 additions & 0 deletions tsconfig-schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@
"measurePerformance": {
"description": "Measure and report performance of the tstl compiler.",
"type": "boolean"
},
"recompileLuaLib": {
"description": "Recompile the Lua standard library with custom plugins.",
"type": "boolean"
}
},
"dependencies": {
Expand Down
Loading