-
-
Notifications
You must be signed in to change notification settings - Fork 181
New option recompileLuaLib: Recompile the Lua standard library with custom plugins. #1656
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
ae668e1
dd428ba
4424e02
d0ac73b
2efb99c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,11 @@ | ||
| import * as path from "path"; | ||
| import { EmitHost } from "./transpilation"; | ||
| import { EmitHost, transpileProject } from "./transpilation"; | ||
| import * as lua from "./LuaAST"; | ||
| import { LuaTarget } from "./CompilerOptions"; | ||
| import { LuaTarget, type CompilerOptions } from "./CompilerOptions"; | ||
| import { getOrUpdate } from "./utils"; | ||
| import { createEmitOutputCollector, type TranspiledFile } from "./transpilation/output-collector"; | ||
| import { parseConfigFileWithSystem } from "./cli/tsconfig"; | ||
| import { createDiagnosticReporter } from "./cli/report"; | ||
|
|
||
| export enum LuaLibFeature { | ||
| ArrayAt = "ArrayAt", | ||
|
|
@@ -139,12 +142,16 @@ export function resolveLuaLibDir(luaTarget: LuaTarget) { | |
| export const luaLibModulesInfoFileName = "lualib_module_info.json"; | ||
| const luaLibModulesInfo = new Map<LuaTarget, LuaLibModulesInfo>(); | ||
|
|
||
| export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost): LuaLibModulesInfo { | ||
| if (!luaLibModulesInfo.has(luaTarget)) { | ||
| export function getLuaLibModulesInfo(luaTarget: LuaTarget, emitHost: EmitHost, useCache = true): LuaLibModulesInfo { | ||
| if (!useCache || !luaLibModulesInfo.has(luaTarget)) { | ||
| const lualibPath = path.join(resolveLuaLibDir(luaTarget), luaLibModulesInfoFileName); | ||
| const result = emitHost.readFile(lualibPath); | ||
| if (result !== undefined) { | ||
| luaLibModulesInfo.set(luaTarget, JSON.parse(result) as LuaLibModulesInfo); | ||
| const info = JSON.parse(result) as LuaLibModulesInfo; | ||
| if (!useCache) { | ||
| return info; | ||
| } | ||
| luaLibModulesInfo.set(luaTarget, info); | ||
| } else { | ||
| throw new Error(`Could not load lualib dependencies from '${lualibPath}'`); | ||
| } | ||
|
|
@@ -175,7 +182,21 @@ export function getLuaLibExportToFeatureMap( | |
|
|
||
| const lualibFeatureCache = new Map<LuaTarget, Map<LuaLibFeature, string>>(); | ||
|
|
||
| export function readLuaLibFeature(feature: LuaLibFeature, luaTarget: LuaTarget, emitHost: EmitHost): string { | ||
| export function readLuaLibFeature( | ||
| feature: LuaLibFeature, | ||
| luaTarget: LuaTarget, | ||
| emitHost: EmitHost, | ||
| useCache = true | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't get what the point of this getCache in these functions is, why would we no longer want the cache? In my mind either we read from disk or we recompile, but either way we want to cache the result there.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Because every time we recompile, we need new fresh LuaLib code.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I still don't understand this: Either we read the file from disk and we want to cache the data we read, or we recompile and we want to cache the compilation result. Why would we ever want to skip the cache in either case? I don't see why the lualib would change after first compilation within a single tstl run |
||
| ): string { | ||
| if (!useCache) { | ||
| const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`); | ||
| const luaLibFeature = emitHost.readFile(featurePath); | ||
| if (luaLibFeature === undefined) { | ||
| throw new Error(`Could not load lualib feature from '${featurePath}'`); | ||
| } | ||
| return luaLibFeature; | ||
| } | ||
|
|
||
| const featureMap = getOrUpdate(lualibFeatureCache, luaTarget, () => new Map()); | ||
| if (!featureMap.has(feature)) { | ||
| const featurePath = path.join(resolveLuaLibDir(luaTarget), `${feature}.lua`); | ||
|
|
@@ -257,9 +278,69 @@ export function loadImportedLualibFeatures( | |
| return statements; | ||
| } | ||
|
|
||
| const recompileLualibCache = new WeakMap<EmitHost, TranspiledFile[]>(); | ||
|
|
||
| function recompileLuaLibFiles(sourceOptions: CompilerOptions, emitHost: EmitHost): TranspiledFile[] { | ||
| let transpiledFiles = recompileLualibCache.get(emitHost); | ||
| if (!transpiledFiles) { | ||
| const tsconfigPath = | ||
| sourceOptions.luaTarget === LuaTarget.Lua50 | ||
| ? path.join(__dirname, "../src/lualib/tsconfig.lua50.json") | ||
| : path.join(__dirname, "../src/lualib/tsconfig.json"); | ||
| const config = parseConfigFileWithSystem(tsconfigPath); | ||
| const options = config.options; | ||
| const sourcePlugins = (sourceOptions.luaPlugins ?? []).filter(p => !p.skipRecompileLuaLib); | ||
| options.luaPlugins = [...(options.luaPlugins ?? []), ...sourcePlugins]; | ||
|
|
||
| const collector = createEmitOutputCollector(options.extension); | ||
| const reportDiagnostic = createDiagnosticReporter(false); | ||
|
|
||
| const { diagnostics } = transpileProject(tsconfigPath, options, collector.writeFile); | ||
| diagnostics.forEach(reportDiagnostic); | ||
|
|
||
| transpiledFiles = collector.files; | ||
| recompileLualibCache.set(emitHost, transpiledFiles); | ||
| } | ||
|
|
||
| return transpiledFiles; | ||
| } | ||
|
|
||
| function recompileLuaLibBundle(sourceOptions: CompilerOptions, emitHost: EmitHost): string | undefined { | ||
| const transpiledFiles = recompileLuaLibFiles(sourceOptions, emitHost); | ||
| const lualibBundle = transpiledFiles.find(f => f.outPath.endsWith("lualib_bundle.lua")); | ||
| return lualibBundle?.lua; | ||
| } | ||
|
|
||
| export function recompileInlineLualibFeatures( | ||
| features: Iterable<LuaLibFeature>, | ||
| options: CompilerOptions, | ||
| emitHost: EmitHost | ||
| ): string { | ||
| const luaTarget = options.luaTarget ?? LuaTarget.Universal; | ||
| const transpiledFiles = recompileLuaLibFiles(options, emitHost); | ||
| emitHost = { | ||
| readFile(filePath: string) { | ||
| const file = transpiledFiles.find(f => f.outPath === filePath); | ||
| return file ? file.text : undefined; | ||
| }, | ||
| } as any as EmitHost; | ||
| const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, false); | ||
| return resolveRecursiveLualibFeatures(features, luaTarget, emitHost, moduleInfo) | ||
| .map(feature => readLuaLibFeature(feature, luaTarget, emitHost, false)) | ||
| .join("\n"); | ||
| } | ||
|
|
||
| const luaLibBundleContent = new Map<string, string>(); | ||
|
|
||
| export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost): string { | ||
| export function getLuaLibBundle(luaTarget: LuaTarget, emitHost: EmitHost, options: CompilerOptions): string { | ||
| if (options.recompileLuaLib) { | ||
| const result = recompileLuaLibBundle(options, emitHost); | ||
| if (!result) { | ||
| throw new Error(`Failed to recompile lualib bundle`); | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| const lualibPath = path.join(resolveLuaLibDir(luaTarget), "lualib_bundle.lua"); | ||
| if (!luaLibBundleContent.has(lualibPath)) { | ||
| const result = emitHost.readFile(lualibPath); | ||
|
|
@@ -279,13 +360,26 @@ export function getLualibBundleReturn(exportedValues: string[]): string { | |
|
|
||
| export function buildMinimalLualibBundle( | ||
| features: Iterable<LuaLibFeature>, | ||
| luaTarget: LuaTarget, | ||
| options: CompilerOptions, | ||
| emitHost: EmitHost | ||
| ): string { | ||
| const code = loadInlineLualibFeatures(features, luaTarget, emitHost); | ||
| const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost); | ||
| const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports); | ||
| const luaTarget = options.luaTarget ?? LuaTarget.Universal; | ||
| let code; | ||
| if (options.recompileLuaLib) { | ||
| code = recompileInlineLualibFeatures(features, options, emitHost); | ||
| const transpiledFiles = recompileLuaLibFiles(options, emitHost); | ||
| emitHost = { | ||
| readFile(filePath: string) { | ||
| const file = transpiledFiles.find(f => f.outPath === filePath); | ||
| return file ? file.text : undefined; | ||
| }, | ||
| } as any as EmitHost; | ||
| } else { | ||
| code = loadInlineLualibFeatures(features, luaTarget, emitHost); | ||
| } | ||
|
|
||
| const moduleInfo = getLuaLibModulesInfo(luaTarget, emitHost, !options.recompileLuaLib); | ||
| const exports = Array.from(features).flatMap(feature => moduleInfo[feature].exports); | ||
| return code + getLualibBundleReturn(exports); | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -145,9 +145,16 @@ export function getProgramTranspileResult( | |
| transpiledFiles = []; | ||
| } | ||
|
|
||
| const proxyEmitHost = | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't understand what this does or why it's needed
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Because we write
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Couldn't you solve this in lualib.ts where you start the new transpile (with this explanation as comment because code is very hard to understand), instead of here in the global transpile process? |
||
| writeFileResult !== emitHost.writeFile | ||
| ? new Proxy(emitHost, { | ||
| get: (target, prop) => (prop === "writeFile" ? writeFileResult : target[prop as keyof EmitHost]), | ||
| }) | ||
| : emitHost; | ||
|
|
||
| for (const plugin of plugins) { | ||
| if (plugin.afterPrint) { | ||
| const pluginDiagnostics = plugin.afterPrint(program, options, emitHost, transpiledFiles) ?? []; | ||
| const pluginDiagnostics = plugin.afterPrint(program, options, proxyEmitHost, transpiledFiles) ?? []; | ||
| diagnostics.push(...pluginDiagnostics); | ||
| } | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.