fix(frontend): locale inliner is not working (#17543)

* feat: support facade module

* refactor: migrate typings to ESTree from rolldown/utils

* fix: name conflict from function parameter are not detected correctly

* refactor: migrate typings to ESTree from rolldown/utils

* fix: name conflict from function parameter are not detected correctly

* fix: template literal in member expression not supported

* fix: improve identifier conflict

* feat: add error when no localization are applied by locale inliner

* lint: fix lints

* fix: let rolldown to not hoist i18n modules with other modules

* chore: make error if there is unexpected specifiers

* fix license header
This commit is contained in:
anatawa12
2026-06-05 12:36:44 +09:00
committed by GitHub
parent 312d7c1866
commit 67a0ae460d
6 changed files with 253 additions and 100 deletions
@@ -9,6 +9,7 @@ import MagicString from 'magic-string';
import { collectModifications } from './locale-inliner/collect-modifications.js';
import { applyWithLocale } from './locale-inliner/apply-with-locale.js';
import { blankLogger } from './logger.js';
import { detectI18nFacadeChunk } from './locale-inliner/facade-chunk-detection.js';
import type { Logger } from './logger.js';
import type { Locale } from 'i18n';
import type { Manifest as ViteManifest } from 'vite';
@@ -18,6 +19,7 @@ export class LocaleInliner {
scriptsDir: string;
i18nFile: string;
i18nFileName: string;
i18nSymbol: string;
logger: Logger;
chunks: ScriptChunk[];
@@ -43,8 +45,10 @@ export class LocaleInliner {
this.i18nFile = options.i18nFile;
this.i18nFileName = this.stripScriptDir(options.manifest[this.i18nFile].file);
this.logger = options.logger;
this.i18nSymbol = 'i18n';
this.chunks = Object.values(options.manifest).filter(chunk => this.isScriptFile(chunk.file)).map(chunk => ({
fileName: this.stripScriptDir(chunk.file),
src: chunk.src,
chunkName: chunk.name,
}));
}
@@ -57,13 +61,41 @@ export class LocaleInliner {
}
collectsModifications() {
this.#detectI18nFacadeChunk();
for (const chunk of this.chunks) {
if (chunk.sourceCode == null) {
throw new Error(`Source code for ${chunk.fileName} is not loaded.`);
}
if (chunk.isFacadeOfI18n) {
chunk.modifications = [];
continue;
}
const fileLogger = this.logger.prefixed(`${chunk.fileName} (${chunk.chunkName}): `);
chunk.modifications = collectModifications(chunk.sourceCode, chunk.fileName, fileLogger, this);
}
if (!this.chunks.flatMap(x => x.modifications ?? []).some(x => x.type === 'localized')) {
throw new Error('No localizations are inlined! this should mean locale inliner is not working well!');
}
}
#detectI18nFacadeChunk() {
// For some reason, even with `preserveEntrySignatures: 'allow-extension'`, rolldown may generate facade chunk
// This method detects facade chunk and replace i18nFile / i18nFileName with correct file name
const chunk = this.chunks.find(x => x.fileName === this.i18nFileName);
if (chunk == null) throw new Error(`i18n script file '${this.i18nFile}' not found`);
if (chunk.sourceCode == null) throw new Error(`Source code for '${this.i18nFile}' not loaded`);
const fileLogger = this.logger.prefixed(`${chunk.fileName} (${chunk.chunkName}): `);
const facadeInfo = detectI18nFacadeChunk(chunk.sourceCode, chunk.fileName, fileLogger);
if (facadeInfo != null) {
const i18nSymbol = facadeInfo.nameMap[this.i18nSymbol];
if (i18nSymbol == null) throw new Error(`Facade module for i18n file does not map ${this.i18nSymbol}. mapping: ${JSON.stringify(facadeInfo.nameMap)}`);
this.logger.info(`We detected ${this.i18nFileName} is facade chunk maps ${facadeInfo.fileName} with ${i18nSymbol} as ${this.i18nSymbol}`);
chunk.isFacadeOfI18n = true;
this.i18nFileName = facadeInfo.fileName;
this.i18nSymbol = i18nSymbol;
}
}
async saveAllLocales(locales: Record<string, Locale>) {
@@ -107,6 +139,7 @@ interface ScriptChunk {
fileName: string;
chunkName?: string;
sourceCode?: string;
isFacadeOfI18n?: true;
modifications?: TextModification[];
}
@@ -5,10 +5,8 @@
import { parseAst } from 'rolldown/parseAst';
import * as estreeWalker from 'estree-walker';
import { assertNever, assertType } from '../utils.js';
import type { ESTree as RolldownESTree } from 'rolldown/utils';
import type { AstNode } from 'rollup';
import type * as estree from 'estree';
import { assertNever } from '../utils.js';
import type { ESTree } from 'rolldown/utils';
import type { LocaleInliner, TextModification } from '../locale-inliner.js';
import type { Logger } from '../logger.js';
@@ -17,9 +15,15 @@ interface WalkerContext {
skip: () => void;
}
const walk = estreeWalker.walk as {
(node: ESTree.Node, callback: {
enter?: (this: WalkerContext, node: ESTree.Node, parent: ESTree.Node | null, property: string | number | symbol | null | undefined) => void;
}): void;
};
export function collectModifications(sourceCode: string, fileName: string, fileLogger: Logger, inliner: LocaleInliner): TextModification[] {
if (sourceCode === '') return [];
let programNode: RolldownESTree.Program;
let programNode: ESTree.Program;
try {
programNode = parseAst(sourceCode);
} catch (err) {
@@ -37,11 +41,8 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
// 1) replace all `scripts/` path literals with locale code
// 2) replace all `localStorage.getItem("lang")` with `localeName` variable
// 3) replace all `await window.fetch(`/assets/locales/${d}.${x}.json`).then(u=>u.json())` with `localeJson` variable
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(estreeWalker.walk as any)(programNode, {
enter(this: WalkerContext, node: Node) {
assertType<AstNode>(node);
walk(programNode, {
enter(this: WalkerContext, node: ESTree.Node) {
if (node.type === 'Literal' && typeof node.value === 'string' && node.raw) {
if (node.raw.substring(1).startsWith(inliner.scriptsDir)) {
// we find `scripts/\w+\.js` literal and replace 'scripts' part with locale code
@@ -92,7 +93,7 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
},
});
const importSpecifierResult = findImportSpecifier(programNode, inliner.i18nFileName, 'i18n');
const importSpecifierResult = findImportSpecifier(programNode, inliner.i18nFileName, inliner.i18nSymbol);
switch (importSpecifierResult.type) {
case 'no-import':
@@ -108,7 +109,7 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
});
return modifications;
case 'unexpected-specifiers':
fileLogger.info(`Importing ${inliner.i18nFileName} found but with unexpected specifiers. Skipping inlining.`);
fileLogger.error(`Importing ${inliner.i18nFileName} found but with unexpected specifiers. Skipping inlining.`);
return modifications;
case 'specifier':
fileLogger.debug(`Found import i18n as ${importSpecifierResult.localI18nIdentifier}`);
@@ -118,42 +119,18 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
const i18nImport = importSpecifierResult.importNode;
const localI18nIdentifier = importSpecifierResult.localI18nIdentifier;
// Check if the identifier is already declared in the file.
// If it is, we may overwrite it and cause issues so we skip inlining
let isSupported = true;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(estreeWalker.walk as any)(programNode, {
enter(node: Node) {
if (node.type === 'VariableDeclaration') {
assertType<estree.VariableDeclaration>(node);
for (const id of node.declarations.flatMap(x => declsOfPattern(x.id))) {
if (id === localI18nIdentifier) {
isSupported = false;
}
}
}
},
});
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (!isSupported) {
fileLogger.error(`Duplicated identifier "${localI18nIdentifier}" in variable declaration. Skipping inlining.`);
return modifications;
}
fileLogger.debug(`imports i18n as ${localI18nIdentifier}`);
fileLogger.debug(`imports ${inliner.i18nSymbol} /*i18n*/ as ${localI18nIdentifier}`);
// In case of substitution failure, we will preserve the import statement
// otherwise we will remove it.
let preserveI18nImport = false;
const codeModifications: TextModification[] = [];
const toSkip = new Set();
toSkip.add(i18nImport);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(estreeWalker.walk as any)(programNode, {
enter(this: WalkerContext, node: Node, parent: Node | null, property: string | number | symbol | null | undefined) {
assertType<AstNode>(node);
assertType<AstNode>(parent);
walk(programNode, {
enter(this: WalkerContext, node, parent, property) {
if (toSkip.has(node)) {
// This is the import specifier, skip processing it
this.skip();
@@ -164,25 +141,24 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
if (node.type === 'ImportDeclaration') this.skip();
if (node.type === 'Identifier') {
assertType<estree.Identifier>(node);
assertType<estree.Property | estree.MemberExpression | estree.ExportSpecifier>(parent);
if (parent == null) throw new Error();
if (parent.type === 'Property' && !parent.computed && property === 'key') return; // we don't care 'id' part of { id: expr }
if (parent.type === 'MemberExpression' && !parent.computed && property === 'property') return; // we don't care 'id' part of { id: expr }
if (parent.type === 'ExportSpecifier' && property === 'exported') return; // we don't care 'id' part of { id: expr }
if (node.name === localI18nIdentifier) {
// the use of identifier is either direct reference to i18n, or unsupported conflict of the identifier, which should report error.
fileLogger.error(`${lineCol(sourceCode, node)}: Using i18n identifier "${localI18nIdentifier}" directly. Skipping inlining.`);
preserveI18nImport = true;
}
} else if (node.type === 'MemberExpression') {
assertType<estree.MemberExpression>(node);
const i18nPath = parseI18nPropertyAccess(node);
if (i18nPath != null && i18nPath.length >= 2 && i18nPath[0] === 'ts') {
if (parent.type === 'CallExpression' && property === 'callee') return; // we don't want to process `i18n.ts.property.stringBuiltinMethod()`
if (parent != null && parent.type === 'CallExpression' && property === 'callee') return; // we don't want to process `i18n.ts.property.stringBuiltinMethod()`
if (i18nPath.at(-1)?.startsWith('_')) fileLogger.debug(`found i18n grouped property access ${i18nPath.join('.')}`);
else fileLogger.debug(`${lineCol(sourceCode, node)}: found i18n property access ${i18nPath.join('.')}`);
// it's i18n.ts.propertyAccess
// i18n.ts.* will always be resolved to string or object containing strings
modifications.push({
codeModifications.push({
type: 'localized',
begin: node.start,
end: node.end,
@@ -194,7 +170,7 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
// it's parameterized locale substitution (`i18n.tsx.property(parameters)`)
// we expect the parameter to be an object literal
fileLogger.debug(`${lineCol(sourceCode, node)}: found i18n function access (object) ${i18nPath.join('.')}`);
modifications.push({
codeModifications.push({
type: 'parameterized-function',
begin: node.start,
end: node.end,
@@ -203,10 +179,41 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
});
this.skip();
}
} else if (node.type === 'ArrowFunctionExpression') {
assertType<estree.ArrowFunctionExpression>(node);
}
// Scope check
if (node.type === 'FunctionDeclaration'
|| node.type === 'FunctionExpression'
|| node.type === 'ArrowFunctionExpression') {
// if i18n is introduced as the Named Function Expression, interior of the function does not matter
if (node.id?.name === localI18nIdentifier) this.skip();
// If there is 'i18n' in the parameters, we care interior of the function
if (node.params.flatMap(param => declsOfPattern(param)).includes(localI18nIdentifier)) this.skip();
// We find var declation inside the function and if there are
if (findFunctionScopeDecls(node).includes(localI18nIdentifier)) this.skip();
}
if (node.type === 'BlockStatement') {
// We find block-scope declaration inside the block, or from parent node if the block is part of for statement or catch clause
if (findBlockScopeDecls(node).includes(localI18nIdentifier)) this.skip();
}
// statements and clauses introduces new variables in variable scope
if (node.type === 'CatchClause') {
if (node.param != null) {
if (declsOfPattern(node.param).includes(localI18nIdentifier)) this.skip();
}
} else if (node.type === 'ForStatement') {
if (node.init?.type === 'VariableDeclaration') {
if (node.init.declarations.flatMap(x => declsOfPattern(x.id)).includes(localI18nIdentifier)) this.skip();
}
} else if (node.type === 'ForInStatement' || node.type === 'ForOfStatement') {
if (node.left.type === 'VariableDeclaration') {
if (node.left.declarations.flatMap(x => declsOfPattern(x.id)).includes(localI18nIdentifier)) this.skip();
} else {
if (declsOfPattern(node.left).includes(localI18nIdentifier)) this.skip();
}
}
},
});
@@ -214,7 +221,7 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (!preserveI18nImport) {
fileLogger.debug('removing i18n import statement');
modifications.push({
codeModifications.push({
type: 'delete',
begin: i18nImport.start,
end: i18nImport.end,
@@ -222,7 +229,7 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
});
}
function parseI18nPropertyAccess(node: estree.Expression | estree.Super): string[] | null {
function parseI18nPropertyAccess(node: ESTree.Expression | ESTree.Super): string[] | null {
if (node.type === 'Identifier' && node.name === localI18nIdentifier) return []; // i18n itself
if (node.type !== 'MemberExpression') return null;
// super.*
@@ -236,6 +243,9 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
if (node.property.type === 'Literal' && typeof node.property.value === 'string') {
id = node.property.value;
}
if (node.property.type === 'TemplateLiteral' && node.property.quasis.length === 1) {
id = node.property.quasis[0].value.cooked;
}
} else {
if (node.property.type === 'Identifier') {
id = node.property.name;
@@ -249,10 +259,10 @@ export function collectModifications(sourceCode: string, fileName: string, fileL
return [...parentAccess, id];
}
return modifications;
return [...modifications, ...codeModifications];
}
function declsOfPattern(pattern: estree.Pattern | null): string[] {
function declsOfPattern(pattern: ESTree.BindingPattern | ESTree.ParamPattern | ESTree.ArrayAssignmentTarget | ESTree.ObjectAssignmentTarget | ESTree.AssignmentTargetMaybeDefault | ESTree.AssignmentTargetRest | null): string[] {
if (pattern == null) return [];
switch (pattern.type) {
case 'Identifier':
@@ -275,15 +285,19 @@ function declsOfPattern(pattern: estree.Pattern | null): string[] {
case 'AssignmentPattern':
return declsOfPattern(pattern.left);
case 'MemberExpression':
// assignment pattern so no new variable is declared
return [];
case 'TSAsExpression':
case 'TSSatisfiesExpression':
case 'TSTypeAssertion':
case 'TSNonNullExpression':
return []; // not introducing new symbol
case 'TSParameterProperty':
throw new Error();
default:
assertNever(pattern);
}
}
function lineCol(sourceCode: string, node: estree.Node): string {
assertType<AstNode>(node);
function lineCol(sourceCode: string, node: ESTree.Node): string {
const leading = sourceCode.slice(0, node.start);
const lines = leading.split('\n');
const line = lines.length;
@@ -291,35 +305,65 @@ function lineCol(sourceCode: string, node: estree.Node): string {
return `(${line}:${col})`;
}
function findFunctionScopeDecls(fn: ESTree.Function | ESTree.ArrowFunctionExpression): string[] {
if (fn.body == null) return [];
const decls: string[] = [];
walk(fn.body, {
enter(node) {
// The only function-scoped symbol declaration in strict mode is 'var'
// If it's non-strict mode, function declaration will also in function scope.
if (node.type === 'VariableDeclaration' && node.kind === 'var') {
decls.push(...node.declarations.flatMap(x => declsOfPattern(x.id)));
}
if (node.type === 'FunctionDeclaration'
|| node.type === 'FunctionExpression'
|| node.type === 'ArrowFunctionExpression') {
// The function makes new inner scope
this.skip();
}
},
});
return decls;
}
function findBlockScopeDecls(block: ESTree.BlockStatement): string[] {
const decls: string[] = [];
for (const body of block.body) {
walk(body, {
enter(node) {
if (node.type === 'VariableDeclaration' && node.kind !== 'var') {
decls.push(...node.declarations.flatMap(x => declsOfPattern(x.id)));
} else if (node.type === 'FunctionDeclaration') {
if (node.id != null) decls.push(node.id.name);
} else if (node.type === 'ClassDeclaration') {
if (node.id != null) decls.push(node.id.name);
}
if (
node.type === 'FunctionDeclaration'
|| node.type === 'FunctionExpression'
|| node.type === 'ArrowFunctionExpression'
|| node.type === 'BlockStatement'
|| node.type === 'CatchClause'
|| node.type === 'ForStatement'
|| node.type === 'ForInStatement'
|| node.type === 'ForOfStatement'
) {
// The function makes new inner scope
this.skip();
}
},
});
}
return decls;
}
//region checker functions
type Node =
| estree.AssignmentProperty
| estree.CatchClause
| estree.Class
| estree.ClassBody
| estree.Expression
| estree.Function
| estree.Identifier
| estree.Literal
| estree.MethodDefinition
| estree.ModuleDeclaration
| estree.ModuleSpecifier
| estree.Pattern
| estree.PrivateIdentifier
| estree.Program
| estree.Property
| estree.PropertyDefinition
| estree.SpreadElement
| estree.Statement
| estree.Super
| estree.SwitchCase
| estree.TemplateElement
| estree.VariableDeclarator
;
// localStorage.getItem("lang")
function isLocalStorageGetItemLang(getItemCall: Node): boolean {
function isLocalStorageGetItemLang(getItemCall: ESTree.Node): boolean {
if (getItemCall.type !== 'CallExpression') return false;
if (getItemCall.arguments.length !== 1) return false;
@@ -336,7 +380,7 @@ function isLocalStorageGetItemLang(getItemCall: Node): boolean {
}
// await window.fetch(`/assets/locales/${d}.${x}.json`).then(u => u.json(), ....)
function isAwaitFetchLocaleThenJson(awaitNode: Node): boolean {
function isAwaitFetchLocaleThenJson(awaitNode: ESTree.Node): boolean {
if (awaitNode.type !== 'AwaitExpression') return false;
const thenCall = awaitNode.argument;
@@ -379,16 +423,15 @@ function isAwaitFetchLocaleThenJson(awaitNode: Node): boolean {
type SpecifierResult =
| { type: 'no-import' }
| { type: 'no-specifiers', importNode: estree.ImportDeclaration & AstNode }
| { type: 'unexpected-specifiers', importNode: estree.ImportDeclaration & AstNode }
| { type: 'specifier', localI18nIdentifier: string, importNode: estree.ImportDeclaration & AstNode }
| { type: 'no-specifiers', importNode: ESTree.ImportDeclaration }
| { type: 'unexpected-specifiers', importNode: ESTree.ImportDeclaration }
| { type: 'specifier', localI18nIdentifier: string, importNode: ESTree.ImportDeclaration }
;
function findImportSpecifier(programNode: RolldownESTree.Program, i18nFileName: string, i18nSymbol: string): SpecifierResult {
function findImportSpecifier(programNode: ESTree.Program, i18nFileName: string, i18nSymbol: string): SpecifierResult {
const imports = programNode.body.filter(x => x.type === 'ImportDeclaration');
const importNode = imports.find(x => x.source.value === `./${i18nFileName}`) as estree.ImportDeclaration | undefined;
const importNode = imports.find(x => x.source.value === `./${i18nFileName}`);
if (!importNode) return { type: 'no-import' };
assertType<AstNode>(importNode);
if (importNode.specifiers.length === 0) {
return { type: 'no-specifiers', importNode };
@@ -415,15 +458,15 @@ function findImportSpecifier(programNode: RolldownESTree.Program, i18nFileName:
}
// checker helpers
function isMemberExpression(node: Node, property: string): node is estree.MemberExpression {
function isMemberExpression(node: ESTree.Node, property: string): node is ESTree.MemberExpression {
return node.type === 'MemberExpression' && !node.computed && node.property.type === 'Identifier' && node.property.name === property;
}
function isStringLiteral(node: Node, value: string): node is estree.Literal {
function isStringLiteral(node: ESTree.Node, value: string): node is ESTree.StringLiteral {
return node.type === 'Literal' && typeof node.value === 'string' && node.value === value;
}
function isIdentifier(node: Node, name: string): node is estree.Identifier {
function isIdentifier(node: ESTree.Node, name: string): node is ESTree.IdentifierReference {
return node.type === 'Identifier' && node.name === name;
}
@@ -0,0 +1,73 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import path from 'node:path';
import { parseAst } from 'rolldown/parseAst';
import type { Logger } from '../logger.js';
import type { ESTree as RolldownESTree } from 'rolldown/utils';
interface FacadeInfo {
fileName: string,
// facade export name => internal name
nameMap: Partial<Record<string, string>>,
}
export function detectI18nFacadeChunk(
sourceCode: string,
fileName: string,
fileLogger: Logger,
): FacadeInfo | null {
let programNode: RolldownESTree.Program;
try {
programNode = parseAst(sourceCode);
} catch (err) {
fileLogger.error(`Failed to parse source code: ${err}`);
return null;
}
if (programNode.sourceType !== 'module') {
fileLogger.error('Source code is not a module.');
return null;
}
// check if the file is like facade.
// if file is like following we treat them as facade.
// ```
// import { something } from "file";
// export { something };
// ```
if (programNode.body.length !== 2) return null; // not a facade
if (programNode.body[0].type !== 'ImportDeclaration') return null; // not a facade
if (programNode.body[1].type !== 'ExportNamedDeclaration') return null; // not a facade
const importDecl = programNode.body[0];
const exportDecl = programNode.body[1];
// the file is a facade file.
const sourcePath = importDecl.source.value;
const sourceName = path.posix.basename(sourcePath);
const importNameMap = Object.fromEntries(importDecl.specifiers
.map(specifier => {
if (specifier.type !== 'ImportSpecifier') throw new Error(`${fileName}: Unexpected import specifier in facade module: ${specifier.type}`);
const exportName = getExportName(specifier.imported);
const localName = specifier.local.name;
return [localName, exportName];
}));
const nameMap = Object.fromEntries(exportDecl.specifiers.map(spec => {
const localName = getExportName(spec.local);
const facadeExportName = getExportName(spec.exported);
const moduleExportName = importNameMap[localName];
return [facadeExportName, moduleExportName];
}));
return {
fileName: sourceName,
nameMap,
};
}
function getExportName(node: RolldownESTree.ModuleExportName): string {
return node.type === 'Literal' ? node.value : node.name;
}
+1 -1
View File
@@ -8,5 +8,5 @@ export function assertNever(x: never): never {
throw new Error(`Unexpected type: ${(x as any)?.type ?? x}`);
}
export function assertType<T>(node: unknown): asserts node is T {
export function assertType<T>(_node: unknown): asserts node is T {
}
+5 -3
View File
@@ -153,9 +153,11 @@ export function getConfig(): UserConfig {
name: 'vue',
test: /node_modules[\\/]vue/,
}, {
// dependencies of i18n.ts
name: 'config',
test: /@@[\\/]js[\\/]config\.js/,
// split each i18n related module to each distinct module, deny hoisting
name: 'i18n',
test: /i18n\.ts/,
minSize: 0,
maxSize: 1,
}],
},
entryFileNames: `scripts/${localesHash}-[hash:8].js`,
+5 -3
View File
@@ -194,9 +194,11 @@ export function getConfig(): UserConfig {
name: 'photoswipe',
test: /node_modules[\\/]photoswipe/,
}, {
// dependencies of i18n.ts
name: 'config',
test: /@@[\\/]js[\\/]config\.js/,
// split each i18n related module to each distinct module, deny hoisting
name: 'i18n',
test: /i18n\.ts/,
minSize: 0,
maxSize: 1,
}],
},
entryFileNames: `scripts/${localesHash}-[hash:8].js`,