chore: switch formatter to format-message (#4088)

Co-authored-by: Robert Korulczyk <robert@korulczyk.pl>
This commit is contained in:
Sami Mazouz 2024-10-24 16:48:33 +01:00 committed by GitHub
parent 0464324485
commit 73a029641a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 297 additions and 106 deletions

View File

@ -5,14 +5,13 @@
"type": "module",
"prettier": "@flarum/prettier-config",
"dependencies": {
"@askvortsov/rich-icu-message-formatter": "^0.2.4",
"@ultraq/icu-message-formatter": "^0.12.0",
"body-scroll-lock": "^4.0.0-beta.0",
"bootstrap": "^3.4.1",
"clsx": "^1.1.1",
"color-thief-browser": "^2.0.2",
"dayjs": "^1.10.7",
"focus-trap": "^6.7.1",
"format-message": "^6.2.4",
"jquery": "^3.6.0",
"jquery.hotkeys": "^0.1.0",
"mithril": "^2.2",

View File

@ -1,26 +0,0 @@
declare module '@askvortsov/rich-icu-message-formatter' {
type IValues = Record<string, any>;
type ITypeHandler = (
value: string,
matches: string,
locale: string,
values: IValues,
format: (message: string, values: IValues) => string
) => string;
type IRichHandler = (tag: any, values: IValues, contents: string) => any;
type ValueOrArray<T> = T | ValueOrArray<T>[];
export type NestedStringArray = ValueOrArray<string>;
export class RichMessageFormatter {
locale: string | null;
constructor(locale: string | null, typeHandlers: Record<string, ITypeHandler>, richHandler: IRichHandler);
format(message: string, values: IValues): string;
process(message: string, values: IValues): NestedStringArray;
rich(message: string, values: IValues): NestedStringArray;
}
export function mithrilRichHandler(tag: any, values: IValues, contents: string): any;
}

View File

@ -1,17 +0,0 @@
declare module '@ultraq/icu-message-formatter' {
export function pluralTypeHandler(
value: string,
matches: string,
locale: string,
values: Record<string, any>,
format: (text: string, values: Record<string, any>) => string
): string;
export function selectTypeHandler(
value: string,
matches: string,
locale: string,
values: Record<string, any>,
format: (text: string, values: Record<string, any>) => string
): string;
}

View File

@ -1,13 +1,13 @@
import type { Dayjs } from 'dayjs';
import { RichMessageFormatter, mithrilRichHandler, NestedStringArray } from '@askvortsov/rich-icu-message-formatter';
import { pluralTypeHandler, selectTypeHandler } from '@ultraq/icu-message-formatter';
import username from './helpers/username';
import type { Dayjs } from 'dayjs';
import User from './models/User';
import extract from './utils/extract';
import formatMessage, { Translation } from 'format-message';
import fireDebugWarning from './helpers/fireDebugWarning';
import extractText from './utils/extractText';
import ItemList from './utils/ItemList';
type Translations = Record<string, string>;
type Translations = { [key: string]: string | Translation };
type TranslatorParameters = Record<string, unknown>;
type DateTimeFormatCallback = (id?: string) => string | void;
@ -15,7 +15,9 @@ export default class Translator {
/**
* A map of translation keys to their translated values.
*/
translations: Translations = {};
get translations(): Translations {
return this.formatter.setup().translations[this.getLocale()] ?? {};
}
/**
* A item list of date time format callbacks.
@ -25,44 +27,44 @@ export default class Translator {
/**
* The underlying ICU MessageFormatter util.
*/
protected formatter = new RichMessageFormatter(null, this.formatterTypeHandlers(), mithrilRichHandler);
protected formatter = formatMessage;
/**
* Sets the formatter's locale to the provided value.
*/
setLocale(locale: string) {
this.formatter.locale = locale;
this.formatter.setup({
locale,
translations: {
[locale]: this.formatter.setup().translations[locale] ?? {},
},
});
}
/**
* Returns the formatter's current locale.
*/
getLocale() {
return this.formatter.locale;
getLocale(): string {
return (Array.isArray(this.formatter.setup().locale) ? this.formatter.setup().locale[0] : this.formatter.setup().locale) as string;
}
addTranslations(translations: Translations) {
Object.assign(this.translations, translations);
}
const locale = this.getLocale();
/**
* An extensible entrypoint for extenders to register type handlers for translations.
*/
protected formatterTypeHandlers() {
return {
plural: pluralTypeHandler,
select: selectTypeHandler,
};
this.formatter.setup({
translations: {
[locale]: Object.assign(this.translations, translations),
},
});
}
/**
* A temporary system to preprocess parameters.
* Should not be used by extensions.
* TODO: An extender will be added in v1.x.
*
* @internal
*/
protected preprocessParameters(parameters: TranslatorParameters) {
protected preprocessParameters(parameters: TranslatorParameters, translation: string | Translation) {
// If we've been given a user model as one of the input parameters, then
// we'll extract the username and use that for the translation. In the
// future there should be a hook here to inspect the user and change the
@ -75,23 +77,66 @@ export default class Translator {
if (!parameters.username) parameters.username = username(user);
}
// To maintain backwards compatibility, we will catch HTML elements and
// push the tags as mithril children to the parameters keyed by the tag name.
// Will be removed in v2.0
translation = typeof translation === 'string' ? translation : translation.message;
const elements = translation.match(/<(\w+)[^>]*>.*?<\/\1>/g);
const tags = elements?.map((element) => element.match(/^<(\w+)/)![1]) || [];
for (const tag of tags) {
if (!parameters[tag]) {
fireDebugWarning(
`Any HTML tags used within translations must have corresponding mithril component parameters.\nCaught in translation: \n\n"""\n${translation}\n"""`,
'',
'v2.0',
'flarum/framework'
);
parameters[tag] = ({ children }: any) => m(tag, children);
}
}
// The old formatter allowed rich parameters as such:
// { link: <Link href="https://flarum.org"/> }
// The new formatter dictates that the rich parameter must be a function,
// like so: { link: ({ children }) => <Link href="https://flarum.org">{children}</Link> }
// This layer allows the old format to be used, and converts it to the new format.
for (const key in parameters) {
const value: any = parameters[key];
if (tags.includes(key) && typeof value === 'object' && value.attrs && value.tag) {
parameters[key] = ({ children }: any) => {
return m(value.tag, value.attrs, children);
};
}
}
return parameters;
}
trans(id: string, parameters: TranslatorParameters): NestedStringArray;
trans(id: string, parameters: TranslatorParameters, extract: false): NestedStringArray;
trans(id: string, parameters: TranslatorParameters): any[];
trans(id: string, parameters: TranslatorParameters, extract: false): any[];
trans(id: string, parameters: TranslatorParameters, extract: true): string;
trans(id: string): NestedStringArray | string;
trans(id: string): any[] | string;
trans(id: string, parameters: TranslatorParameters = {}, extract = false) {
const translation = this.translations[id];
const translation = this.preprocessTranslation(this.translations[id]);
if (translation) {
parameters = this.preprocessParameters(parameters);
const locale = this.formatter.rich(translation, parameters);
parameters = this.preprocessParameters(parameters, translation);
this.translations[id] = translation;
let locale = this.formatter.rich({ id, default: id }, parameters);
// convert undefined args to {undefined}.
locale = locale instanceof Array ? locale.map((arg) => (arg === undefined ? '{undefined}' : arg)) : locale;
if (extract) return extractText(locale);
return locale;
} else {
fireDebugWarning(`Missing translation for key: "${id}"`);
}
return id;
@ -113,6 +158,32 @@ export default class Translator {
if (result) return result;
}
return time.format(this.translations[id]);
return time.format(this.preprocessTranslation(this.translations[id]));
}
/**
* Backwards compatibility for translations such as `<a href='{href}'>`, the old
* formatter supported that, but the new one doesn't, so attributes are auto dropped
* to avoid errors.
*
* @private
*/
private preprocessTranslation(translation: string | Translation | undefined): string | undefined {
if (!translation) return;
translation = typeof translation === 'string' ? translation : translation.message;
// If the translation contains a <x ...attrs> tag, then we'll need to
// remove the attributes for backwards compatibility. Will be removed in v2.0.
// And if it did have attributes, then we'll fire a warning
if (translation.match(/<\w+ [^>]+>/g)) {
fireDebugWarning(
`Any HTML tags used within translations must be simple tags, without attributes.\nCaught in translation: \n\n"""\n${translation}\n"""`
);
return translation.replace(/<(\w+)([^>]*)>/g, '<$1>');
}
return translation;
}
}

View File

@ -12,7 +12,7 @@ import app from '../app';
* can fix.
*/
export default function fireDebugWarning(...args: Parameters<typeof console.warn>): void {
if (!app.forum.attribute('debug')) return;
if (!app.data.resources.find((r) => r.type === 'forums')?.attributes?.debug) return;
console.warn(...args);
}

View File

@ -9,7 +9,6 @@ import extractText from './extractText';
* // "1.2K"
*/
export default function abbreviateNumber(number: number): string {
// TODO: translation
if (number >= 1000000) {
return Math.floor(number / 1000000) + extractText(app.translator.trans('core.lib.number_suffix.mega_text'));
} else if (number >= 1000) {

View File

@ -9,7 +9,6 @@ import classList from '../../common/utils/classList';
import Tooltip from '../../common/components/Tooltip';
import type Mithril from 'mithril';
import type AccessToken from '../../common/models/AccessToken';
import { NestedStringArray } from '@askvortsov/rich-icu-message-formatter';
import Icon from '../../common/components/Icon';
export interface IAccessTokensListAttrs extends ComponentAttrs {
@ -187,7 +186,7 @@ export default class AccessTokensList<CustomAttrs extends IAccessTokensListAttrs
m.redraw();
}
generateTokenTitle(token: AccessToken): NestedStringArray {
generateTokenTitle(token: AccessToken): any[] | string {
const name = token.title() || app.translator.trans('core.forum.security.token_title_placeholder');
const value = this.tokenValueDisplay(token);

View File

@ -0,0 +1,69 @@
import Translator from '../../../../src/common/Translator';
import extractText from '../../../../src/common/utils/extractText';
/*
* These tests should be in sync with PHP tests in `tests/unit/Locale/TranslatorTest.php`, to make sure that JS
* translator works in the same way as JS translator.
*/
test('placeholders encoding', () => {
const translator = new Translator();
translator.addTranslations({
test1: 'test1 {placeholder} test1',
test2: 'test2 {placeholder} test2',
});
expect(extractText(translator.trans('test1', { placeholder: "'" }))).toBe("test1 ' test1");
expect(extractText(translator.trans('test1', { placeholder: translator.trans('test2', { placeholder: "'" }) }))).toBe("test1 test2 ' test2 test1");
});
// This is how the backend translator behaves. The only discrepancy with the frontend translator.
// test('missing placeholders', () => {
// const translator = new Translator();
// translator.addTranslations({
// test1: 'test1 {placeholder} test1',
// });
//
// expect(extractText(translator.trans('test1', {}))).toBe('test1 {placeholder} test1');
// });
test('missing placeholders', () => {
const translator = new Translator();
translator.addTranslations({
test1: 'test1 {placeholder} test1',
});
expect(extractText(translator.trans('test1', {}))).toBe('test1 {undefined} test1');
});
test('escaped placeholders', () => {
const translator = new Translator();
translator.addTranslations({
test3: "test1 {placeholder} '{placeholder}' test1",
});
expect(extractText(translator.trans('test3', { placeholder: "'" }))).toBe("test1 ' {placeholder} test1");
});
test('plural rules', () => {
const translator = new Translator();
translator.addTranslations({
test4: '{pageNumber, plural, =1 {{forumName}} other {Page # - {forumName}}}',
});
expect(extractText(translator.trans('test4', { forumName: 'A & B', pageNumber: 1 }))).toBe('A & B');
expect(extractText(translator.trans('test4', { forumName: 'A & B', pageNumber: 2 }))).toBe('Page 2 - A & B');
});
test('plural rules 2', () => {
const translator = new Translator();
translator.setLocale('pl');
translator.addTranslations({
test5: '{count, plural, one {# post} few {# posty} many {# postów} other {# posta}}',
});
expect(extractText(translator.trans('test5', { count: 1 }))).toBe('1 post');
expect(extractText(translator.trans('test5', { count: 2 }))).toBe('2 posty');
expect(extractText(translator.trans('test5', { count: 5 }))).toBe('5 postów');
expect(extractText(translator.trans('test5', { count: 1.5 }))).toBe('1,5 posta');
});

View File

@ -0,0 +1,91 @@
<?php
/*
* This file is part of Flarum.
*
* For detailed copyright and license information, please view the
* LICENSE file that was distributed with this source code.
*/
namespace Flarum\Tests\unit\Locale;
use Flarum\Locale\Translator;
use Flarum\Testing\unit\TestCase;
use Symfony\Component\Translation\Loader\ArrayLoader;
use Symfony\Component\Translation\MessageCatalogueInterface;
class TranslatorTest extends TestCase
{
private const DOMAIN = 'messages'.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX;
/*
* These tests should be in sync with JS tests in `js/tests/unit/common/utils/Translator.test.ts`, to make sure that JS
* translator works in the same way as JS translator.
*/
/** @test */
public function placeholders_encoding()
{
$translator = new Translator('en');
$translator->addLoader('array', new ArrayLoader());
$translator->addResource('array', [
'test1' => 'test1 {placeholder} test1',
'test2' => 'test2 {placeholder} test2',
], 'en', self::DOMAIN);
$this->assertSame("test1 ' test1", $translator->trans('test1', ['placeholder' => "'"]));
$this->assertSame("test1 test2 ' test2 test1", $translator->trans('test1', ['placeholder' => $translator->trans('test2', ['placeholder' => "'"])]));
}
/** @test */
public function missing_placeholders()
{
$translator = new Translator('en');
$translator->addLoader('array', new ArrayLoader());
$translator->addResource('array', [
'test1' => 'test1 {placeholder} test1',
], 'en', self::DOMAIN);
$this->assertSame('test1 {placeholder} test1', $translator->trans('test1', []));
}
/** @test */
public function escaped_placeholders()
{
$translator = new Translator('en');
$translator->addLoader('array', new ArrayLoader());
$translator->addResource('array', [
'test3' => "test1 {placeholder} '{placeholder}' test1",
], 'en', self::DOMAIN);
$this->assertSame("test1 ' {placeholder} test1", $translator->trans('test3', ['placeholder' => "'"]));
}
/** @test */
public function plural_rules()
{
$translator = new Translator('en');
$translator->addLoader('array', new ArrayLoader());
$translator->addResource('array', [
'test4' => '{pageNumber, plural, =1 {{forumName}} other {Page # - {forumName}}}',
], 'en', self::DOMAIN);
$this->assertSame('A & B', $translator->trans('test4', ['forumName' => 'A & B', 'pageNumber' => 1]));
$this->assertSame('Page 2 - A & B', $translator->trans('test4', ['forumName' => 'A & B', 'pageNumber' => 2]));
}
/** @test */
public function plural_rules_2()
{
$translator = new Translator('pl');
$translator->addLoader('array', new ArrayLoader());
$translator->addResource('array', [
'test4' => '{count, plural, one {# post} few {# posty} many {# postów} other {# posta}}',
], 'pl', self::DOMAIN);
$this->assertSame('1 post', $translator->trans('test4', ['count' => 1]));
$this->assertSame('2 posty', $translator->trans('test4', ['count' => 2]));
$this->assertSame('5 postów', $translator->trans('test4', ['count' => 5]));
$this->assertSame('1,5 posta', $translator->trans('test4', ['count' => 1.5]));
}
}

View File

@ -36,6 +36,7 @@ export default function bootstrap(Application, app, payload = {}) {
...payload,
});
app.translator.setLocale('en');
app.translator.addTranslations(flatten(jsYaml.load(fs.readFileSync('../locale/core.yml', 'utf8'))));
app.drawer = new Drawer();
}

View File

@ -10,15 +10,6 @@
"@jridgewell/gen-mapping" "^0.3.5"
"@jridgewell/trace-mapping" "^0.3.24"
"@askvortsov/rich-icu-message-formatter@^0.2.4":
version "0.2.4"
resolved "https://registry.yarnpkg.com/@askvortsov/rich-icu-message-formatter/-/rich-icu-message-formatter-0.2.4.tgz#5810886d6d6751e9b800640748355a87ea985556"
integrity sha512-JOdZ7iw7qF3uxC3cfY8dighM3rgrV0WufgwVeFD9VEkxB7IwA7DX2kHs24zk4CYPR6HQXUEnM6fwOy+VKUrc8w==
dependencies:
"@babel/runtime" "^7.11.2"
"@ultraq/array-utils" "^2.1.0"
"@ultraq/icu-message-formatter" "^0.12.0"
"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.25.7":
version "7.25.7"
resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.25.7.tgz#438f2c524071531d643c6f0188e1e28f130cebc7"
@ -987,7 +978,7 @@
"@babel/plugin-transform-modules-commonjs" "^7.25.7"
"@babel/plugin-transform-typescript" "^7.25.7"
"@babel/runtime@^7.1.2", "@babel/runtime@^7.11.2", "@babel/runtime@^7.20.1", "@babel/runtime@^7.8.4":
"@babel/runtime@^7.1.2", "@babel/runtime@^7.20.1", "@babel/runtime@^7.8.4":
version "7.25.7"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.7.tgz#7ffb53c37a8f247c8c4d335e89cdf16a2e0d0fb6"
integrity sha512-FjoyLe754PMiYsFaN5C94ttGiOmBNYTf6pLr4xXHAT5uctHb092PBszndLDR5XA/jghQvn4n7JMHl7dmTgbm9w==
@ -1545,25 +1536,6 @@
dependencies:
"@types/yargs-parser" "*"
"@ultraq/array-utils@^2.1.0":
version "2.1.0"
resolved "https://registry.yarnpkg.com/@ultraq/array-utils/-/array-utils-2.1.0.tgz#56f16a1ea3ef46c5d5f04638b47c4fca4d71a8c1"
integrity sha512-TKO1zE6foqs5HG3+QH32yKwJ0zhZrm6J3UmltscveQmxCdbgIPXhNf3A8C9HakjyZDHVRK5pYZOU0tTl28YGFg==
"@ultraq/function-utils@^0.3.0":
version "0.3.0"
resolved "https://registry.yarnpkg.com/@ultraq/function-utils/-/function-utils-0.3.0.tgz#63eb7dceff18fdca212fae11a59b3ee01f556917"
integrity sha512-AwFCYorRn0GE34hfgxaCmfnReHqcwWE6QwWPQf/1Zj7k3Zi0FATSJhbtDA+6ayV8p6AnhEntntXaMWMkK17tEQ==
"@ultraq/icu-message-formatter@^0.12.0":
version "0.12.0"
resolved "https://registry.yarnpkg.com/@ultraq/icu-message-formatter/-/icu-message-formatter-0.12.0.tgz#15a812a323395d7e5b5e3c6c2cc92df3989b26ce"
integrity sha512-ebd/ZyC1lCVPPrX3AQ9h77NDK4d1nor0Grmv43e97+omWvJB29lbuT+9yM3sq4Ri1QKwTvKG1BUhXBz0oAAR2w==
dependencies:
"@babel/runtime" "^7.11.2"
"@ultraq/array-utils" "^2.1.0"
"@ultraq/function-utils" "^0.3.0"
"@webassemblyjs/ast@1.12.1", "@webassemblyjs/ast@^1.12.1":
version "1.12.1"
resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.12.1.tgz#bb16a0e8b1914f979f45864c23819cc3e3f0d4bb"
@ -2723,6 +2695,34 @@ form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
format-message-formats@^6.2.4:
version "6.2.4"
resolved "https://registry.yarnpkg.com/format-message-formats/-/format-message-formats-6.2.4.tgz#68b782e70c3c15f017377848c3225731e52ac4ea"
integrity sha512-smT/fAqBLqusWfWCKRAx6QBDAAbmYznWsIyTyk66COmvwt2Byiqd7SJe2ma9a5oV0kwRaOJpN/F4lr4YK/n6qQ==
format-message-interpret@^6.2.4:
version "6.2.4"
resolved "https://registry.yarnpkg.com/format-message-interpret/-/format-message-interpret-6.2.4.tgz#28f579b9cd4b57f3de2ec2a4d9623f9870e9ed03"
integrity sha512-dRvz9mXhITApyOtfuFEb/XqvCe1u6RMkQW49UJHXS8w2S8cAHCqq5LNDFK+QK6XVzcofROycLb/k1uybTAKt2w==
dependencies:
format-message-formats "^6.2.4"
lookup-closest-locale "^6.2.0"
format-message-parse@^6.2.4:
version "6.2.4"
resolved "https://registry.yarnpkg.com/format-message-parse/-/format-message-parse-6.2.4.tgz#2c9b39a32665bd247cb1c31ba2723932d9edf3f9"
integrity sha512-k7WqXkEzgXkW4wkHdS6Cv2Ou0rIFtiDelZjgoe1saW4p7FT7zS8OeAUpAekhormqzpeecR97e4vBft1zMsfFOQ==
format-message@^6.2.4:
version "6.2.4"
resolved "https://registry.yarnpkg.com/format-message/-/format-message-6.2.4.tgz#0bd4b6161b036e3fbcf3207dce14a62e318b4c48"
integrity sha512-/24zYeSRy2ZlEO2OIctm7jOHvMpoWf+uhqFCaqqyZKi1C229zAAy2E5vF4lSSaMH0a2kewPrOzq6xN4Yy7cQrw==
dependencies:
format-message-formats "^6.2.4"
format-message-interpret "^6.2.4"
format-message-parse "^6.2.4"
lookup-closest-locale "^6.2.0"
frappe-charts@^1.6.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-1.6.2.tgz#4671a943a8606e5020180fa65c8ea1835c510baf"
@ -3748,6 +3748,11 @@ lodash@^4.17.15:
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
lookup-closest-locale@^6.2.0:
version "6.2.0"
resolved "https://registry.yarnpkg.com/lookup-closest-locale/-/lookup-closest-locale-6.2.0.tgz#57f665e604fd26f77142d48152015402b607bcf3"
integrity sha512-/c2kL+Vnp1jnV6K6RpDTHK3dgg0Tu2VVp+elEiJpjfS1UyY7AjOYHohRug6wT0OpoX2qFgNORndE9RqesfVxWQ==
loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"