Lay the groundwork for translation & refactor asset compilation

Ditched the idea of having language packs as extensions. Reasoning:

1. Because we use machine keys for translations (rather than English
keys), extensions need to be able to define default translations. If
English translations are to be included in extensions and not in a
language pack extension, then it doesn’t make sense to have other
languages as language pack extensions. Inconsistency → complexity.

2. Translations should maintain version parity with their respective
extensions. There’s no way to do this if extension translations are
external to the extension.

Instead, localisation will be a core effort, as well as a per-extension
effort. Translators will be encouraged to send PRs to core + extensions.

In core, each locale has a directory containing three files:
- translations.yml
- config.js: contains pluralisation logic for the JS app, as well as
moment.js localisation if necessary
- config.php: contains pluralisation logic for the PHP app

Extensions can use the Flarum\Extend\Locale extender to add/override
translations/config to a locale.

Asset compilation has been completely refactored with a better
architecture. Translations + config.js are compiled and cached for the
currently active locale.

framework/core/js/lib/utils/app.js

@@ -1,10 +1,12 @@
 import ItemList from 'flarum/utils/item-list';
 import Alert from 'flarum/components/alert';
 import ServerError from 'flarum/utils/server-error';
+import Translator from 'flarum/utils/translator';
 
 class App {
   constructor() {
     this.initializers = new ItemList();
+    this.translator = new Translator();
     this.cache = {};
     this.serverError = null;
   }
@@ -55,6 +57,10 @@ class App {
     var queryString = m.route.buildQueryString(params);
     return url+(queryString ? '?'+queryString : '');
   }
+
+  translate(key, input) {
+    return this.translator.translate(key, input);
+  }
 }
 
 export default App;

framework/core/js/lib/utils/translator.js

@@ -0,0 +1,32 @@
+export default class Translator {
+  constructor() {
+    this.translations = {};
+  }
+
+  plural(count) {
+    return count == 1 ? 'one' : 'other';
+  }
+
+  translate(key, input) {
+    var parts = key.split('.');
+    var translation = this.translations;
+
+    parts.forEach(function(part) {
+      translation = translation && translation[part];
+    });
+
+    if (typeof translation === 'object' && typeof input.count !== 'undefined') {
+      translation = translation[this.plural(input.count)];
+    }
+
+    if (typeof translation === 'string') {
+      for (var i in input) {
+        translation = translation.replace(new RegExp('{'+i+'}', 'gi'), input[i]);
+      }
+
+      return translation;
+    } else {
+      return key;
+    }
+  }
+}

framework/core/locale/en/config.js

@@ -0,0 +1,3 @@
+app.translator.plural = function(count) {
+  return count == 1 ? 'one' : 'other';
+};

framework/core/locale/en/config.php

@@ -0,0 +1,7 @@
+<?php
+
+return [
+    'plural' => function ($count) {
+        return $count == 1 ? 'one' : 'other';
+    }
+];

framework/core/locale/en/translations.yml

@@ -0,0 +1,2 @@
+core:
+

framework/core/migrations/2015_02_24_000000_create_users_table.php

@@ -19,6 +19,7 @@ class CreateUsersTable extends Migration
             $table->string('email', 150)->unique();
             $table->boolean('is_activated')->default(0);
             $table->string('password', 100);
+            $table->string('locale', 10)->default('en');
             $table->text('bio')->nullable();
             $table->text('bio_html')->nullable();
             $table->string('avatar_path', 100)->nullable();

framework/core/src/Assets/AssetManager.php

@@ -0,0 +1,60 @@
+<?php namespace Flarum\Assets;
+
+use RuntimeException;
+
+class AssetManager
+{
+    protected $less;
+
+    protected $js;
+
+    public function __construct(CompilerInterface $js, CompilerInterface $less)
+    {
+        $this->js = $js;
+        $this->less = $less;
+    }
+
+    public function addFile($file)
+    {
+        $ext = pathinfo($file, PATHINFO_EXTENSION);
+
+        switch ($ext) {
+            case 'js':
+                $this->js->addFile($file);
+                break;
+
+            case 'css':
+            case 'less':
+                $this->less->addFile($file);
+                break;
+
+            default:
+                throw new RuntimeException('Unsupported asset type: '.$ext);
+        }
+    }
+
+    public function addFiles(array $files)
+    {
+        array_walk($files, [$this, 'addFile']);
+    }
+
+    public function addLess($string)
+    {
+        $this->less->addString($string);
+    }
+
+    public function addJs($strings)
+    {
+        $this->js->addString($string);
+    }
+
+    public function getCssFile()
+    {
+        return $this->less->getFile();
+    }
+
+    public function getJsFile()
+    {
+        return $this->js->getFile();
+    }
+}

framework/core/src/Assets/CompilerInterface.php

@@ -0,0 +1,10 @@
+<?php namespace Flarum\Assets;
+
+interface CompilerInterface
+{
+    public function addFile($file);
+
+    public function addString($string);
+
+    public function getFile();
+}

framework/core/src/Assets/JsCompiler.php

@@ -0,0 +1,9 @@
+<?php namespace Flarum\Assets;
+
+class JsCompiler extends RevisionCompiler
+{
+    public function format($string)
+    {
+        return $string.';';
+    }
+}

framework/core/src/Assets/LessCompiler.php

@@ -0,0 +1,26 @@
+<?php namespace Flarum\Assets;
+
+use Less_Parser;
+
+class LessCompiler extends RevisionCompiler
+{
+    public function compile()
+    {
+        ini_set('xdebug.max_nesting_level', 200);
+
+        $parser = new Less_Parser([
+            'compress' => true,
+            'cache_dir' => storage_path().'/less'
+        ]);
+
+        foreach ($this->files as $file) {
+            $parser->parseFile($file);
+        }
+
+        foreach ($this->strings as $string) {
+            $parser->parse($string);
+        }
+
+        return $parser->getCss();
+    }
+}

framework/core/src/Assets/RevisionCompiler.php

@@ -0,0 +1,95 @@
+<?php namespace Flarum\Assets;
+
+use Illuminate\Support\Str;
+
+class RevisionCompiler implements CompilerInterface
+{
+    protected $files = [];
+
+    protected $strings = [];
+
+    public function __construct($path, $filename)
+    {
+        $this->path = $path;
+        $this->filename = $filename;
+    }
+
+    public function addFile($file)
+    {
+        $this->files[] = $file;
+    }
+
+    public function addString($string)
+    {
+        $this->strings[] = $string;
+    }
+
+    public function getFile()
+    {
+        if (! ($revision = $this->getRevision())) {
+            $revision = Str::quickRandom();
+            $this->putRevision($revision);
+        }
+
+        $lastModTime = 0;
+        foreach ($this->files as $file) {
+            $lastModTime = max($lastModTime, filemtime($file));
+        }
+
+        $ext = pathinfo($this->filename, PATHINFO_EXTENSION);
+        $file = $this->path.'/'.substr_replace($this->filename, '-'.$revision, -strlen($ext) - 1, 0);
+
+        if (! file_exists($file)
+            || filemtime($file) < $lastModTime) {
+            file_put_contents($file, $this->compile());
+        }
+
+        return $file;
+    }
+
+    protected function format($string)
+    {
+        return $string;
+    }
+
+    protected function compile()
+    {
+        $output = '';
+
+        foreach ($this->files as $file) {
+            $output .= $this->format(file_get_contents($file));
+        }
+
+        foreach ($this->strings as $string) {
+            $output .= $this->format($string);
+        }
+
+        return $output;
+    }
+
+    protected function getRevisionFile()
+    {
+        return $this->path.'/rev-manifest.json';
+    }
+
+    protected function getRevision()
+    {
+        if (file_exists($file = $this->getRevisionFile())) {
+            $manifest = json_decode(file_get_contents($file), true);
+            return array_get($manifest, $this->filename);
+        }
+    }
+
+    protected function putRevision($revision)
+    {
+        if (file_exists($file = $this->getRevisionFile())) {
+            $manifest = json_decode(file_get_contents($file), true);
+        } else {
+            $manifest = [];
+        }
+
+        $manifest[$this->filename] = $revision;
+
+        return file_put_contents($this->getRevisionFile(), json_encode($manifest));
+    }
+}

framework/core/src/Core/CoreServiceProvider.php

@@ -18,6 +18,7 @@ use Flarum\Core\Events\RegisterUserGambits;
 use Flarum\Extend\Permission;
 use Flarum\Extend\ActivityType;
 use Flarum\Extend\NotificationType;
+use Flarum\Extend\Locale;
 
 class CoreServiceProvider extends ServiceProvider
 {
@@ -56,6 +57,17 @@ class CoreServiceProvider extends ServiceProvider
             (new ActivityType('Flarum\Core\Activity\StartedDiscussionActivity', 'Flarum\Api\Serializers\PostBasicSerializer')),
             (new ActivityType('Flarum\Core\Activity\JoinedActivity', 'Flarum\Api\Serializers\UserBasicSerializer'))
         );
+
+        foreach (['en'] as $locale) {
+            $dir = __DIR__.'/../../locale/'.$locale;
+
+            $this->extend(
+                (new Locale($locale))
+                    ->translations($dir.'/translations.yml')
+                    ->config($dir.'/config.php')
+                    ->js($dir.'/config.js')
+            );
+        }
     }
 
     /**
@@ -72,6 +84,8 @@ class CoreServiceProvider extends ServiceProvider
 
         $this->app->singleton('flarum.formatter', 'Flarum\Core\Formatter\FormatterManager');
 
+        $this->app->singleton('flarum.localeManager', 'Flarum\Locale\LocaleManager');
+
         $this->app->bind(
             'Flarum\Core\Repositories\DiscussionRepositoryInterface',
             'Flarum\Core\Repositories\EloquentDiscussionRepository'

framework/core/src/Core/Seeders/ConfigTableSeeder.php

@@ -20,6 +20,7 @@ class ConfigTableSeeder extends Seeder
             'welcome_message'       => 'Flarum is now at a point where you can have basic conversations, so here is a little demo for you to break.',
             'welcome_title'         => 'Welcome to Flarum Demo Forum',
             'extensions_enabled'    => '[]',
+            'locale'                => 'en',
             'theme_primary_color'   => '#536F90',
             'theme_secondary_color' => '#536F90',
             'theme_dark_mode'       => false,

framework/core/src/Extend/AdminTranslations.php

@@ -0,0 +1,18 @@
+<?php namespace Flarum\Extend;
+
+use Illuminate\Contracts\Container\Container;
+
+class AdminTranslations implements ExtenderInterface
+{
+    protected $keys;
+
+    public function __construct($keys)
+    {
+        $this->keys = $keys;
+    }
+
+    public function extend(Container $container)
+    {
+
+    }
+}

framework/core/src/Extend/ForumAssets.php

@@ -14,7 +14,7 @@ class ForumAssets implements ExtenderInterface
     public function extend(Application $app)
     {
         $app['events']->listen('Flarum\Forum\Events\RenderView', function ($event) {
-            $event->assets->addFile($this->files);
+            $event->assets->addFiles($this->files);
         });
     }
 }

framework/core/src/Extend/ForumTranslations.php

@@ -0,0 +1,19 @@
+<?php namespace Flarum\Extend;
+
+use Illuminate\Foundation\Application;
+use Flarum\Forum\Actions\IndexAction;
+
+class ForumTranslations implements ExtenderInterface
+{
+    protected $keys;
+
+    public function __construct($keys)
+    {
+        $this->keys = $keys;
+    }
+
+    public function extend(Application $container)
+    {
+        IndexAction::$translations = array_merge(IndexAction::$translations, $this->keys);
+    }
+}

framework/core/src/Extend/Locale.php

@@ -0,0 +1,57 @@
+<?php namespace Flarum\Extend;
+
+use Illuminate\Foundation\Application;
+
+class Locale implements ExtenderInterface
+{
+    protected $locale;
+
+    protected $translations;
+
+    protected $config;
+
+    protected $js;
+
+    public function __construct($locale)
+    {
+        $this->locale = $locale;
+    }
+
+    public function translations($translations)
+    {
+        $this->translations = $translations;
+
+        return $this;
+    }
+
+    public function config($config)
+    {
+        $this->config = $config;
+
+        return $this;
+    }
+
+    public function js($js)
+    {
+        $this->js = $js;
+
+        return $this;
+    }
+
+    public function extend(Application $container)
+    {
+        $manager = $container->make('flarum.localeManager');
+
+        if ($this->translations) {
+            $manager->addTranslations($this->locale, $this->translations);
+        }
+
+        if ($this->config) {
+            $manager->addConfig($this->locale, $this->config);
+        }
+
+        if ($this->js) {
+            $manager->addJsFile($this->locale, $this->js);
+        }
+    }
+}

framework/core/src/Forum/Actions/IndexAction.php

@@ -10,9 +10,15 @@ use DB;
 use Flarum\Forum\Events\RenderView;
 use Flarum\Api\Request as ApiRequest;
 use Flarum\Core;
+use Flarum\Assets\AssetManager;
+use Flarum\Assets\JsCompiler;
+use Flarum\Assets\LessCompiler;
+use Flarum\Locale\JsCompiler as LocaleJsCompiler;
 
 class IndexAction extends BaseAction
 {
+    public static $translations = [];
+
     public function handle(Request $request, $params = [])
     {
         $config = DB::table('config')->whereIn('key', ['base_url', 'api_url', 'forum_title', 'welcome_title', 'welcome_message'])->lists('value', 'key');
@@ -44,23 +50,63 @@ class IndexAction extends BaseAction
             ->with('session', $session)
             ->with('alert', $alert);
 
-        $assetManager = app('flarum.forum.assetManager');
         $root = __DIR__.'/../../..';
-        $assetManager->addFile([
+        $public = public_path().'/assets';
-            $root.'/js/forum/dist/app.js',
-            $root.'/less/forum/app.less'
-        ]);
-        $assetManager->addLess('
-            @fl-primary-color: '.Core::config('theme_primary_color').';
-            @fl-secondary-color: '.Core::config('theme_secondary_color').';
-            @fl-dark-mode: '.(Core::config('theme_dark_mode') ? 'true' : 'false').';
-            @fl-colored_header: '.(Core::config('theme_colored_header') ? 'true' : 'false').';
-        ');
 
-        event(new RenderView($view, $assetManager, $this));
+        $assets = new AssetManager(
+            new JsCompiler($public, 'forum.js'),
+            new LessCompiler($public, 'forum.css')
+        );
+
+        $assets->addFile($root.'/js/forum/dist/app.js');
+        $assets->addFile($root.'/less/forum/app.less');
+
+        $variables = [
+            'fl-primary-color'   => Core::config('theme_primary_color', '#000'),
+            'fl-secondary-color' => Core::config('theme_secondary_color', '#000'),
+            'fl-dark-mode'       => Core::config('theme_dark_mode') ? 'true' : 'false',
+            'fl-colored-header'  => Core::config('theme_colored_header') ? 'true' : 'false'
+        ];
+        foreach ($variables as $name => $value) {
+            $assets->addLess("@$name: $value;");
+        }
+
+        $locale = $user->locale ?: Core::config('locale', 'en');
+
+        $localeManager = app('flarum.localeManager');
+        $translations = $localeManager->getTranslations($locale);
+        $jsFiles = $localeManager->getJsFiles($locale);
+
+        $localeCompiler = new LocaleJsCompiler($public, 'locale-'.$locale.'.js');
+        $localeCompiler->setTranslations(static::filterTranslations($translations));
+        array_walk($jsFiles, [$localeCompiler, 'addFile']);
+
+        event(new RenderView($view, $assets, $this));
 
         return $view
-            ->with('styles', $assetManager->getCSSFiles())
+            ->with('styles', [$assets->getCssFile()])
-            ->with('scripts', $assetManager->getJSFiles());
+            ->with('scripts', [$assets->getJsFile(), $localeCompiler->getFile()]);
+    }
+
+    protected static function filterTranslations($translations)
+    {
+        $filtered = [];
+
+        foreach (static::$translations as $key) {
+            $parts = explode('.', $key);
+            $level = &$filtered;
+
+            foreach ($parts as $part) {
+                if (! isset($level[$part])) {
+                    $level[$part] = [];
+                }
+
+                $level = &$level[$part];
+            }
+
+            $level = array_get($translations, $key);
+        }
+
+        return $filtered;
     }
 }

framework/core/src/Forum/ForumServiceProvider.php

@@ -1,7 +1,8 @@
 <?php namespace Flarum\Forum;
 
-use Illuminate\Support\ServiceProvider;
+use Flarum\Support\ServiceProvider;
 use Flarum\Support\AssetManager;
+use Flarum\Extend\ForumTranslations;
 
 class ForumServiceProvider extends ServiceProvider
 {
@@ -21,6 +22,12 @@ class ForumServiceProvider extends ServiceProvider
         ]);
 
         include __DIR__.'/routes.php';
+
+        $this->extend(
+            new ForumTranslations([
+                //
+            ])
+        );
     }
 
     /**

framework/core/src/Locale/JsCompiler.php

@@ -0,0 +1,24 @@
+<?php namespace Flarum\Locale;
+
+use Flarum\Assets\RevisionCompiler;
+
+class JsCompiler extends RevisionCompiler
+{
+    protected $translations = [];
+
+    public function setTranslations(array $translations)
+    {
+        $this->translations = $translations;
+    }
+
+    public function compile()
+    {
+        $output = "var app = require('flarum/app')['default']; app.translator.translations = ".json_encode($this->translations).";";
+
+        foreach ($this->files as $filename) {
+            $output .= file_get_contents($filename);
+        }
+
+        return $output;
+    }
+}

framework/core/src/Locale/LocaleManager.php

@@ -0,0 +1,65 @@
+<?php namespace Flarum\Locale;
+
+class LocaleManager
+{
+    protected $translations = [];
+
+    protected $js = [];
+
+    protected $config = [];
+
+    public function addTranslations($locale, $translations)
+    {
+        if (! isset($this->translations[$locale])) {
+            $this->translations[$locale] = [];
+        }
+
+        $this->translations[$locale][] = $translations;
+    }
+
+    public function addJsFile($locale, $js)
+    {
+        if (! isset($this->js[$locale])) {
+            $this->js[$locale] = [];
+        }
+
+        $this->js[$locale][] = $js;
+    }
+
+    public function addConfig($locale, $config)
+    {
+        if (! isset($this->config[$locale])) {
+            $this->config[$locale] = [];
+        }
+
+        $this->config[$locale][] = $config;
+    }
+
+    public function getTranslations($locale)
+    {
+        $files = array_get($this->translations, $locale, []);
+
+        $parts = explode('-', $locale);
+
+        if (count($parts) > 1) {
+            $files = array_merge(array_get($this->translations, $parts[0], []), $files);
+        }
+
+        $compiler = new TranslationCompiler($locale, $files);
+
+        return $compiler->getTranslations();
+    }
+
+    public function getJsFiles($locale)
+    {
+        $files = array_get($this->js, $locale, []);
+
+        $parts = explode('-', $locale);
+
+        if (count($parts) > 1) {
+            $files = array_merge(array_get($this->js, $parts[0], []), $files);
+        }
+
+        return $files;
+    }
+}

framework/core/src/Locale/TranslationCompiler.php

@@ -0,0 +1,29 @@
+<?php namespace Flarum\Locale;
+
+use Symfony\Component\Yaml\Yaml;
+
+class TranslationCompiler
+{
+    protected $locale;
+
+    protected $filenames;
+
+    public function __construct($locale, array $filenames)
+    {
+        $this->locale = $locale;
+        $this->filenames = $filenames;
+    }
+
+    public function getTranslations()
+    {
+        // @todo caching
+
+        $translations = [];
+
+        foreach ($this->filenames as $filename) {
+            $translations = array_replace_recursive($translations, Yaml::parse(file_get_contents($filename)));
+        }
+
+        return $translations;
+    }
+}

framework/core/src/Locale/Translator.php

@@ -0,0 +1,42 @@
+<?php namespace Flarum\Locale;
+
+use Closure;
+
+class Translator
+{
+    protected $translations;
+
+    protected $plural;
+
+    public function __construct(array $translations, Closure $plural)
+    {
+        $this->translations = $translations;
+        $this->plural = $plural;
+    }
+
+    public function plural($count)
+    {
+        $callback = $this->plural;
+
+        return $callback($count);
+    }
+
+    public function translate($key, array $input = [])
+    {
+        $translation = array_get($this->translations, $key);
+
+        if (is_array($translation) && isset($input['count'])) {
+            $translation = $translation[$this->plural($input['count'])];
+        }
+
+        if (is_string($translation)) {
+            foreach ($input as $k => $v) {
+                $translation = str_replace('{'.$k.'}', $v, $translation);
+            }
+
+            return $translation;
+        } else {
+            return $key;
+        }
+    }
+}

framework/core/src/Support/AssetManager.php

@@ -1,163 +0,0 @@
-<?php namespace Flarum\Support;
-
-use Illuminate\Filesystem\Filesystem;
-use Illuminate\Support\Str;
-use Cache;
-use Less_Parser;
-use Closure;
-
-class AssetManager
-{
-    protected $files = [
-        'css' => [],
-        'js' => [],
-        'less' => []
-    ];
-
-    protected $less = [];
-
-    protected $publicPath;
-
-    protected $name;
-
-    protected $storage;
-
-    public function __construct(Filesystem $storage, $publicPath, $name)
-    {
-        $this->storage = $storage;
-        $this->publicPath = $publicPath;
-        $this->name = $name;
-    }
-
-    public function addFile($files)
-    {
-        foreach ((array) $files as $file) {
-            $ext = pathinfo($file, PATHINFO_EXTENSION);
-            $this->files[$ext][] = $file;
-        }
-    }
-
-    public function addLess($strings)
-    {
-        foreach ((array) $strings as $string) {
-            $this->less[] = $string;
-        }
-    }
-
-    protected function getAssetDirectory()
-    {
-        $dir = $this->publicPath;
-        if (! $this->storage->isDirectory($dir)) {
-            $this->storage->makeDirectory($dir);
-        }
-        return $dir;
-    }
-
-    protected function getRevisionFile()
-    {
-        return $this->getAssetDirectory().'/'.$this->name;
-    }
-
-    protected function getRevision()
-    {
-        if (file_exists($file = $this->getRevisionFile())) {
-            return file_get_contents($file);
-        }
-    }
-
-    protected function putRevision($revision)
-    {
-        return file_put_contents($this->getRevisionFile(), $revision);
-    }
-
-    protected function getFiles($type, Closure $callback)
-    {
-        $dir = $this->getAssetDirectory();
-
-        if (! ($revision = $this->getRevision())) {
-            $revision = Str::quickRandom();
-            $this->putRevision($revision);
-        }
-
-        $lastModTime = 0;
-        foreach ($this->files[$type] as $file) {
-            $lastModTime = max($lastModTime, filemtime($file));
-        }
-
-        if (! file_exists($file = $dir.'/'.$this->name.'-'.$revision.'.'.$type)
-            || filemtime($file) < $lastModTime) {
-            $this->storage->put($file, $callback());
-        }
-
-        return [$file];
-    }
-
-    public function clearCache()
-    {
-        if ($revision = $this->getRevision()) {
-            $dir = $this->getAssetDirectory();
-            foreach (['css', 'js'] as $type) {
-                @unlink($dir.'/'.$this->name.'-'.$revision.'.'.$type);
-            }
-        }
-    }
-
-    public function getCSSFiles()
-    {
-        return $this->getFiles('css', function () {
-            return $this->compileCSS();
-        });
-    }
-
-    public function getJSFiles()
-    {
-        return $this->getFiles('js', function () {
-            return $this->compileJS();
-        });
-    }
-
-    public function compileLess()
-    {
-        ini_set('xdebug.max_nesting_level', 200);
-
-        $parser = new Less_Parser(['compress' => true, 'cache_dir' => storage_path().'/less']);
-
-        $css = [];
-        $dir = $this->getAssetDirectory();
-        foreach ($this->files['less'] as $file) {
-            $parser->parseFile($file);
-        }
-
-        foreach ($this->less as $less) {
-            $parser->parse($less);
-        }
-
-        return $parser->getCss();
-    }
-
-    public function compileCSS()
-    {
-        $css = $this->compileLess();
-
-        foreach ($this->files['css'] as $file) {
-            $css .= $this->storage->get($file);
-        }
-
-        // minify
-
-        return $css;
-    }
-
-    public function compileJS()
-    {
-        $js = '';
-
-        foreach ($this->files['js'] as $file) {
-            $js .= $this->storage->get($file).';';
-        }
-
-        // minify
-
-        return $js;
-    }
-}