Refactor i18n so that only relevant translations are fetched

instead of bundling all translations in the dist file.

CHANGES.md

@@ -1,17 +1,31 @@
 # Changelog
 
-## 3.2.2 (Unreleased)
+## 3.3.0 (Unreleased)
 
-- Don't hang indefinitely and provide nicer error messages when a connection
-  can't be established.
-- Remove `Login` and `Registration` tabs and consolidate into one panel.
-- Add validation message for an invalid JID in the login form.
+### Bugfixes
 - Don't require `auto_login` to be `true` when using the API to log in.
-- Use CSS3 fade transitions to render various elements.
-- Consolidate error and validation reporting on the registration form.
+
+### New Features
 - #828 Add routing for the `#converse-login` and `#converse-register` URL
   fragments, which will render the registration and login forms respectively.
 
+### UX/UI changes
+- Use CSS3 fade transitions to render various elements.
+- Remove `Login` and `Registration` tabs and consolidate into one panel.
+- Show validation error messages on the login form.
+- Don't hang indefinitely and provide nicer error messages when a connection
+  can't be established.
+- Consolidate error and validation reporting on the registration form.
+
+### Technical changes
+- Converse.js now includes a [Virtual DOM](https://github.com/Matt-Esch/virtual-dom)
+  and uses it to render the login form.
+- Converse.js no longer includes all the translations in its build. Instead,
+  only the currently relevant translation is requested. This results in a much
+  smaller filesize but means that the translations you want to provide need to
+  be available. See the [locales_url](https://conversejs.org/docs/html/configurations.html#locales-url)
+  configuration setting for more info.
+
 ## 3.2.1 (2017-08-29)
 
 ### Bugfixes

docs/source/configuration.rst

@@ -648,11 +648,18 @@ state, then you can set this option to `true` to enable it.
 i18n
 ----
 
-* Default:  Auto-detection of the User/Browser language
+* Default:  Auto-detection of the User/Browser language or ``en``;
 
-If no locale is matching available locales, the default is ``en``.
-Specify the locale/language. The language must be in the ``locales`` object. Refer to
-``./locale/locales.js`` to see which locales are supported.
+Specify the locale/language.
+
+The translations for that locale must be available in JSON format at the
+`locales_url`_
+
+If an explicit locale is specified via the ``i18n`` setting and the
+translations for that locale are not found at the `locales_url``, then 
+then Converse.js will fall back to trying to determine the browser's language
+and fetching those translations, or if that fails the default English texts
+will be used.
 
 jid
 ---
@@ -692,6 +699,33 @@ See also:
     `XEP-0198 <http://xmpp.org/extensions/xep-0198.html>`_, specifically
     with regards to "stream resumption".
 
+locales_url
+-----------
+
+* Default: ``/locale/{{{locale}}}/LC_MESSAGES/converse.json``,
+
+The URL from where Converse.js should fetch translation JSON.
+
+The three curly braces ``{{{ }}}`` are
+`Mustache<https://github.com/janl/mustache.js#readme>`_-style
+variable interpolation which HTML-escapes the value being inserted. It's
+important that the inserted value is HTML-escaped, otherwise a malicious script
+injection attack could be attempted.
+
+The variable being interpolated via the curly braces is ``locale``, which is
+the value passed in to the `i18n`_ setting, or the browser's locale or the
+default local or `en` (resolved in that order).
+
+From version 3.3.0, Converse.js no longer bundles all translations into its
+final build file. Instead, only the relevant translations are fetched at
+runtime.
+
+This change also means that it's no longer possible to pass in the translation
+JSON data directly into ``_converse.initialize`` via the `i18n`_ setting.
+Instead, you only specify the language code (e.g. `de`) and that language's
+JSON translations will automatically be fetched via XMLHTTPRequest at
+``locales_url``.
+
 locked_domain
 -------------
 

locale/af/LC_MESSAGES/converse.po

@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: Converse.js 0.4\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2017-09-24 10:57+0200\n"
-"PO-Revision-Date: 2017-09-24 11:02+0200\n"
+"PO-Revision-Date: 2017-09-24 11:04+0200\n"
 "Last-Translator: JC Brand <jc@opkode.com>\n"
 "Language-Team: Afrikaans <https://hosted.weblate.org/projects/conversejs/"
 "translations/af/>\n"

spec/chatroom.js

@@ -1010,7 +1010,7 @@
 
                     _converse.connection._dataRecv(test_utils.createRequest(presence));
                     var info_text = view.$el.find('.chat-content .chat-info').text();
-                    expect(info_text).toBe('Your nickname has been automatically set to: thirdwitch');
+                    expect(info_text).toBe('Your nickname has been automatically set to thirdwitch');
                     done();
                 });
             }));
@@ -1299,7 +1299,7 @@
                  *  </x>
                  *  </presence>
                  */
-                var __ = utils.__.bind(_converse);
+                var __ = _converse.__;
                 test_utils.openAndEnterChatRoom(_converse, 'lounge', 'localhost', 'oldnick').then(function () {
                     var view = _converse.chatboxviews.get('lounge@localhost');
                     var $chat_content = view.$el.find('.chat-content');
@@ -1328,7 +1328,9 @@
 
                     expect($chat_content.find('div.chat-info').length).toBe(2);
                     expect($chat_content.find('div.chat-info:first').html()).toBe("oldnick has joined the room.");
-                    expect($chat_content.find('div.chat-info:last').html()).toBe(__(_converse.muc.new_nickname_messages["210"], "oldnick"));
+                    expect($chat_content.find('div.chat-info:last').html()).toBe(
+                        __(_converse.muc.new_nickname_messages["210"], "oldnick")
+                    );
 
                     presence = $pres().attrs({
                             from:'lounge@localhost/oldnick',
@@ -1349,7 +1351,8 @@
                     _converse.connection._dataRecv(test_utils.createRequest(presence));
                     expect($chat_content.find('div.chat-info').length).toBe(3);
                     expect($chat_content.find('div.chat-info').last().html()).toBe(
-                        __(_converse.muc.new_nickname_messages["303"], "newnick"));
+                        __(_converse.muc.new_nickname_messages["303"], "newnick")
+                    );
 
                     $occupants = view.$('.occupant-list');
                     expect($occupants.children().length).toBe(0);
@@ -1370,7 +1373,8 @@
                     _converse.connection._dataRecv(test_utils.createRequest(presence));
                     expect($chat_content.find('div.chat-info').length).toBe(4);
                     expect($chat_content.find('div.chat-info').get(2).textContent).toBe(
-                        __(_converse.muc.new_nickname_messages["303"], "newnick"));
+                        __(_converse.muc.new_nickname_messages["303"], "newnick")
+                    );
                     expect($chat_content.find('div.chat-info').last().html()).toBe(
                         "newnick has joined the room.");
                     $occupants = view.$('.occupant-list');

spec/register.js

@@ -45,7 +45,7 @@
             var $registration = $panels.children().last();
 
             var $register_link = cbview.$('a.register-account');
-            expect($register_link.text()).toBe("Register an account");
+            expect($register_link.text()).toBe("Create an account");
             $register_link.click();
             test_utils.waitUntil(function () {
                 return $registration.is(':visible');

src/converse-bookmarks.js

@@ -200,8 +200,7 @@
              * loaded by converse.js's plugin machinery.
              */
             const { _converse } = this,
-                { __,
-                ___ } = _converse;
+                  { __ } = _converse;
 
             // Configuration values for this plugin
             // ====================================
@@ -223,7 +222,7 @@
                     ev.preventDefault();
                     const name = ev.target.getAttribute('data-bookmark-name');
                     const jid = ev.target.getAttribute('data-room-jid');
-                    if (confirm(__(___("Are you sure you want to remove the bookmark \"%1$s\"?"), name))) {
+                    if (confirm(__("Are you sure you want to remove the bookmark \"%1$s\"?", name))) {
                         _.invokeMap(_converse.bookmarks.where({'jid': jid}), Backbone.Model.prototype.destroy);
                     }
                 },

src/converse-core.js

@@ -32,6 +32,19 @@
     const b64_sha1 = Strophe.SHA1.b64_sha1;
     Strophe = Strophe.Strophe;
 
+    // Add Strophe Namespaces
+    Strophe.addNamespace('CARBONS', 'urn:xmpp:carbons:2');
+    Strophe.addNamespace('CHATSTATES', 'http://jabber.org/protocol/chatstates');
+    Strophe.addNamespace('CSI', 'urn:xmpp:csi:0');
+    Strophe.addNamespace('DELAY', 'urn:xmpp:delay');
+    Strophe.addNamespace('HINTS', 'urn:xmpp:hints');
+    Strophe.addNamespace('MAM', 'urn:xmpp:mam:2');
+    Strophe.addNamespace('NICK', 'http://jabber.org/protocol/nick');
+    Strophe.addNamespace('PUBSUB', 'http://jabber.org/protocol/pubsub');
+    Strophe.addNamespace('ROSTERX', 'http://jabber.org/protocol/rosterx');
+    Strophe.addNamespace('RSM', 'http://jabber.org/protocol/rsm');
+    Strophe.addNamespace('XFORM', 'jabber:x:data');
+
     // Use Mustache style syntax for variable interpolation
     /* Configuration of Lodash templates (this config is distinct to the
      * config of requirejs-tpl in main.js). This one is for normal inline templates.
@@ -214,34 +227,16 @@
         Strophe.log = function (level, msg) { _converse.log(level+' '+msg, level); };
         Strophe.error = function (msg) { _converse.log(msg, Strophe.LogLevel.ERROR); };
 
-        // Add Strophe Namespaces
-        Strophe.addNamespace('CARBONS', 'urn:xmpp:carbons:2');
-        Strophe.addNamespace('CHATSTATES', 'http://jabber.org/protocol/chatstates');
-        Strophe.addNamespace('CSI', 'urn:xmpp:csi:0');
-        Strophe.addNamespace('DELAY', 'urn:xmpp:delay');
-        Strophe.addNamespace('HINTS', 'urn:xmpp:hints');
-        Strophe.addNamespace('MAM', 'urn:xmpp:mam:2');
-        Strophe.addNamespace('NICK', 'http://jabber.org/protocol/nick');
-        Strophe.addNamespace('PUBSUB', 'http://jabber.org/protocol/pubsub');
-        Strophe.addNamespace('ROSTERX', 'http://jabber.org/protocol/rosterx');
-        Strophe.addNamespace('RSM', 'http://jabber.org/protocol/rsm');
-        Strophe.addNamespace('XFORM', 'jabber:x:data');
-
         // Instance level constants
         this.TIMEOUTS = { // Set as module attr so that we can override in tests.
             'PAUSED':     10000,
             'INACTIVE':   90000
         };
 
-        // Internationalization
-        this.locale = utils.getLocale(settings.i18n, utils.isConverseLocale);
-        if (!moment.locale) {
-            //moment.lang is deprecated after 2.8.1, use moment.locale instead
-            moment.locale = moment.lang;
-        }
+        /* Internationalization */
         moment.locale(utils.getLocale(settings.i18n, utils.isMomentLocale));
-        const __ = _converse.__ = utils.__.bind(_converse);
-        _converse.___ = utils.___;
+        _converse.locale = utils.getLocale(settings.i18n, utils.isLocaleSupported);
+        const __ = _converse.__ = _.partial(utils.__, _converse);
 
         // XEP-0085 Chat states
         // http://xmpp.org/extensions/xep-0085.html
@@ -277,6 +272,7 @@
             include_offline_state: false,
             jid: undefined,
             keepalive: true,
+            locales_url: '/locale/{{{locale}}}/LC_MESSAGES/converse.json',
             message_carbons: true,
             message_storage: 'session',
             password: undefined,
@@ -1870,18 +1866,34 @@
         if (settings.connection) {
             this.connection = settings.connection;
         }
-        _converse.initPlugins();
-        _converse.initConnection();
-        _converse.setUpXMLLogging();
-        _converse.logIn();
-        _converse.registerGlobalEventHandlers();
 
+        // TODO: fallback when global history has already been started
         Backbone.history.start();
 
+        function finishInitialization () {
+            _converse.initPlugins();
+            _converse.initConnection();
+            _converse.setUpXMLLogging();
+            _converse.logIn();
+            _converse.registerGlobalEventHandlers();
+        }
+
         if (!_.isUndefined(_converse.connection) &&
             _converse.connection.service === 'jasmine tests') {
+
+            finishInitialization();
             return _converse;
         } else {
+            utils.fetchLocale(
+                _converse.locale,
+                _converse.locales_url
+            ).then((jed) => {
+                _converse.jed = jed;
+                finishInitialization();
+            }).catch((reason) => {
+                finishInitialization();
+                _converse.log(reason, Strophe.LogLevel.FATAL);
+            });
             return init_promise.promise;
         }
     };

src/converse-muc.js

@@ -246,8 +246,8 @@
              * loaded by converse.js's plugin machinery.
              */
             const { _converse } = this,
-                { __,
-                ___ } = _converse;
+                  { __ } = _converse,
+                  { ___ } = utils;
             // XXX: Inside plugins, all calls to the translation machinery
             // (e.g. utils.__) should only be done in the initialize function.
             // If called before, we won't know what language the user wants,
@@ -316,8 +316,8 @@
                 },
 
                 new_nickname_messages: {
-                    210: ___('Your nickname has been automatically set to: %1$s'),
-                    303: ___('Your nickname has been changed to: %1$s')
+                    210: ___('Your nickname has been automatically set to %1$s'),
+                    303: ___('Your nickname has been changed to %1$s')
                 }
             };
 

src/converse-notification.js

@@ -19,10 +19,7 @@
              * loaded by converse.js's plugin machinery.
              */
             const { _converse } = this;
-
-            // For translations
             const { __ } = _converse;
-            const { ___ } = _converse;
 
             _converse.supports_html5_notification = "Notification" in window;
 
@@ -131,15 +128,15 @@
                       from_jid = Strophe.getBareJidFromJid(full_from_jid);
                 if (message.getAttribute('type') === 'headline') {
                     if (!_.includes(from_jid, '@') || _converse.allow_non_roster_messaging) {
-                        title = __(___("Notification from %1$s"), from_jid);
+                        title = __("Notification from %1$s", from_jid);
                     } else {
                         return;
                     }
                 } else if (!_.includes(from_jid, '@')) {
                     // XXX: workaround for Prosody which doesn't give type "headline"
-                    title = __(___("Notification from %1$s"), from_jid);
+                    title = __("Notification from %1$s", from_jid);
                 } else if (message.getAttribute('type') === 'groupchat') {
-                    title = __(___("%1$s says"), Strophe.getResourceFromJid(full_from_jid));
+                    title = __("%1$s says", Strophe.getResourceFromJid(full_from_jid));
                 } else {
                     if (_.isUndefined(_converse.roster)) {
                         _converse.log(
@@ -149,10 +146,10 @@
                     }
                     roster_item = _converse.roster.get(from_jid);
                     if (!_.isUndefined(roster_item)) {
-                        title = __(___("%1$s says"), roster_item.get('fullname'));
+                        title = __("%1$s says", roster_item.get('fullname'));
                     } else {
                         if (_converse.allow_non_roster_messaging) {
-                            title = __(___("%1$s says"), from_jid);
+                            title = __("%1$s says", from_jid);
                         } else {
                             return;
                         }

src/converse-roomslist.js

@@ -40,7 +40,7 @@
              * loaded by converse.js's plugin machinery.
              */
             const { _converse } = this,
-                  { __, ___ } = _converse;
+                  { __ } = _converse;
 
             _converse.RoomsList = Backbone.Model.extend({
                 defaults: {
@@ -114,7 +114,7 @@
                     ev.preventDefault();
                     const name = ev.target.getAttribute('data-room-name');
                     const jid = ev.target.getAttribute('data-room-jid');
-                    if (confirm(__(___("Are you sure you want to leave the room \"%1$s\"?"), name))) {
+                    if (confirm(__("Are you sure you want to leave the room \"%1$s\"?", name))) {
                         _converse.chatboxviews.get(jid).leave();
                     }
                 },

src/converse-rosterview.js

@@ -68,8 +68,7 @@
              * loaded by converse.js's plugin machinery.
              */
             const { _converse } = this,
-                { __,
-                ___ } = _converse;
+                  { __ } = _converse;
 
             _converse.api.settings.update({
                 allow_chat_pending_contacts: true,
@@ -572,7 +571,7 @@
                         this.el.classList.add('pending-xmpp-contact');
                         this.$el.html(tpl_pending_contact(
                             _.extend(item.toJSON(), {
-                                'desc_remove': __(___('Click to remove %1$s as a contact'), item.get('fullname')),
+                                'desc_remove': __('Click to remove %1$s as a contact', item.get('fullname')),
                                 'allow_chat_pending_contacts': _converse.allow_chat_pending_contacts
                             })
                         ));
@@ -580,8 +579,8 @@
                         this.el.classList.add('requesting-xmpp-contact');
                         this.$el.html(tpl_requesting_contact(
                             _.extend(item.toJSON(), {
-                                'desc_accept': __(___("Click to accept the contact request from %1$s"), item.get('fullname')),
-                                'desc_decline': __(___("Click to decline the contact request from %1$s"), item.get('fullname')),
+                                'desc_accept': __("Click to accept the contact request from %1$s", item.get('fullname')),
+                                'desc_decline': __("Click to decline the contact request from %1$s", item.get('fullname')),
                                 'allow_chat_pending_contacts': _converse.allow_chat_pending_contacts
                             })
                         ));
@@ -600,7 +599,7 @@
                         _.extend(item.toJSON(), {
                             'desc_status': STATUSES[chat_status||'offline'],
                             'desc_chat': __('Click to chat with this contact'),
-                            'desc_remove': __(___('Click to remove %1$s as a contact'), item.get('fullname')),
+                            'desc_remove': __('Click to remove %1$s as a contact', item.get('fullname')),
                             'title_fullname': __('Name'),
                             'allow_contact_removal': _converse.allow_contact_removal,
                             'num_unread': item.get('num_unread') || 0

src/utils.js

@@ -111,19 +111,44 @@
 
     // Translation machinery
     // ---------------------
-    u.__ = function (str) {
+    u.fetchLocale = (locale, locales_url) =>
+        new Promise((resolve, reject) => {
+            if (!u.isLocaleSupported(locale) || locale === 'en') {
+                resolve();
+            }
+            const xhr = new XMLHttpRequest();
+            xhr.open(
+                'GET',
+                _.template(locales_url)({'locale': locale}),
+                true
+            );
+            xhr.setRequestHeader(
+                'Accept',
+                "application/json, text/javascript"
+            );
+            xhr.onload = function () {
+                if (xhr.status >= 200 && xhr.status < 400) {
+                    resolve(new Jed(window.JSON.parse(xhr.responseText)));
+                } else {
+                    xhr.onerror();
+                }
+            };
+            xhr.onerror = function () {
+                reject(xhr.statusText);
+            };
+            xhr.send();
+        });
+
+    u.__ = function (_converse, str) {
         if (_.isUndefined(window.Jed)) {
             return str;
         }
-        if (!u.isConverseLocale(this.locale) || this.locale === 'en') {
-            return Jed.sprintf.apply(window.Jed, arguments);
-        }
-        if (typeof this.jed === "undefined") {
-            this.jed = new Jed(window.JSON.parse(locales[this.locale]));
+        if (_.isUndefined(_converse.jed)) {
+            return Jed.sprintf.apply(window.Jed, [].slice.call(arguments, 1));
         }
-        var t = this.jed.translate(str);
+        var t = _converse.jed.translate(str);
         if (arguments.length>1) {
-            return t.fetch.apply(t, [].slice.call(arguments,1));
+            return t.fetch.apply(t, [].slice.call(arguments, 2));
         } else {
             return t.fetch();
         }
@@ -485,7 +510,8 @@
         return locale || 'en';
     };
 
-    u.isConverseLocale = function (locale) {
+    u.isLocaleSupported = function (locale) {
+        /* Check whether the passed in locale is supported by Converse */
         if (!_.isString(locale)) { return false; }
         return _.includes(_.keys(locales || {}), locale);
     };
@@ -500,12 +526,6 @@
             if (preferred_locale === 'en' || isSupportedByLibrary(preferred_locale)) {
                 return preferred_locale;
             }
-            try {
-                var obj = window.JSON.parse(preferred_locale);
-                return obj.locale_data.converse[""].lang;
-            } catch (e) {
-                logger.error(e);
-            }
         }
         return u.detectLocale(isSupportedByLibrary) || 'en';
     };