Add a po loader for webpack.

- Removes the need for generating and keeping json files

CHANGES.md

@@ -2,10 +2,13 @@
 
 ## 5.0.2 (Unreleased)
 
+- `po` translations files are now loaded via Webpack. As a result the `locales_url`
+  config option is now removed given that the path to the locale JSON files is now
+  determined by the webpack config and can't be changed at runtime.
 - The JSON representing emojis is now fetched asynchronously as a separate file `converse.emojis.js`.
 - Webpack is now configured with a `publicPath` set to `/dist/`. This is necessary
-  so that chunks (such as the Emojis JSON) can be fetched asynchronously. This
-  means that all your assets need to be served at `/dist`. If you need to set a
+  so that chunks (such as the emojis and locales JSON files) can be fetched asynchronously.
+  This means that all your assets need to be served at `/dist`. If you need to set a
   different path, you'll need to set `publicPath` in `webpack.config.js` to
   your preferred path and then rebuild all assets (e.g. `make dist`).
 

Makefile

@@ -14,7 +14,6 @@ JSDOC			?=  ./node_modules/.bin/jsdoc
 LERNA			?= ./node_modules/.bin/lerna
 OXIPNG			?= oxipng
 PAPER		   	=
-PO2JSON		 	?= ./node_modules/.bin/po2json
 RJS				?= ./node_modules/.bin/r.js
 NPX				?= ./node_modules/.bin/npx
 SASS			?= ./node_modules/.bin/node-sass
@@ -43,7 +42,6 @@ help:
 	@echo " dev             Set up the development environment. To force a fresh start, run 'make clean' first."
 	@echo " html            Make standalone HTML files of the documentation."
 	@echo " po              Generate gettext PO files for each i18n language."
-	@echo " po2json         Generate JSON files from the language PO files."
 	@echo " pot             Generate a gettext POT file to be used for translations."
 	@echo " release         Prepare a new release of converse.js. E.g. make release VERSION=0.9.5"
 	@echo " serve           Serve this directory via a webserver on port 8000."
@@ -77,10 +75,6 @@ pot: dist/converse-no-dependencies-es2015.js
 po:
 	find ./locale -maxdepth 1 -mindepth 1 -type d -exec msgmerge {}/LC_MESSAGES/converse.po ./locale/converse.pot -U \;
 
-.PHONY: po2json
-po2json:
-	find ./locale -maxdepth 1 -mindepth 1 -type d -exec $(PO2JSON) -f jed1.x -d converse {}/LC_MESSAGES/converse.po {}/LC_MESSAGES/converse.json \;
-
 ########################################################################
 ## Release management
 
@@ -99,7 +93,6 @@ release:
 	$(SED) -ri 's,cdn.conversejs.org/$(VERSION_FORMAT),cdn.conversejs.org/$(VERSION),' demo/*.html
 	make pot
 	make po
-	make po2json
 	make build
 	npm pack
 

docs/source/configuration.rst

@@ -747,14 +747,13 @@ i18n
 
 Specify the locale/language.
 
-The translations for that locale must be available in JSON format at the
-`locales_url`_
+The translations for that locale must be served in JSON format at `/dist/locales/${i18n}-LC_MESSAGES-converse-po.js`.
+The default webpack configuration for Converse.js ensures that these files are
+generated and placed in the right location.
 
 If an explicit locale is specified via the ``i18n`` setting and the
-translations for that locale are not found at the `locales_url``, then
-then Converse will fall back to trying to determine the browser's language
-and fetching those translations, or if that fails the default English texts
-will be used.
+translations for that locale are not found, then Converse will fall back to trying to determine the browser's language
+and fetching those translations, or if that fails the default English strings will be used.
 
 idle_presence_timeout
 ---------------------
@@ -812,34 +811,6 @@ Any other locales will be ignored.
 When self-hosting, also make sure that the locales are served and therefore
 fetchable (via ``XMLHttpRequest``) at the URL specified by :ref:`locales-url`.
 
-.. _`locales-url`:
-
-locales_url
------------
-
-* Default: ``/locale/{{{locale}}}/LC_MESSAGES/converse.json``,
-
-The URL from where Converse should fetch translation JSON.
-
-The three curly braces ``{{{ }}}`` are
-`Mustache <https://github.com/janl/mustache.js#readme>`_-style
-variable interpolation which HTML-escapes the value being inserted. It's
-important that the inserted value is HTML-escaped, otherwise a malicious script
-injection attack could be attempted.
-
-The variable being interpolated via the curly braces is ``locale``, which is
-the value passed in to the `i18n`_ setting, or the browser's locale or the
-default local or ``en`` (resolved in that order).
-
-From version 3.3.0, Converse no longer bundles all translations into its
-final build file. Instead, only the relevant translations are fetched at
-runtime.
-
-This change also means that it's no longer possible to pass in the translation
-JSON data directly into ``_converse.initialize`` via the `i18n`_ setting.
-Instead, you only specify the language code (e.g. `de`) and that language's
-JSON translations will automatically be fetched via XMLHTTPRequest at
-``locales_url``.
 
 locked_domain
 -------------

index.html

@@ -18,7 +18,7 @@
     <!-- *********************************************************************** -->
 
     <![if gte IE 11]>
-        <link type="text/css" rel="stylesheet" media="screen" href="https://cdn.conversejs.org/5.0.1/dist/converse.min.css" />
+        <link type="text/css" rel="stylesheet" media="screen" href="/dist/converse.min.css" />
         <script src="https://cdn.conversejs.org/3rdparty/libsignal-protocol.min.js"></script>
         <script src="https://cdn.conversejs.org/5.0.1/dist/converse.min.js"></script>
     <![endif]>

locale/af/LC_MESSAGES/converse.po

@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: Converse.js 0.4\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2019-08-14 12:46+0200\n"
-"PO-Revision-Date: 2019-08-14 12:42+0200\n"
+"PO-Revision-Date: 2019-08-23 10:16+0200\n"
 "Last-Translator: JC Brand <jc@opkode.com>\n"
 "Language-Team: Afrikaans <https://hosted.weblate.org/projects/conversejs/"
 "translations/af/>\n"

locale/oc/LC_MESSAGES/converse.po

@@ -8,7 +8,7 @@ msgstr ""
 "Project-Id-Version: Converse.js 4.2.0\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2019-08-14 12:46+0200\n"
-"PO-Revision-Date: 2019-08-06 14:23+0000\n"
+"PO-Revision-Date: 2019-08-23 10:17+0200\n"
 "Last-Translator: Quentí <quentinantonin@free.fr>\n"
 "Language-Team: Occitan <https://hosted.weblate.org/projects/conversejs/"
 "translations/oc/>\n"

package-lock.json

@@ -6187,12 +6187,42 @@
       }
     },
     "gettext-parser": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/gettext-parser/-/gettext-parser-1.1.0.tgz",
-      "integrity": "sha1-LFpmONiTk0ubVQN9CtgstwBLJnk=",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/gettext-parser/-/gettext-parser-2.0.0.tgz",
+      "integrity": "sha512-FDs/7XjNw58ToQwJFO7avZZbPecSYgw8PBYhd0An+4JtZSrSzKhEvTsVV2uqdO7VziWTOGSgLGD5YRPdsCjF7Q==",
+      "dev": true,
+      "requires": {
+        "encoding": "^0.1.12",
+        "safe-buffer": "^5.1.2"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
+          "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==",
+          "dev": true
+        }
+      }
+    },
+    "gettext-to-messageformat": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/gettext-to-messageformat/-/gettext-to-messageformat-0.3.0.tgz",
+      "integrity": "sha512-HlEGFECqAavbOYJTo1I2qh8IqWetAenixaH/AbdIuNTY0easvzrPn+yYUHy63GEjx9pXoLx2nCJcDcE27LrB2g==",
       "dev": true,
       "requires": {
-        "encoding": "^0.1.11"
+        "gettext-parser": "^1.3.0"
+      },
+      "dependencies": {
+        "gettext-parser": {
+          "version": "1.4.0",
+          "resolved": "https://registry.npmjs.org/gettext-parser/-/gettext-parser-1.4.0.tgz",
+          "integrity": "sha512-sedZYLHlHeBop/gZ1jdg59hlUEcpcZJofLq2JFwJT1zTqAU3l2wFv6IsuwFHGqbiT9DWzMUW4/em2+hspnmMMA==",
+          "dev": true,
+          "requires": {
+            "encoding": "^0.1.12",
+            "safe-buffer": "^5.1.1"
+          }
+        }
       }
     },
     "git-raw-commits": {
@@ -6482,12 +6512,6 @@
         "ansi-regex": "^2.0.0"
       }
     },
-    "has-color": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/has-color/-/has-color-0.1.7.tgz",
-      "integrity": "sha1-ZxRKUmDDT8PMpnfQQdr1L+e3iy8=",
-      "dev": true
-    },
     "has-flag": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
@@ -8590,47 +8614,6 @@
         }
       }
     },
-    "nomnom": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.8.1.tgz",
-      "integrity": "sha1-IVH3Ikcrp55Qp2/BJbuMjy5Nwqc=",
-      "dev": true,
-      "requires": {
-        "chalk": "~0.4.0",
-        "underscore": "~1.6.0"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz",
-          "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg=",
-          "dev": true
-        },
-        "chalk": {
-          "version": "0.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.4.0.tgz",
-          "integrity": "sha1-UZmj3c0MHv4jvAjBsCewYXbgxk8=",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "~1.0.0",
-            "has-color": "~0.1.0",
-            "strip-ansi": "~0.1.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.1.1.tgz",
-          "integrity": "sha1-OeipjQRNFQZgq+SmgIrPcLt7yZE=",
-          "dev": true
-        },
-        "underscore": {
-          "version": "1.6.0",
-          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz",
-          "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=",
-          "dev": true
-        }
-      }
-    },
     "nopt": {
       "version": "3.0.6",
       "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
@@ -12437,14 +12420,60 @@
         "lodash": "^4.17.11"
       }
     },
+    "po-loader": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/po-loader/-/po-loader-0.5.0.tgz",
+      "integrity": "sha512-2VyoECqHL6pLUxQqszi9WskwOgRPtO3y+fhVmYMb46L+5S03whRf/lHqWhbTE3AbwEc6DdPuGKFD4IUbWuM/YQ==",
+      "dev": true,
+      "requires": {
+        "loader-utils": "^1.2.3"
+      },
+      "dependencies": {
+        "big.js": {
+          "version": "5.2.2",
+          "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
+          "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==",
+          "dev": true
+        },
+        "json5": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
+          "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
+          "dev": true,
+          "requires": {
+            "minimist": "^1.2.0"
+          }
+        },
+        "loader-utils": {
+          "version": "1.2.3",
+          "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz",
+          "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==",
+          "dev": true,
+          "requires": {
+            "big.js": "^5.2.2",
+            "emojis-list": "^2.0.0",
+            "json5": "^1.0.1"
+          }
+        }
+      }
+    },
     "po2json": {
-      "version": "0.4.5",
-      "resolved": "https://registry.npmjs.org/po2json/-/po2json-0.4.5.tgz",
-      "integrity": "sha1-R7spUtoy1Yob4vJWpZjuvAt0URg=",
+      "version": "1.0.0-alpha",
+      "resolved": "https://registry.npmjs.org/po2json/-/po2json-1.0.0-alpha.tgz",
+      "integrity": "sha512-DsP/L4JsMB/gTEpXm9B1I+S1W0z0wNXu4Ky47MUlfS3ruwUZT1nUm+FvWt+ZMehFTh1EcXAhx6oljw8Ly4qTaA==",
       "dev": true,
       "requires": {
-        "gettext-parser": "1.1.0",
-        "nomnom": "1.8.1"
+        "commander": "^2.18.0",
+        "gettext-parser": "2.0.0",
+        "gettext-to-messageformat": "^0.3.0"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "2.20.0",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
+          "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
+          "dev": true
+        }
       }
     },
     "portfinder": {

package.json

@@ -85,7 +85,8 @@
     "minimist": "^1.2.0",
     "node-sass": "^4.12.0",
     "npm": "^6.4.1",
-    "po2json": "^0.4.4",
+    "po-loader": "^0.5.0",
+    "po2json": "^1.0.0-alpha",
     "prettierx": "^0.6.0",
     "requirejs": "2.3.6",
     "run-headless-chromium": "^0.1.1",

src/headless/converse-core.js

@@ -237,7 +237,6 @@ _converse.default_settings = {
     idle_presence_timeout: 300, // Seconds after which an idle presence is sent
     jid: undefined,
     keepalive: true,
-    locales_url: '/locale/{{{locale}}}/LC_MESSAGES/converse.json',
     locales: [
         'af', 'ar', 'bg', 'ca', 'cs', 'de', 'eo', 'es', 'eu', 'en', 'fr', 'gl',
         'he', 'hi', 'hu', 'id', 'it', 'ja', 'nb', 'nl', 'oc',
@@ -1336,9 +1335,8 @@ _converse.initialize = async function (settings, callback) {
         await finishInitialization();
         return _converse;
     } else if (i18n !== undefined) {
-        const url = u.interpolate(_converse.locales_url, {'locale': _converse.locale});
         try {
-            await i18n.fetchTranslations(_converse.locale, _converse.locales, url);
+            await i18n.fetchTranslations(_converse.locale, _converse.locales);
         } catch (e) {
             _converse.log(e.message, Strophe.LogLevel.FATAL);
         }

src/headless/i18n.js

@@ -129,37 +129,12 @@ export default {
      * @method i18n#fetchTranslations
      * @param { String } locale -The given i18n locale
      * @param { Array } supported_locales -  List of locales supported
-     * @param { String } locale_url - The URL from which the translations should be fetched
      */
-    fetchTranslations (locale, supported_locales, locale_url) {
-        return new Promise((resolve, reject) => {
-            if (!isConverseLocale(locale, supported_locales) || locale === 'en') {
-                return resolve();
-            }
-            const xhr = new XMLHttpRequest();
-            xhr.open('GET', locale_url, true);
-            xhr.setRequestHeader(
-                'Accept',
-                "application/json, text/javascript"
-            );
-            xhr.onload = function () {
-                if (xhr.status >= 200 && xhr.status < 400) {
-                    try {
-                        const data = window.JSON.parse(xhr.responseText);
-                        jed_instance = new Jed(data);
-                        resolve();
-                    } catch (e) {
-                        xhr.onerror(e);
-                    }
-                } else {
-                    xhr.onerror();
-                }
-            };
-            xhr.onerror = (e) => {
-                const err_message = e ? ` Error: ${e.message}` : '';
-                reject(new Error(`Could not fetch translations. Status: ${xhr.statusText}. ${err_message}`));
-            }
-            xhr.send();
-        });
+    async fetchTranslations (locale, supported_locales) {
+        if (!isConverseLocale(locale, supported_locales) || locale === 'en') {
+            return;
+        }
+        const { default: data } = await import(/*webpackChunkName: "locales/[request]" */ `../../locale/${locale}/LC_MESSAGES/converse.po`);
+        jed_instance = new Jed(data);
     }
 };

webpack.config.js

@@ -14,7 +14,7 @@ const config = {
         path: path.resolve(__dirname, 'dist'), // Output path for generated bundles
         publicPath: '/dist/', // URL base path for all assets
         filename: 'converse.js',
-        chunkFilename: 'converse.[name].js'
+        chunkFilename: '[name].js'
     },
     devtool: 'source-map',
     plugins: [new MiniCssExtractPlugin({filename: '../dist/converse.css'})],
@@ -46,6 +46,19 @@ const config = {
                     "prependFilenameComment": __dirname
                 }
             }]
+        }, {
+            test: /LC_MESSAGES\/converse.po$/,
+            type: "json",
+            use: [
+            {
+                loader: 'po-loader',
+                options: {
+                    'format': 'jed',
+                    'domain': 'converse'
+                }
+            }
+            ]
+        }, {
         }, {
             test: /webfonts\/.*\.(woff(2)?|ttf|eot|truetype|svg)(\?v=\d+\.\d+\.\d+)?$/,
             use: [