Use webpack's dynamic imports feature for fetching emoji JSON

.eslintrc.json

@@ -1,7 +1,9 @@
 {
+    "parser": "babel-eslint",
     "parserOptions": {
         "ecmaVersion": 2017,
-        "sourceType": "module"
+        "sourceType": "module",
+        "allowImportExportEverywhere": true
     },
     "env": {
         "browser": true,

CHANGES.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 5.0.2 (Unreleased)
+
+- The JSON representing emojis is now fetched asynchronously as a separate file `converse.emojis.js`.
+- Webpack is now configured with a `publicPath` set to `/dist/`. This is necessary
+  so that chunks (such as the Emojis JSON) can be fetched asynchronously. This
+  means that all your assets need to be served at `/dist`. If you need to set a
+  different path, you'll need to set `publicPath` in `webpack.config.js` to
+  your preferred path and then rebuild all assets (e.g. `make dist`).
+
 ## 5.0.1 (2019-08-14)
 
 - Add a new GUI for moderator actions. You can trigger it by entering `/modtools` in a MUC.

docs/source/configuration.rst

@@ -647,10 +647,6 @@ domain_placeholder
 The placeholder text shown in the domain input on the registration form.
 
 
-emoji_json_path
----------------
-
-* Default: ``emojis/``
 
 
 emoji_image_path

package-lock.json

@@ -2681,6 +2681,32 @@
       "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==",
       "dev": true
     },
+    "babel-eslint": {
+      "version": "10.0.2",
+      "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.2.tgz",
+      "integrity": "sha512-UdsurWPtgiPgpJ06ryUnuaSXC2s0WoSZnQmEpbAH65XZSdwowgN5MvyP7e88nW07FYXv72erVtpBkxyDVKhH1Q==",
+      "dev": true,
+      "requires": {
+        "@babel/code-frame": "^7.0.0",
+        "@babel/parser": "^7.0.0",
+        "@babel/traverse": "^7.0.0",
+        "@babel/types": "^7.0.0",
+        "eslint-scope": "3.7.1",
+        "eslint-visitor-keys": "^1.0.0"
+      },
+      "dependencies": {
+        "eslint-scope": {
+          "version": "3.7.1",
+          "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.1.tgz",
+          "integrity": "sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=",
+          "dev": true,
+          "requires": {
+            "esrecurse": "^4.1.0",
+            "estraverse": "^4.1.1"
+          }
+        }
+      }
+    },
     "babel-loader": {
       "version": "8.0.6",
       "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.0.6.tgz",
@@ -2710,6 +2736,12 @@
         "object.assign": "^4.1.0"
       }
     },
+    "babel-plugin-syntax-dynamic-import": {
+      "version": "6.18.0",
+      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-dynamic-import/-/babel-plugin-syntax-dynamic-import-6.18.0.tgz",
+      "integrity": "sha1-jWomIpyDdFqZgqRBBRVyyqF5sdo=",
+      "dev": true
+    },
     "backbone": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/backbone/-/backbone-1.4.0.tgz",

package.json

@@ -55,7 +55,9 @@
     "@babel/preset-env": "^7.5.4",
     "@converse/headless": "file:src/headless",
     "@fortawesome/fontawesome-free": "5.9.0",
+    "babel-eslint": "^10.0.2",
     "babel-loader": "^8.0.6",
+    "babel-plugin-syntax-dynamic-import": "^6.18.0",
     "backbone.nativeview": "conversejs/Backbone.NativeView#5997c8197ca594e6b8469447f28310c78bd1d95e",
     "backbone.overview": "1.0.3",
     "backbone.vdomview": "^1.0.1",

src/headless/converse-emoji.js

@@ -170,7 +170,6 @@ converse.plugins.add('converse-emoji', {
 
         _converse.api.settings.update({
             'emoji_image_path': twemoji.default.base,
-            'emoji_json_path': '/dist/emojis.json',
             'emoji_categories': {
                 "smileys": ":grinning:",
                 "people": ":thumbsup:",
@@ -225,36 +224,6 @@ converse.plugins.add('converse-emoji', {
             return _converse.toned_emojis;
         }
 
-        function fetchEmojiJSON () {
-            _converse.emojis.json = {};
-            const promise = u.getResolveablePromise();
-            const xhr = new XMLHttpRequest();
-            xhr.open('GET', _converse.emoji_json_path, true);
-            xhr.setRequestHeader('Accept', "application/json, text/javascript");
-            xhr.onload = function () {
-                if (xhr.status >= 200 && xhr.status < 400) {
-                    try {
-                        _converse.emojis.json = JSON.parse(xhr.responseText);
-                    } catch (e) {
-                        xhr.onerror(e);
-                    }
-                } else {
-                    xhr.onerror();
-                }
-                promise.resolve();
-            };
-            xhr.onerror = (e) => {
-                const err_message = e ? ` Error: ${e.message}` : '';
-                _converse.log(
-                    `Could not fetch Emoji JSON. Status: ${xhr.statusText}. ${err_message}`,
-                    Strophe.LogLevel.ERROR
-                );
-                promise.resolve();
-            }
-            xhr.send();
-            return promise;
-        }
-
 
         /************************ BEGIN Utils ************************/
         // Closured cache
@@ -350,26 +319,25 @@ converse.plugins.add('converse-emoji', {
                 if (attr === 'category') {
                     return _converse.emojis.json;
                 }
-                emojis_by_attribute[attr] = {};
                 const all_variants = _converse.emojis_list
                     .map(e => e[attr])
                     .filter((c, i, arr) => arr.indexOf(c) == i);
 
-                all_variants.forEach(v => {
-                    emojis_by_attribute[attr][v] = _.find(_converse.emojis_list, i => (i[attr] === v));
-                });
+                emojis_by_attribute[attr] = {};
+                all_variants.forEach(v => (emojis_by_attribute[attr][v] = _.find(_converse.emojis_list, i => (i[attr] === v))));
                 return emojis_by_attribute[attr];
             }
         });
         /************************ END Utils ************************/
 
-        await fetchEmojiJSON();
+        const { default: json } = await import(/*webpackChunkName: "emojis" */ './emojis.json');
+        _converse.emojis.json = json;
         _converse.emojis_map = Object.keys(_converse.emojis.json).reduce((result, cat) => Object.assign(result, _converse.emojis.json[cat]), {});
-        _converse.emojis_list = Object.keys(_converse.emojis.json).reduce((result, cat) => [...result, ...Object.values(_converse.emojis.json[cat])], []);
+        _converse.emojis_list = Object.values(_converse.emojis_map);
         _converse.emoji_shortnames = _converse.emojis_list.map(m => m.sn);
         _converse.emoji_shortnames.sort();
 
-        const getShortNames = () => _converse.emojis_list.map(emoji => emoji.sn.replace(/[+]/g, "\\$&")).join('|');
+        const getShortNames = () => _converse.emoji_shortnames.map(s => s.replace(/[+]/g, "\\$&")).join('|');
         _converse.emojis.shortnames_regex = new RegExp("<object[^>]*>.*?<\/object>|<span[^>]*>.*?<\/span>|<(?:object|embed|svg|img|div|span|p|a)[^>]*>|("+getShortNames()+")", "gi");
 
         const excluded_categories = ['modifier', 'regional'];

webpack.config.js

@@ -11,8 +11,10 @@ const config = {
         "window": "window"
     }],
     output: {
-        path: path.resolve(__dirname, 'dist'),
-        filename: 'converse.js'
+        path: path.resolve(__dirname, 'dist'), // Output path for generated bundles
+        publicPath: '/dist/', // URL base path for all assets
+        filename: 'converse.js',
+        chunkFilename: 'converse.[name].js'
     },
     devtool: 'source-map',
     plugins: [new MiniCssExtractPlugin({filename: '../dist/converse.css'})],
@@ -139,37 +141,26 @@ function parameterize () {
 
     if (type === 'headless') {
         console.log("Making a headless build");
-        extend(config, {
-            entry: "@converse/headless/headless.js",
-            output: {
-                path: path.resolve(__dirname, 'dist'),
-                filename: 'converse-headless.js'
-            },
-        });
+        config.entry = "@converse/headless/headless.js";
+        config.output.filename = 'converse-headless.js';
     }
 
     if (type === 'nodeps') {
         console.log("Making a build without 3rd party dependencies");
-        extend(config, {
-            entry: path.resolve(__dirname, 'src/converse.js'),
-            externals: [{
-                "backbone": "backbone",
-                "backbone.nativeview": "backbone.nativeview",
-                "backbone.vdomview": "backbone.vdomview",
-                "backbone.browserStorage": "backbone.browserStorage",
-                "backbone.overview": "backbone.overview",
-                "es6-promise": "es6-promise",
-                "lodash": "lodash",
-                "lodash.converter": "lodash.converter",
-                "lodash.noconflict": "lodash.noconflict",
-                "strophe": "strophe",
-                "window": "window"
-            }],
-            output: {
-                path: path.resolve(__dirname, 'dist'),
-                filename: 'converse-no-dependencies.js'
-            },
-        });
+        config.output.filename = 'converse-no-dependencies.js';
+        config.externals = [{
+            "backbone": "backbone",
+            "backbone.nativeview": "backbone.nativeview",
+            "backbone.vdomview": "backbone.vdomview",
+            "backbone.browserStorage": "backbone.browserStorage",
+            "backbone.overview": "backbone.overview",
+            "es6-promise": "es6-promise",
+            "lodash": "lodash",
+            "lodash.converter": "lodash.converter",
+            "lodash.noconflict": "lodash.noconflict",
+            "strophe": "strophe",
+            "window": "window"
+        }];
     }
 
     if (type === 'css') {