Refactor MAM to fetch a limited number of newest messages first. Fixes #1810

- Clarify types and distinguish between MAM query options and RSM query options.

- Remove the `fetchMessagesOnScrollUp` event handler. Instead messages will be fetched as a placeholder comes into view.

- Remove support for mam2:#extended querying
    It's too much of a headache with patchy server support.
    Instead, use RSM's `<before>some-id</before>` together with `start` to
    bracket queries for placeholders.

- Create a placeholder after restoring cached messages.
    This placeholder will automatically fetch older messages when the user scrolls up.

- Create new config option `auto_fill_history_gaps`

- Don't duplicate already existing MAM placeholders.
    Pre-existing ones could have been created from the cache.

- If we have a most recent message, query from it

.gitignore

@@ -68,3 +68,6 @@ node_modules
 .sv?
 /vendor/
 .aider*
+.prompts/
+
+Session.vim

CHANGES.md

@@ -15,6 +15,7 @@
 - #1303: Display non-contacts who sent us a message somehow in fullscreen 
 - #1349: XEP-0392 Consistent Color Generation
 - #1700: Deleted pending contacts reappear after page reload
+- #1810: Create clickable link to load older MAM messages if there is no scrollbars.
 - #2118: Show reflected message in MUC 
 - #2383: Add modal to start chats with JIDs not in the roster
 - #2586: Add support for XEP-0402 Bookmarks

docs/source/configuration.rst

@@ -328,6 +328,14 @@ autocomplete_add_contact
 
 Determines whether search suggestions are shown in the "Add Contact" modal.
 
+auto_fill_history_gaps
+----------------------
+
+* Default:  ``true``
+
+Determins whether Converse automatically fills gaps in the chat history.
+If set to false, a placeholder appears which can be clicked to fetch the
+missing messages.
 
 auto_focus
 ----------
@@ -1113,25 +1121,16 @@ If no nickame value is found, then an error will be raised.
 mam_request_all_pages
 ---------------------
 
-* Default: ``true``
+* Default: ``false``
 
-When requesting messages from the archive, Converse will ask only for messages
+When requesting messages from the archive, Converse will query for messages
 newer than the most recent cached message.
 
-When there are many archived messages since that one, the returned results will
+When there are many archived messages that matches the query, the returned results will
 be broken up in to pages, set by `archived_messages_page_size`_.
 
-By default Converse will request all the pages until all messages have been
-fetched, however for large archives this can slow things down dramatically.
-
-This setting turns the paging off, and Converse will only fetch the latest
-page.
-
-.. note::
-
-  If paging is turned off, there will appear gaps in the message history.
-  Converse currently doesn't yet have a way to inform the user of these gaps or
-  to let them be filled.
+Set this option to ``true`` to request all pages of archived messages, but be
+aware that this can have performance implications.
 
 
 muc_hats

local.html

@@ -25,6 +25,8 @@
         });
 
         converse.initialize({
+            auto_fill_history_gaps: false,
+            archived_messages_page_size: 2,
             muc_subscribe_to_rai: true,
             theme: 'dracula',
             auto_away: 300,

package-lock.json

@@ -9570,8 +9570,8 @@
     },
     "node_modules/strophe.js": {
       "version": "3.1.0",
-      "resolved": "git+ssh://git@github.com/strophe/strophejs.git#f54d83199ec62272ee8b4987b2e63f188e1b2e29",
-      "integrity": "sha512-ivy/25C19VudvLDMPhW4oZ4gIpicc0+AnnBzzV/YUikTbaS/ujy4Y/vO416alCFovqEmcc3AEXoQ4O8KqYEKug==",
+      "resolved": "git+ssh://git@github.com/strophe/strophejs.git#fb70dcb4e202f632bc9932915b4522f70ad4d47c",
+      "integrity": "sha512-8C7bVpBI4fZwdrFvjiz9oVLIfP0fmxOkt17Th9MMqLQTeAX61sEyFKdkH4kZ/qO+/QwtxW0+MeHqE2B5OZpjvA==",
       "license": "MIT",
       "optionalDependencies": {
         "@types/jsdom": "^21.1.7",
@@ -10817,7 +10817,7 @@
         "pluggable.js": "3.0.1",
         "sizzle": "^2.3.5",
         "sprintf-js": "^1.1.2",
-        "strophe.js": "strophe/strophejs#f54d83199ec62272ee8b4987b2e63f188e1b2e29",
+        "strophe.js": "strophe/strophejs#fb70dcb4e202f632bc9932915b4522f70ad4d47c",
         "urijs": "^1.19.10"
       },
       "devDependencies": {}

src/headless/package.json

@@ -42,7 +42,7 @@
     "pluggable.js": "3.0.1",
     "sizzle": "^2.3.5",
     "sprintf-js": "^1.1.2",
-    "strophe.js": "strophe/strophejs#f54d83199ec62272ee8b4987b2e63f188e1b2e29",
+    "strophe.js": "strophe/strophejs#fb70dcb4e202f632bc9932915b4522f70ad4d47c",
     "urijs": "^1.19.10"
   },
   "devDependencies": {}

src/headless/plugins/bookmarks/collection.js

@@ -136,9 +136,9 @@ class Bookmarks extends Collection {
                             ${model.get('password') ? stx`<password>${model.get('password')}</password>` : ''}
                         ${
                             extensions.length
-                                ? stx`<extensions>${extensions.map((e) => Stanza.unsafeXML(e))}</extensions>`
+                                ? stx`<extensions>${extensions.map((e) => Stanza.fromString(e))}</extensions>`
                                 : ''
-                        };
+                        }
                         </conference>
                     </item>`;
                 }

src/headless/plugins/disco/entity.js

@@ -53,8 +53,8 @@ class DiscoEntity extends Model {
      * Returns a Promise which resolves with a map indicating
      * whether a given identity is provided by this entity.
      * @method _converse.DiscoEntity#getIdentity
-     * @param { String } category - The identity category
-     * @param { String } type - The identity type
+     * @param {String} category - The identity category
+     * @param {String} type - The identity type
      */
     async getIdentity (category, type) {
         await this.waitUntilFeaturesDiscovered;
@@ -68,7 +68,7 @@ class DiscoEntity extends Model {
      * Returns a Promise which resolves with a map indicating
      * whether a given feature is supported.
      * @method _converse.DiscoEntity#getFeature
-     * @param { String } feature - The feature that might be supported.
+     * @param {String} feature - The feature that might be supported.
      */
     async getFeature (feature) {
         await this.waitUntilFeaturesDiscovered;

src/headless/plugins/mam/api.js

@@ -5,11 +5,11 @@ import dayjs from 'dayjs';
 import log from '../../log.js';
 import sizzle from "sizzle";
 import { RSM } from '../../shared/rsm';
-import { Strophe, $iq } from 'strophe.js';
+import { Strophe, Stanza } from 'strophe.js';
 import { TimeoutError } from '../../shared/errors.js';
 
 const { NS } = Strophe;
-const u = converse.env.utils;
+const { stx, u } = converse.env;
 
 
 export default {
@@ -33,7 +33,7 @@ export default {
           * RSM to enable easy querying between results pages.
           *
           * @method _converse.api.archive.query
-          * @param {import('./types').ArchiveQueryOptions} options - An object containing query parameters
+          * @param {import('./types').ArchiveQueryOptions} [options={}] - Optional query parameters
           * @throws {Error} An error is thrown if the XMPP server responds with an error.
           * @returns {Promise<import('./types').MAMQueryResult>}
           *
@@ -66,7 +66,7 @@ export default {
           * // For a particular user
           * let result;
           * try {
-          *    result = await api.archive.query({'with': 'john@doe.net'});
+          *    result = await api.archive.query({ mam: { with: 'john@doe.net' }});
           * } catch (e) {
           *     // The query was not successful
           * }
@@ -74,7 +74,7 @@ export default {
           * // For a particular room
           * let result;
           * try {
-          *    result = await api.archive.query({'with': 'discuss@conference.doglovers.net', 'groupchat': true});
+          *    result = await api.archive.query({ mam: { with: 'discuss@conference.doglovers.net' }}, is_groupchat: true });
           * } catch (e) {
           *     // The query was not successful
           * }
@@ -83,14 +83,16 @@ export default {
           * // Requesting all archived messages before or after a certain date
           * // ===============================================================
           * //
-          * // The `start` and `end` parameters are used to query for messages
+          * // The MAM `start` and `end` parameters are used to query for messages
           * // within a certain timeframe. The passed in date values may either be ISO8601
           * // formatted date strings, or JavaScript Date objects.
           *
           *  const options = {
-          *      'with': 'john@doe.net',
-          *      'start': '2010-06-07T00:00:00Z',
-          *      'end': '2010-07-07T13:23:54Z'
+          *      mam: {
+          *          'with': 'john@doe.net',
+          *          'start': '2010-06-07T00:00:00Z',
+          *          'end': '2010-07-07T13:23:54Z'
+          *      },
           *  };
           * let result;
           * try {
@@ -109,7 +111,7 @@ export default {
           * // Return maximum 10 archived messages
           * let result;
           * try {
-          *     result = await api.archive.query({'with': 'john@doe.net', 'max':10});
+          *     result = await api.archive.query({ mam: { with: 'john@doe.net', max:10 }});
           * } catch (e) {
           *     // The query was not successful
           * }
@@ -131,7 +133,7 @@ export default {
           * // archived messages. Please note, when calling these methods, pass in an integer
           * // to limit your results.
           *
-          * const options = {'with': 'john@doe.net', 'max':10};
+          * const options = { mam: { with: 'john@doe.net' }, rsm: { max:10 }};
           * let result;
           * try {
           *     result = await api.archive.query(options);
@@ -143,7 +145,13 @@ export default {
           *
           * while (!result.complete) {
           *     try {
-          *         result = await api.archive.query(Object.assign(options, rsm.next(10).query));
+          *         result = await api.archive.query({
+          *             mam: { ...options.mam },
+          *             rsm: {
+          *                 ...options.rsm,
+          *                 ...rsm.next(10).query
+          *                 }
+          *             });
           *     } catch (e) {
           *         // The query was not successful
           *     }
@@ -161,7 +169,7 @@ export default {
           * // message, pass in the `before` parameter with an empty string value `''`.
           *
           * let result;
-          * const options = {'before': '', 'max':5};
+          * const options = { rsm: { before: '', max:5 }};
           * try {
           *     result = await api.archive.query(options);
           * } catch (e) {
@@ -172,7 +180,14 @@ export default {
           *
           * // Now we query again, to get the previous batch.
           * try {
-          *      result = await api.archive.query(Object.assign(options, rsm.previous(5).query));
+          *     try {
+          *         result = await api.archive.query({
+          *             mam: { ...options.mam },
+          *             rsm: {
+          *                 ...options.rsm,
+          *                 ...rsm.previous(5).query
+          *                 }
+          *             });
           * } catch (e) {
           *     // The query was not successful
           * }
@@ -180,57 +195,68 @@ export default {
           * result.messages.forEach(m => this.showMessage(m));
           *
           */
-        async query (options) {
+        async query (options={}) {
             if (!api.connection.connected()) {
                 throw new Error('Can\'t call `api.archive.query` before having established an XMPP session');
             }
-            const attrs = {'type':'set'};
-            if (options && options.groupchat) {
-                if (!options['with']) {
+
+            let toJID;
+            if (options && options.is_groupchat) {
+                if (!options.mam?.with) {
                     throw new Error(
                         'You need to specify a "with" value containing '+
-                        'the chat room JID, when querying groupchat messages.');
+                        'the groupchat JID, when querying groupchat messages.');
                 }
-                attrs.to = options['with'];
+                toJID = options.mam.with;
             }
 
+            const withJID = !options.is_groupchat && options.mam?.with || null;
+
             const bare_jid = _converse.session.get('bare_jid');
-            const jid = attrs.to || bare_jid;
+            const jid = toJID || bare_jid;
             const supported = await api.disco.supports(NS.MAM, jid);
             if (!supported) {
                 log.warn(`Did not fetch MAM archive for ${jid} because it doesn't support ${NS.MAM}`);
-                return {'messages': []};
+                return { messages: [] };
             }
 
-            const queryid = u.getUniqueId();
-            const stanza = $iq(attrs).c('query', {'xmlns':NS.MAM, 'queryid':queryid});
-            if (options) {
-                stanza.c('x', {'xmlns':NS.XFORM, 'type': 'submit'})
-                        .c('field', {'var':'FORM_TYPE', 'type': 'hidden'})
-                        .c('value').t(NS.MAM).up().up();
-
-                if (options['with'] && !options.groupchat) {
-                    stanza.c('field', {'var':'with'}).c('value')
-                        .t(options['with']).up().up();
-                }
-                ['start', 'end'].forEach(t => {
-                    if (options[t]) {
-                        const date = dayjs(options[t]);
-                        if (date.isValid()) {
-                            stanza.c('field', {'var':t}).c('value').t(date.toISOString()).up().up();
-                        } else {
-                            throw new TypeError(`archive.query: invalid date provided for: ${t}`);
-                        }
+            // Validate start and end dates and add them to attrs (in the right format)
+            const { start: startDate, end: endDate } = ['start', 'end'].reduce((acc, t) => {
+                if (options.mam?.[t]) {
+                    const date = dayjs(options.mam[t]);
+                    if (date.isValid()) {
+                        acc[t] = date.toISOString();
+                    } else {
+                        throw new TypeError(`archive.query: invalid date provided for: ${t}`);
                     }
-                });
-                stanza.up();
-                const rsm = new RSM(options);
-                if (Object.keys(rsm.query).length) {
-                    stanza.cnode(rsm.toXML());
                 }
-            }
+                return acc;
+            }, { start: null, end: null });
 
             const connection = api.connection.get();
+            const rsm = options.rsm ? new RSM(options.rsm) : {};
+            const queryid = u.getUniqueId();
+
+            const stanza = stx`
+                <iq id="${u.getUniqueId()}"
+                        ${toJID ? Stanza.unsafeXML(`to="${Strophe.xmlescape(toJID)}"`) : ""}
+                        type="set"
+                        xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="${NS.MAM}">
+                        ${
+                            withJID || startDate || endDate
+                                ? stx`
+                            <x type="submit" xmlns="${NS.XFORM}">
+                                <field type="hidden" var="FORM_TYPE"><value>${NS.MAM}</value></field>
+                                ${withJID ? stx`<field var="with"><value>${withJID}</value></field>` : ""}
+                                ${startDate ? stx`<field var="start"><value>${startDate}</value></field>` : ""}
+                                ${endDate ? stx`<field var="end"><value>${endDate}</value></field>` : ""}
+                            </x>`
+                                : ""
+                        }
+                        ${Object.keys(rsm.query ?? {}).length ? stx`${Stanza.unsafeXML(rsm.toXML().outerHTML)}` : ""}
+                    </query>
+                </iq>`;
 
             const messages = [];
             const message_handler = connection.addHandler(/** @param {Element} stanza */(stanza) => {
@@ -239,8 +265,8 @@ export default {
                     return true;
                 }
                 const from = stanza.getAttribute('from') || bare_jid;
-                if (options.groupchat) {
-                    if (from !== options['with']) {
+                if (options.is_groupchat) {
+                    if (from !== options.mam?.with) {
                         log.warn(`Ignoring alleged groupchat MAM message from ${stanza.getAttribute('from')}`);
                         return true;
                     }
@@ -254,7 +280,7 @@ export default {
 
             let error;
             const timeout = api.settings.get('message_archiving_timeout');
-            const iq_result = await api.sendIQ(stanza, timeout, false)
+            const iq_result = await api.sendIQ(stanza, timeout, false);
             if (iq_result === null) {
                 const { __ } = _converse;
                 const err_msg = __("Timeout while trying to fetch archived messages.");
@@ -272,14 +298,14 @@ export default {
             }
             connection.deleteHandler(message_handler);
 
-            let rsm;
+            let rsm_result;
             const fin = iq_result && sizzle(`fin[xmlns="${NS.MAM}"]`, iq_result).pop();
-            const complete = fin?.getAttribute('complete') === 'true'
+            const complete = fin?.getAttribute('complete') === 'true';
             const set = sizzle(`set[xmlns="${NS.RSM}"]`, fin).pop();
             if (set) {
-                rsm = new RSM({...options, 'xml': set});
+                rsm_result = new RSM({...options.rsm, xml: set});
             }
-            return { messages, rsm, complete };
+            return { messages, rsm: rsm_result, complete };
         }
     }
 }

src/headless/plugins/mam/placeholder.js

@@ -1,12 +1,26 @@
-import { Model } from '@converse/skeletor';
-import { getUniqueId } from '../../utils/index.js';
+import { Model } from "@converse/skeletor";
+import { getUniqueId } from "../../utils/index.js";
+import u from "../../utils/index.js";
 
 export default class MAMPlaceholderMessage extends Model {
-
-    defaults () { // eslint-disable-line class-methods-use-this
+    defaults() {
         return {
             msgid: getUniqueId(),
-            is_ephemeral: false
+            is_ephemeral: false,
+        };
+    }
+
+    async fetchMissingMessages() {
+        this.set("fetching", true);
+        const options = {
+            rsm: {
+                before: this.get("before") ?? "", // We always query backwards (newest first)
+            },
+            mam: {
+                start: this.get("start"),
+            },
         };
+        await u.mam.fetchArchivedMessages(this.collection.chatbox, options);
+        this.destroy();
     }
 }

src/headless/plugins/mam/plugin.js

@@ -3,36 +3,36 @@
  * @copyright 2022, the Converse.js contributors
  * @license Mozilla Public License (MPLv2)
  */
-import { Strophe } from 'strophe.js';
-import _converse from '../../shared/_converse.js';
-import api from '../../shared/api/index.js';
-import converse from '../../shared/api/public.js';
-import { PRIVATE_CHAT_TYPE } from '../..//shared/constants.js';
-import '../disco/index.js';
-import mam_api from './api.js';
-import MAMPlaceholderMessage from './placeholder.js';
+import { Strophe } from "strophe.js";
+import _converse from "../../shared/_converse.js";
+import api from "../../shared/api/index.js";
+import converse from "../../shared/api/public.js";
+import { CHATROOMS_TYPE, PRIVATE_CHAT_TYPE } from "../..//shared/constants.js";
+import "../disco/index.js";
+import mam_api from "./api.js";
+import MAMPlaceholderMessage from "./placeholder.js";
 import {
+    createScrollupPlaceholder,
     fetchNewestMessages,
     getMAMPrefsFromFeature,
     handleMAMResult,
     onMAMError,
     onMAMPreferences,
     preMUCJoinMAMFetch,
-} from './utils.js';
-
+} from "./utils.js";
 
 const { NS } = Strophe;
 
-converse.plugins.add('converse-mam', {
-    dependencies: ['converse-disco', 'converse-muc'],
+converse.plugins.add("converse-mam", {
+    dependencies: ["converse-disco", "converse-muc"],
 
-    initialize () {
+    initialize() {
         api.settings.extend({
-            archived_messages_page_size: '50',
-            mam_request_all_pages: true,
+            archived_messages_page_size: "50",
+            mam_request_all_pages: false,
             message_archiving: undefined, // Supported values are 'always', 'never', 'roster'
-                                          // https://xmpp.org/extensions/xep-0313.html#prefs
-            message_archiving_timeout: 60000 // Time (in milliseconds) to wait before aborting MAM request
+            // https://xmpp.org/extensions/xep-0313.html#prefs
+            message_archiving_timeout: 60000, // Time (in milliseconds) to wait before aborting MAM request
         });
 
         Object.assign(api, mam_api);
@@ -42,28 +42,34 @@ converse.plugins.add('converse-mam', {
         Object.assign(_converse.exports, exports);
 
         /************************ Event Handlers ************************/
-        api.listen.on('addClientFeatures', () => api.disco.own.features.add(NS.MAM));
-        api.listen.on('serviceDiscovered', getMAMPrefsFromFeature);
-        api.listen.on('chatRoomViewInitialized', view => {
-            if (api.settings.get('muc_show_logs_before_join')) {
-                preMUCJoinMAMFetch(view.model);
+        api.listen.on("addClientFeatures", () => api.disco.own.features.add(NS.MAM));
+        api.listen.on("serviceDiscovered", getMAMPrefsFromFeature);
+        api.listen.on("chatRoomViewInitialized", ({ model }) => {
+            if (api.settings.get("muc_show_logs_before_join")) {
+                preMUCJoinMAMFetch(model);
                 // If we want to show MAM logs before entering the MUC, we need
                 // to be informed once it's clear that this MUC supports MAM.
-                view.model.features.on('change:mam_enabled', () => preMUCJoinMAMFetch(view.model));
+                model.features.on("change:mam_enabled", () => preMUCJoinMAMFetch(model));
             }
         });
-        api.listen.on('enteredNewRoom', muc => muc.features.get('mam_enabled') && fetchNewestMessages(muc));
+        api.listen.on(
+            "enteredNewRoom",
+            /** @param {import('../muc/muc').default} muc */ (muc) =>
+                muc.features.get("mam_enabled") && fetchNewestMessages(muc)
+        );
 
-        api.listen.on('chatReconnected', chat => {
-            if (chat.get('type') === PRIVATE_CHAT_TYPE) {
-                fetchNewestMessages(chat);
+        api.listen.on("chatReconnected", (chat) => {
+            if (![CHATROOMS_TYPE, PRIVATE_CHAT_TYPE].includes(chat.get("type"))) {
+                return;
             }
+            fetchNewestMessages(chat);
         });
 
-        api.listen.on('afterMessagesFetched', chat => {
-            if (chat.get('type') === PRIVATE_CHAT_TYPE) {
+        api.listen.on("afterMessagesFetched", (chat) => {
+            if (chat.get("type") === PRIVATE_CHAT_TYPE) {
                 fetchNewestMessages(chat);
             }
+            createScrollupPlaceholder(chat);
         });
-    }
+    },
 });

src/headless/plugins/mam/tests/api.js

@@ -1,4 +1,5 @@
 /*global mock, converse */
+const { stx } = converse.env;
 const dayjs = converse.env.dayjs;
 const Strophe = converse.env.Strophe;
 const $iq = converse.env.$iq;
@@ -9,7 +10,9 @@ const sizzle = converse.env.sizzle;
 describe("Message Archive Management", function () {
     describe("The archive.query API", function () {
 
-       it("can be used to query for all archived messages",
+        beforeAll(() => jasmine.addMatchers({ toEqualStanza: jasmine.toEqualStanza }));
+
+        it("can be used to query for all archived messages",
                 mock.initConverse(['discoInitialized'], {}, async function (_converse) {
 
             const sendIQ = _converse.api.connection.get().sendIQ;
@@ -22,8 +25,8 @@ describe("Message Archive Management", function () {
             _converse.api.archive.query();
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client"><query queryid="${queryid}" xmlns="urn:xmpp:mam:2"/></iq>`);
+            expect(sent_stanza).toEqualStanza(
+                stx`<iq id="${IQ_id}" type="set" xmlns="jabber:client"><query queryid="${queryid}" xmlns="urn:xmpp:mam:2"/></iq>`);
         }));
 
        it("can be used to query for all messages to/from a particular JID",
@@ -36,53 +39,52 @@ describe("Message Archive Management", function () {
                 sent_stanza = iq;
                 IQ_id = sendIQ.bind(this)(iq, callback, errback);
             });
-            _converse.api.archive.query({'with':'juliet@capulet.lit'});
+            _converse.api.archive.query({ mam: { with:'juliet@capulet.lit' }});
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                        `<field type="hidden" var="FORM_TYPE">`+
-                            `<value>urn:xmpp:mam:2</value>`+
-                        `</field>`+
-                        `<field var="with">`+
-                            `<value>juliet@capulet.lit</value>`+
-                        `</field>`+
-                        `</x>`+
-                    `</query>`+
-                `</iq>`);
+            expect(sent_stanza).toEqualStanza(stx`
+                <iq id="${IQ_id}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2">
+                        <x type="submit" xmlns="jabber:x:data">
+                        <field type="hidden" var="FORM_TYPE">
+                            <value>urn:xmpp:mam:2</value>
+                        </field>
+                        <field var="with">
+                            <value>juliet@capulet.lit</value>
+                        </field>
+                        </x>
+                    </query>
+                </iq>`);
         }));
 
        it("can be used to query for archived messages from a chat room",
                 mock.initConverse(['statusInitialized'], {}, async function (_converse) {
 
+            const { api } = _converse;
             const room_jid = 'coven@chat.shakespeare.lit';
-            _converse.api.archive.query({'with': room_jid, 'groupchat': true});
+            api.archive.query({ mam: { with: room_jid }, is_groupchat: true });
             await mock.waitUntilDiscoConfirmed(_converse, room_jid, null, [Strophe.NS.MAM]);
 
-            const sent_stanzas = _converse.api.connection.get().sent_stanzas;
+            const sent_stanzas = api.connection.get().sent_stanzas;
             const stanza = await u.waitUntil(
                 () => sent_stanzas.filter(s => sizzle(`[xmlns="${Strophe.NS.MAM}"]`, s).length).pop());
 
             const queryid = stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(stanza)).toBe(
-                `<iq id="${stanza.getAttribute('id')}" to="coven@chat.shakespeare.lit" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE">`+
-                                `<value>urn:xmpp:mam:2</value>`+
-                            `</field>`+
-                        `</x>`+
-                    `</query>`+
-                `</iq>`);
+            expect(stanza).toEqualStanza(stx`
+                <iq id="${stanza.getAttribute('id')}" to="${room_jid}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2"></query>
+                </iq>`);
        }));
 
         it("checks whether returned MAM messages from a MUC room are from the right JID",
                 mock.initConverse(['statusInitialized'], {}, async function (_converse) {
 
             const room_jid = 'coven@chat.shakespeare.lit';
-            const promise = _converse.api.archive.query({'with': room_jid, 'groupchat': true, 'max':'10'});
+            const promise = _converse.api.archive.query({
+                is_groupchat: true,
+                mam: { with: room_jid },
+                rsm: { "max": "10" }
+            });
 
             await mock.waitUntilDiscoConfirmed(_converse, room_jid, null, [Strophe.NS.MAM]);
 
@@ -90,56 +92,32 @@ describe("Message Archive Management", function () {
             const sent_stanza = await u.waitUntil(
                 () => sent_stanzas.filter(s => sizzle(`[xmlns="${Strophe.NS.MAM}"]`, s).length).pop());
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-
-            /* <message id='iasd207' from='coven@chat.shakespeare.lit' to='hag66@shakespeare.lit/pda'>
-             *     <result xmlns='urn:xmpp:mam:2' queryid='g27' id='34482-21985-73620'>
-             *         <forwarded xmlns='urn:xmpp:forward:0'>
-             *         <delay xmlns='urn:xmpp:delay' stamp='2002-10-13T23:58:37Z'/>
-             *         <message xmlns="jabber:client"
-             *             from='coven@chat.shakespeare.lit/firstwitch'
-             *             id='162BEBB1-F6DB-4D9A-9BD8-CFDCC801A0B2'
-             *             type='groupchat'>
-             *             <body>Thrice the brinded cat hath mew'd.</body>
-             *             <x xmlns='http://jabber.org/protocol/muc#user'>
-             *             <item affiliation='none'
-             *                     jid='witch1@shakespeare.lit'
-             *                     role='participant' />
-             *             </x>
-             *         </message>
-             *         </forwarded>
-             *     </result>
-             * </message>
-             */
-            const msg1 = $msg({'id':'iasd207', 'from': 'other@chat.shakespear.lit', 'to': 'romeo@montague.lit'})
-                        .c('result',  {'xmlns': 'urn:xmpp:mam:2', 'queryid':queryid, 'id':'34482-21985-73620'})
-                            .c('forwarded', {'xmlns':'urn:xmpp:forward:0'})
-                                .c('delay', {'xmlns':'urn:xmpp:delay', 'stamp':'2010-07-10T23:08:25Z'}).up()
-                                .c('message', {
-                                    'xmlns':'jabber:client',
-                                    'to':'romeo@montague.lit',
-                                    'id':'162BEBB1-F6DB-4D9A-9BD8-CFDCC801A0B2',
-                                    'from':'coven@chat.shakespeare.lit/firstwitch',
-                                    'type':'groupchat' })
-                                .c('body').t("Thrice the brinded cat hath mew'd.");
+            const msg1 = stx`<message id='iasd207' from='other@chat.shakespear.lit' to='romeo@montague.lit' xmlns="jabber:client">
+                        <result xmlns='urn:xmpp:mam:2' queryid='${queryid}' id='34482-21985-73620'>
+                            <forwarded xmlns='urn:xmpp:forward:0'>
+                                <delay xmlns='urn:xmpp:delay' stamp='2010-07-10T23:08:25Z'/>
+                                <message xmlns='jabber:client'
+                                    to='romeo@montague.lit'
+                                    id='162BEBB1-F6DB-4D9A-9BD8-CFDCC801A0B2'
+                                    from='coven@chat.shakespeare.lit/firstwitch'
+                                    type='groupchat'>
+                                    <body>Thrice the brinded cat hath mew'd.</body>
+                                </message>
+                            </forwarded>
+                        </result>
+                    </message>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(msg1));
 
-            /* Send an <iq> stanza to indicate the end of the result set.
-             *
-             * <iq type='result' id='juliet1'>
-             *     <fin xmlns='urn:xmpp:mam:2'>
-             *     <set xmlns='http://jabber.org/protocol/rsm'>
-             *         <first index='0'>28482-98726-73623</first>
-             *         <last>09af3-cc343-b409f</last>
-             *         <count>20</count>
-             *     </set>
-             * </iq>
-             */
-            const stanza = $iq({'type': 'result', 'id': sent_stanza.getAttribute('id')})
-                .c('fin', {'xmlns': 'urn:xmpp:mam:2'})
-                    .c('set',  {'xmlns': 'http://jabber.org/protocol/rsm'})
-                        .c('first', {'index': '0'}).t('23452-4534-1').up()
-                        .c('last').t('09af3-cc343-b409f').up()
-                        .c('count').t('16');
+            // Send an <iq> stanza to indicate the end of the result set.
+            const stanza = stx`<iq type='result' id='${sent_stanza.getAttribute('id')}' xmlns="jabber:client">
+                <fin xmlns='urn:xmpp:mam:2'>
+                    <set xmlns='http://jabber.org/protocol/rsm'>
+                        <first index='0'>23452-4534-1</first>
+                        <last>09af3-cc343-b409f</last>
+                        <count>16</count>
+                    </set>
+                </fin>
+            </iq>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(stanza));
 
             const result = await promise;
@@ -149,46 +127,46 @@ describe("Message Archive Management", function () {
        it("can be used to query for all messages in a certain timespan",
                 mock.initConverse([], {}, async function (_converse) {
 
+            const { api } = _converse;
             await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
             let sent_stanza, IQ_id;
-            const sendIQ = _converse.api.connection.get().sendIQ;
-            spyOn(_converse.api.connection.get(), 'sendIQ').and.callFake(function (iq, callback, errback) {
+            const sendIQ = api.connection.get().sendIQ;
+            spyOn(api.connection.get(), 'sendIQ').and.callFake(function (iq, callback, errback) {
                 sent_stanza = iq;
                 IQ_id = sendIQ.bind(this)(iq, callback, errback);
             });
             const start = '2010-06-07T00:00:00Z';
             const end = '2010-07-07T13:23:54Z';
-            _converse.api.archive.query({
-                'start': start,
-                'end': end
-            });
+            api.archive.query({ mam: { start, end }});
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                        `<field type="hidden" var="FORM_TYPE">`+
-                            `<value>urn:xmpp:mam:2</value>`+
-                        `</field>`+
-                        `<field var="start">`+
-                            `<value>${dayjs(start).toISOString()}</value>`+
-                        `</field>`+
-                        `<field var="end">`+
-                            `<value>${dayjs(end).toISOString()}</value>`+
-                        `</field>`+
-                        `</x>`+
-                    `</query>`+
-                `</iq>`
+            expect(sent_stanza).toEqualStanza(
+                stx`<iq id="${IQ_id}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2">
+                        <x type="submit" xmlns="jabber:x:data">
+                            <field type="hidden" var="FORM_TYPE">
+                                <value>urn:xmpp:mam:2</value>
+                            </field>
+                            <field var="start">
+                                <value>${dayjs(start).toISOString()}</value>
+                            </field>
+                            <field var="end">
+                                <value>${dayjs(end).toISOString()}</value>
+                            </field>
+                        </x>
+                    </query>
+                </iq>`
             );
        }));
 
        it("throws a TypeError if an invalid date is provided",
                 mock.initConverse([], {}, async function (_converse) {
 
-            await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
+            let promise;
             try {
-                await _converse.api.archive.query({'start': 'not a real date'});
+                promise = _converse.api.archive.query({ mam: { start: 'not a real date' }});
+                await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
+                await promise;
             } catch (e) {
                 expect(() => {throw e}).toThrow(new TypeError('archive.query: invalid date provided for: start'));
             }
@@ -208,22 +186,22 @@ describe("Message Archive Management", function () {
                 _converse.disco_entities.get(_converse.domain).features.create({'var': Strophe.NS.MAM});
             }
             const start = '2010-06-07T00:00:00Z';
-            _converse.api.archive.query({'start': start});
+            _converse.api.archive.query({ mam: { start }});
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                        `<field type="hidden" var="FORM_TYPE">`+
-                            `<value>urn:xmpp:mam:2</value>`+
-                        `</field>`+
-                        `<field var="start">`+
-                            `<value>${dayjs(start).toISOString()}</value>`+
-                        `</field>`+
-                        `</x>`+
-                    `</query>`+
-                `</iq>`
+            expect(sent_stanza).toEqualStanza(
+                stx`<iq id="${IQ_id}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2">
+                        <x type="submit" xmlns="jabber:x:data">
+                            <field type="hidden" var="FORM_TYPE">
+                                <value>urn:xmpp:mam:2</value>
+                            </field>
+                            <field var="start">
+                                <value>${dayjs(start).toISOString()}</value>
+                            </field>
+                        </x>
+                    </query>
+                </iq>`
             );
        }));
 
@@ -238,65 +216,28 @@ describe("Message Archive Management", function () {
                 IQ_id = sendIQ.bind(this)(iq, callback, errback);
             });
             const start = '2010-06-07T00:00:00Z';
-            _converse.api.archive.query({'start': start, 'max':10});
+            _converse.api.archive.query({ mam: { start }, rsm: { max:10 }});
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE">`+
-                                `<value>urn:xmpp:mam:2</value>`+
-                            `</field>`+
-                            `<field var="start">`+
-                                `<value>${dayjs(start).toISOString()}</value>`+
-                            `</field>`+
-                        `</x>`+
-                        `<set xmlns="http://jabber.org/protocol/rsm">`+
-                            `<max>10</max>`+
-                        `</set>`+
-                    `</query>`+
-                `</iq>`
+            expect(sent_stanza).toEqualStanza(
+                stx`<iq id="${IQ_id}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2">
+                        <x type="submit" xmlns="jabber:x:data">
+                            <field type="hidden" var="FORM_TYPE">
+                                <value>urn:xmpp:mam:2</value>
+                            </field>
+                            <field var="start">
+                                <value>${dayjs(start).toISOString()}</value>
+                            </field>
+                        </x>
+                        <set xmlns="http://jabber.org/protocol/rsm">
+                            <max>10</max>
+                        </set>
+                    </query>
+                </iq>`
             );
        }));
 
-       it("can be used to page through results",
-                mock.initConverse([], {}, async function (_converse) {
-
-            await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
-            let sent_stanza, IQ_id;
-            const sendIQ = _converse.api.connection.get().sendIQ;
-            spyOn(_converse.api.connection.get(), 'sendIQ').and.callFake(function (iq, callback, errback) {
-                sent_stanza = iq;
-                IQ_id = sendIQ.bind(this)(iq, callback, errback);
-            });
-            const start = '2010-06-07T00:00:00Z';
-            _converse.api.archive.query({
-                'start': start,
-                'after': '09af3-cc343-b409f',
-                'max':10
-            });
-            await u.waitUntil(() => sent_stanza);
-            const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE">`+
-                                `<value>urn:xmpp:mam:2</value>`+
-                            `</field>`+
-                            `<field var="start">`+
-                                `<value>${dayjs(start).toISOString()}</value>`+
-                            `</field>`+
-                        `</x>`+
-                        `<set xmlns="http://jabber.org/protocol/rsm">`+
-                            `<after>09af3-cc343-b409f</after>`+
-                            `<max>10</max>`+
-                        `</set>`+
-                    `</query>`+
-                `</iq>`);
-       }));
-
        it("accepts \"before\" with an empty string as value to reverse the order",
                 mock.initConverse([], {}, async function (_converse) {
 
@@ -307,23 +248,18 @@ describe("Message Archive Management", function () {
                 sent_stanza = iq;
                 IQ_id = sendIQ.bind(this)(iq, callback, errback);
             });
-            _converse.api.archive.query({'before': '', 'max':10});
+            _converse.api.archive.query({ rsm: { before: '', max: 10 }});
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
-            expect(Strophe.serialize(sent_stanza)).toBe(
-                `<iq id="${IQ_id}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${queryid}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE">`+
-                                `<value>urn:xmpp:mam:2</value>`+
-                            `</field>`+
-                        `</x>`+
-                        `<set xmlns="http://jabber.org/protocol/rsm">`+
-                            `<before></before>`+
-                            `<max>10</max>`+
-                        `</set>`+
-                    `</query>`+
-                `</iq>`);
+            expect(sent_stanza).toEqualStanza(
+                stx`<iq id="${IQ_id}" type="set" xmlns="jabber:client">
+                    <query queryid="${queryid}" xmlns="urn:xmpp:mam:2">
+                        <set xmlns="http://jabber.org/protocol/rsm">
+                            <before></before>
+                            <max>10</max>
+                        </set>
+                    </query>
+                </iq>`);
        }));
 
        it("returns an object which includes the messages and a _converse.RSM object",
@@ -336,66 +272,50 @@ describe("Message Archive Management", function () {
                 sent_stanza = iq;
                 IQ_id = sendIQ.bind(this)(iq, callback, errback);
             });
-            const promise = _converse.api.archive.query({'with': 'romeo@capulet.lit', 'max':'10'});
+            const promise = _converse.api.archive.query({ mam: { with: 'romeo@capulet.lit' }, rsm: { max:'10' }});
 
             await u.waitUntil(() => sent_stanza);
             const queryid = sent_stanza.querySelector('query').getAttribute('queryid');
 
-            /*  <message id='aeb213' to='juliet@capulet.lit/chamber'>
-             *  <result xmlns='urn:xmpp:mam:2' queryid='f27' id='28482-98726-73623'>
-             *      <forwarded xmlns='urn:xmpp:forward:0'>
-             *      <delay xmlns='urn:xmpp:delay' stamp='2010-07-10T23:08:25Z'/>
-             *      <message xmlns='jabber:client'
-             *          to='juliet@capulet.lit/balcony'
-             *          from='romeo@montague.lit/orchard'
-             *          type='chat'>
-             *          <body>Call me but love, and I'll be new baptized; Henceforth I never will be Romeo.</body>
-             *      </message>
-             *      </forwarded>
-             *  </result>
-             *  </message>
-             */
-            const msg1 = $msg({'id':'aeb212', 'to':'juliet@capulet.lit/chamber'})
-                        .c('result',  {'xmlns': 'urn:xmpp:mam:2', 'queryid':queryid, 'id':'28482-98726-73623'})
-                            .c('forwarded', {'xmlns':'urn:xmpp:forward:0'})
-                                .c('delay', {'xmlns':'urn:xmpp:delay', 'stamp':'2010-07-10T23:08:25Z'}).up()
-                                .c('message', {
-                                    'xmlns':'jabber:client',
-                                    'to':'juliet@capulet.lit/balcony',
-                                    'from':'romeo@montague.lit/orchard',
-                                    'type':'chat' })
-                                .c('body').t("Call me but love, and I'll be new baptized;");
+            const msg1 = stx`<message id='aeb212' to='juliet@capulet.lit/chamber' xmlns="jabber:client">
+                        <result xmlns='urn:xmpp:mam:2' queryid='${queryid}' id='28482-98726-73623'>
+                            <forwarded xmlns='urn:xmpp:forward:0'>
+                                <delay xmlns='urn:xmpp:delay' stamp='2010-07-10T23:08:25Z'/>
+                                <message xmlns='jabber:client'
+                                        to='juliet@capulet.lit/balcony'
+                                        from='romeo@montague.lit/orchard'
+                                        type='chat'>
+                                    <body>Call me but love, and I'll be new baptized.</body>
+                                </message>
+                            </forwarded>
+                        </result>
+                    </message>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(msg1));
 
-            const msg2 = $msg({'id':'aeb213', 'to':'juliet@capulet.lit/chamber'})
-                        .c('result',  {'xmlns': 'urn:xmpp:mam:2', 'queryid':queryid, 'id':'28482-98726-73624'})
-                            .c('forwarded', {'xmlns':'urn:xmpp:forward:0'})
-                                .c('delay', {'xmlns':'urn:xmpp:delay', 'stamp':'2010-07-10T23:08:25Z'}).up()
-                                .c('message', {
-                                    'xmlns':'jabber:client',
-                                    'to':'juliet@capulet.lit/balcony',
-                                    'from':'romeo@montague.lit/orchard',
-                                    'type':'chat' })
-                                .c('body').t("Henceforth I never will be Romeo.");
+            const msg2 = stx`<message id='aeb213' to='juliet@capulet.lit/chamber' xmlns="jabber:client">
+                        <result xmlns='urn:xmpp:mam:2' queryid='${queryid}' id='28482-98726-73624'>
+                            <forwarded xmlns='urn:xmpp:forward:0'>
+                                <delay xmlns='urn:xmpp:delay' stamp='2010-07-10T23:08:25Z'/>
+                                <message xmlns='jabber:client'
+                                    to='juliet@capulet.lit/balcony'
+                                    from='romeo@montague.lit/orchard'
+                                    type='chat'>
+                                    <body>Henceforth I never will be Romeo.</body>
+                                </message>
+                            </forwarded>
+                        </result>
+                    </message>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(msg2));
 
-            /* Send an <iq> stanza to indicate the end of the result set.
-             *
-             * <iq type='result' id='juliet1'>
-             *     <fin xmlns='urn:xmpp:mam:2'>
-             *     <set xmlns='http://jabber.org/protocol/rsm'>
-             *         <first index='0'>28482-98726-73623</first>
-             *         <last>09af3-cc343-b409f</last>
-             *         <count>20</count>
-             *     </set>
-             * </iq>
-             */
-            const stanza = $iq({'type': 'result', 'id': IQ_id})
-                .c('fin', {'xmlns': 'urn:xmpp:mam:2'})
-                    .c('set',  {'xmlns': 'http://jabber.org/protocol/rsm'})
-                        .c('first', {'index': '0'}).t('23452-4534-1').up()
-                        .c('last').t('09af3-cc343-b409f').up()
-                        .c('count').t('16');
+            const stanza = stx`<iq type='result' id='${IQ_id}' xmlns="jabber:client">
+                <fin xmlns='urn:xmpp:mam:2'>
+                    <set xmlns='http://jabber.org/protocol/rsm'>
+                        <first index='0'>28482-98726-73623</first>
+                        <last>09af3-cc343-b409f</last>
+                        <count>16</count>
+                    </set>
+                </fin>
+            </iq>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(stanza));
 
             const result = await promise;
@@ -404,7 +324,7 @@ describe("Message Archive Management", function () {
             expect(result.messages[1].outerHTML).toBe(msg2.nodeTree.outerHTML);
             expect(result.rsm.query.max).toBe('10');
             expect(result.rsm.result.count).toBe(16);
-            expect(result.rsm.result.first).toBe('23452-4534-1');
+            expect(result.rsm.result.first).toBe('28482-98726-73623');
             expect(result.rsm.result.last).toBe('09af3-cc343-b409f');
        }));
     });

src/headless/plugins/mam/types.ts

@@ -1,36 +1,31 @@
 import { RSM } from '../../shared/rsm';
 
-export type MAMOptions = {
-    max?: number; // The maximum number of items to return. Defaults to "archived_messages_page_size"
-    after?: string; // The XEP-0359 stanza ID of a message after which messages should be returned. Implies forward paging.
-    before?: string; // The XEP-0359 stanza ID of a message before which messages should be returned. Implies backward paging.
-    end?: string; // A date string in ISO-8601 format, before which messages should be returned. Implies backward paging.
-    start?: string; // A date string in ISO-8601 format, after which messages should be returned. Implies forward paging.
-    with?: string; // The JID of the entity with which messages were exchanged.
-    groupchat?: boolean; // True if archive in groupchat.
+export type MAMQueryOptions = {
+    end?: string; // A date string in ISO-8601 format, before which messages should be returned.
+    start?: string; // A date string in ISO-8601 format, after which messages should be returned.
+    with?: string; // A JID against which to match messages, according to either the `to` or `from` attribute.
+    // An item in a MUC archive matches if the publisher of the item matches the JID.
+    // If `with` is omitted, all messages that match the rest of the query will be returned, regardless of to/from
+    // addresses of each message.
 };
 
 // XEP-0059 RSM Attributes that can be used to filter query results
-type RSMQueryParameters = {
+// Specifying both "after" and "before" is undefined behavior.
+type RSMQueryOptions = {
     after?: string; // The XEP-0359 stanza ID of a message after which messages should be returned. Implies forward paging.
     before?: string; // The XEP-0359 stanza ID of a message before which messages should be returned. Implies backward paging.
     index?: number; // The index of the results page to return.
     max?: number; // The maximum number of items to return.
 };
 
-// Filter parmeters which can be used to filter a MAM XEP-0313 archive
-export type MAMFilterParameters = RSMQueryParameters & {
-    end?: string; // A date string in ISO-8601 format, before which messages should be returned. Implies backward paging.
-    start?: string; // A date string in ISO-8601 format, after which messages should be returned. Implies forward paging.
-    with?: string; // A JID against which to match messages, according to either their `to` or `from` attributes.
-    // An item in a MUC archive matches if the publisher of the item matches the JID.
-    // If `with` is omitted, all messages that match the rest of the query will be returned, regardless of to/from
-    // addresses of each message.
+export type FetchArchivedMessagesOptions = {
+    mam?: MAMQueryOptions;
+    rsm?: RSMQueryOptions
 };
 
 // The options that can be passed in to the api.archive.query method
-export type ArchiveQueryOptions = MAMFilterParameters & {
-    groupchat?: boolean; // Whether the MAM archive is for a groupchat.
+export type ArchiveQueryOptions = FetchArchivedMessagesOptions & {
+    is_groupchat?: boolean; // Whether the MAM archive is for a groupchat.
 };
 
 export type MAMQueryResult = {

src/headless/plugins/mam/utils.js

@@ -1,35 +1,38 @@
 /**
- * @typedef {import('../muc/muc.js').default} MUC
- * @typedef {import('../chat/model.js').default} ChatBox
+ * @typedef {import('../muc/muc').default} MUC
+ * @typedef {import('../chat/model').default} ChatBox
  * @typedef {import('@converse/skeletor/src/types/helpers.js').Model} Model
  */
-import sizzle from 'sizzle';
-import { Strophe, $iq } from 'strophe.js';
-import _converse from '../../shared/_converse.js';
-import api from '../../shared/api/index.js';
-import converse from '../../shared/api/public.js';
-import log from '../../log.js';
-import { parseMUCMessage } from '../../plugins/muc/parsers.js';
-import { parseMessage } from '../../plugins/chat/parsers.js';
-import { CHATROOMS_TYPE } from '../../shared/constants.js';
-import { TimeoutError } from '../../shared/errors.js';
-import MAMPlaceholderMessage from './placeholder.js';
-import { parseErrorStanza } from '../../shared/parsers.js';
+import sizzle from "sizzle";
+import { Strophe, $iq } from "strophe.js";
+import _converse from "../../shared/_converse.js";
+import api from "../../shared/api/index.js";
+import converse from "../../shared/api/public.js";
+import log from "../../log.js";
+import { parseMUCMessage } from "../../plugins/muc/parsers.js";
+import { parseMessage } from "../../plugins/chat/parsers.js";
+import { CHATROOMS_TYPE } from "../../shared/constants.js";
+import { TimeoutError } from "../../shared/errors.js";
+import MAMPlaceholderMessage from "./placeholder.js";
+import { parseErrorStanza } from "../../shared/parsers.js";
 
 const { NS } = Strophe;
 const u = converse.env.utils;
 
 /**
+ * @param {Element|Error} e
  * @param {Element} iq
  */
-export async function onMAMError(iq) {
-    const err = await parseErrorStanza(iq);
-    if (err?.name === 'feature-not-implemented') {
-        log.warn(`Message Archive Management (XEP-0313) not supported by ${iq.getAttribute('from')}`);
-    } else {
-        log.error(`Error while trying to set archiving preferences for ${iq.getAttribute('from')}.`);
-        log.error(iq);
+export async function onMAMError(e, iq) {
+    if (u.isElement(e)) {
+        const err = await parseErrorStanza(e);
+        if (err?.name === "feature-not-implemented") {
+            log.warn(`Message Archive Management (XEP-0313) not supported by ${iq.getAttribute("to")}`);
+            return;
+        }
     }
+    log.error(`Error while trying to set archiving preferences for ${iq.getAttribute("to")}.`);
+    log.error(iq);
 }
 
 /**
@@ -48,11 +51,11 @@ export async function onMAMError(iq) {
  */
 export function onMAMPreferences(iq, feature) {
     const preference = sizzle(`prefs[xmlns="${NS.MAM}"]`, iq).pop();
-    const default_pref = preference.getAttribute('default');
-    if (default_pref !== api.settings.get('message_archiving')) {
-        const stanza = $iq({ 'type': 'set' }).c('prefs', {
-            'xmlns': NS.MAM,
-            'default': api.settings.get('message_archiving'),
+    const default_pref = preference.getAttribute("default");
+    if (default_pref !== api.settings.get("message_archiving")) {
+        const stanza = $iq({ "type": "set" }).c("prefs", {
+            "xmlns": NS.MAM,
+            "default": api.settings.get("message_archiving"),
         });
         Array.from(preference.children).forEach((child) => stanza.cnode(child).up());
 
@@ -60,10 +63,10 @@ export function onMAMPreferences(iq, feature) {
         // (see example 18: https://xmpp.org/extensions/xep-0313.html#config)
         // but Prosody doesn't do this, so we don't rely on it.
         api.sendIQ(stanza)
-            .then(() => feature.save({ 'preferences': { 'default': api.settings.get('message_archiving') } }))
-            .catch(_converse.exports.onMAMError);
+            .then(() => feature.save({ "preferences": { "default": api.settings.get("message_archiving") } }))
+            .catch(/** @param {Error|Element} e */ (e) => _converse.exports.onMAMError(e, stanza.tree()));
     } else {
-        feature.save({ 'preferences': { 'default': api.settings.get('message_archiving') } });
+        feature.save({ "preferences": { "default": api.settings.get("message_archiving") } });
     }
 }
 
@@ -71,39 +74,44 @@ export function onMAMPreferences(iq, feature) {
  * @param {Model} feature
  */
 export function getMAMPrefsFromFeature(feature) {
-    const prefs = feature.get('preferences') || {};
-    if (feature.get('var') !== NS.MAM || api.settings.get('message_archiving') === undefined) {
+    const prefs = feature.get("preferences") || {};
+    if (feature.get("var") !== NS.MAM || api.settings.get("message_archiving") === undefined) {
         return;
     }
-    if (prefs['default'] !== api.settings.get('message_archiving')) {
-        api.sendIQ($iq({ 'type': 'get' }).c('prefs', { 'xmlns': NS.MAM }))
+    if (prefs["default"] !== api.settings.get("message_archiving")) {
+        const stanza = $iq({ "type": "get" }).c("prefs", { "xmlns": NS.MAM });
+        api.sendIQ(stanza)
             .then(/** @param {Element} iq */ (iq) => _converse.exports.onMAMPreferences(iq, feature))
-            .catch(_converse.exports.onMAMError);
+            .catch(/** @param {Error|Element} e */ (e) => _converse.exports.onMAMError(e, stanza.tree()));
     }
 }
 
 /**
  * @param {MUC} muc
  */
-export function preMUCJoinMAMFetch(muc) {
+export async function preMUCJoinMAMFetch(muc) {
     if (
-        !api.settings.get('muc_show_logs_before_join') ||
-        !muc.features.get('mam_enabled') ||
-        muc.get('prejoin_mam_fetched')
+        !api.settings.get("muc_show_logs_before_join") ||
+        !muc.features.get("mam_enabled") ||
+        muc.get("prejoin_mam_fetched")
     ) {
         return;
     }
-    fetchNewestMessages(muc);
-    muc.save({ 'prejoin_mam_fetched': true });
+    await fetchNewestMessages(muc);
+    muc.save({ "prejoin_mam_fetched": true });
 }
 
-async function createMessageFromError (model, error) {
+/**
+ * @param {ChatBox|MUC} model
+ * @param {Error|string} error
+ */
+async function createMessageFromError(model, error) {
     if (error instanceof TimeoutError) {
         const msg = await model.createMessage({
-            'type': 'error',
-            'message': error.message,
-            'retry_event_id': error.retry_event_id,
-            'is_ephemeral': 20000,
+            type: "error",
+            message: error.message,
+            retry_event_id: error.retry_event_id,
+            is_ephemeral: 20000,
         });
         msg.error = error;
     }
@@ -114,10 +122,10 @@ async function createMessageFromError (model, error) {
  * @param {Object} result
  * @param {Object} query
  * @param {Object} options
- * @param {('forwards'|'backwards'|null)} [should_page=null]
+ * @param {('forwards'|'backwards'|false)} [should_page=false]
  */
-export async function handleMAMResult(model, result, query, options, should_page) {
-    const is_muc = model.get('type') === CHATROOMS_TYPE;
+export async function handleMAMResult(model, result, query, options, should_page = false) {
+    const is_muc = model.get("type") === CHATROOMS_TYPE;
     const doParseMessage = /** @param {Element} s*/ (s) =>
         is_muc ? parseMUCMessage(s, /** @type {MUC} */ (model)) : parseMessage(s);
 
@@ -129,8 +137,8 @@ export async function handleMAMResult(model, result, query, options, should_page
      * work based on the MAM result before calling the handlers here.
      * @event _converse#MAMResult
      */
-    const data = { query, 'chatbox': model, messages };
-    await api.trigger('MAMResult', data, { 'synchronous': true });
+    const data = { query, "chatbox": model, messages };
+    await api.trigger("MAMResult", data, { "synchronous": true });
 
     messages.forEach((m) => model.queueMessage(m));
     if (result.error) {
@@ -143,113 +151,131 @@ export async function handleMAMResult(model, result, query, options, should_page
 /**
  * Fetch XEP-0313 archived messages based on the passed in criteria.
  * @param {ChatBox|MUC} model
- * @param {import('./types').MAMOptions} [options]
- * @param {('forwards'|'backwards'|null)} [should_page=null] - Determines whether
+ * @param {import('./types').FetchArchivedMessagesOptions} [options]
+ * @param {('forwards'|'backwards'|false)} [should_page=false] - Determines whether
  *  this function should recursively page through the entire result set if a limited
  *  number of results were returned.
  */
-export async function fetchArchivedMessages(model, options = {}, should_page = null) {
-    if (model.disable_mam) {
-        return;
-    }
-    const is_muc = model.get('type') === CHATROOMS_TYPE;
-    const bare_jid = _converse.session.get('bare_jid');
-    const mam_jid = is_muc ? model.get('jid') : bare_jid;
-    if (!(await api.disco.supports(NS.MAM, mam_jid))) {
-        return;
-    }
-    const max = api.settings.get('archived_messages_page_size');
+export async function fetchArchivedMessages(model, options = {}, should_page = false) {
+    if (model.disable_mam) return;
+
+    const is_muc = model.get("type") === CHATROOMS_TYPE;
+    const bare_jid = _converse.session.get("bare_jid");
+    const mam_jid = is_muc ? model.get("jid") : bare_jid;
 
-    const query = Object.assign(
-        {
-            'groupchat': is_muc,
-            'max': max,
-            'with': model.get('jid'),
+    const supported = await api.disco.supports(NS.MAM, mam_jid);
+    if (!supported) return;
+
+    const max = api.settings.get("archived_messages_page_size");
+    const query = /** @type {import('./types').ArchiveQueryOptions} */ {
+        is_groupchat: is_muc,
+        rsm: {
+            max,
+            ...options.rsm,
+        },
+        mam: {
+            with: model.get("jid"),
+            ...options.mam,
         },
-        options
-    );
+    };
 
     const result = await api.archive.query(query);
     await handleMAMResult(model, result, query, options, should_page);
 
     if (result.rsm && !result.complete) {
         if (should_page) {
-            if (should_page === 'forwards') {
-                options = result.rsm.next(max, options.before).query;
-            } else if (should_page === 'backwards') {
-                options = result.rsm.previous(max, options.after).query;
+            if (should_page === "forwards") {
+                options = result.rsm.next(max, options.rsm.before).query;
+            } else if (should_page === "backwards") {
+                options = result.rsm.previous(max, options.rsm.after).query;
             }
             return fetchArchivedMessages(model, options, should_page);
         } else {
-            createPlaceholder(model, options, result);
+            createGapPlaceholder(model, options, result);
         }
     }
 }
 
 /**
- * Create a placeholder message which is used to indicate gaps in the history.
+ * Creates a placeholder to fill gaps in the history.
  * @param {ChatBox|MUC} model
- * @param {import('./types').MAMOptions} options
- * @param {object} result - The RSM result object
+ * @param {import('./types').ArchiveQueryOptions} [options]
+ * @param {import('./types').MAMQueryResult} [result]
  */
-async function createPlaceholder(model, options, result) {
-    if (options.before == '' && (model.messages.length === 0 || !options.start)) {
-        // Fetching the latest MAM messages with an empty local cache
-        return;
-    }
-    if (options.before && !options.start) {
-        // Infinite scrolling upward
-        return;
-    }
-    if (options.before == null) {
-        // eslint-disable-line no-eq-null
-        // Adding placeholders when paging forwards is not supported yet,
-        // since currently with standard Converse, we only page forwards
-        // when fetching the entire history (i.e. no gaps should arise).
-        return;
-    }
+async function createGapPlaceholder(model, options, result) {
     const msgs = await Promise.all(result.messages);
+
+    const is_muc = model.get("type") === CHATROOMS_TYPE;
+    const mam_jid = is_muc ? model.get("jid") : _converse.session.get("bare_jid");
+
     const { rsm } = result;
-    const key = `stanza_id ${model.get('jid')}`;
+    const key = `stanza_id ${mam_jid}`;
     const adjacent_message = msgs.find((m) => m[key] === rsm.result.first);
-    const adjacent_message_date = new Date(adjacent_message['time']);
+    const adjacent_message_date = new Date(adjacent_message["time"]);
 
     const msg_data = {
-        'template_hook': 'getMessageTemplate',
-        'time': new Date(adjacent_message_date.getTime() - 1).toISOString(),
-        'before': rsm.result.first,
-        'start': options.start,
+        before: rsm.result.first,
+        start: options.mam?.start,
+        template_hook: "getMessageTemplate",
+        time: new Date(adjacent_message_date.getTime() - 1).toISOString(),
     };
+
+    if (model.messages.findWhere(msg_data)) {
+        log.debug("Gap placeholder already exists, not recreating.");
+        return;
+    }
+
     model.messages.add(new MAMPlaceholderMessage(msg_data));
 }
 
+/**
+ * Creates a placeholder to fetch messages at the top of the chat history.
+ * @param {ChatBox|MUC} model
+ */
+export function createScrollupPlaceholder(model) {
+    if (model.messages.length) {
+        const is_muc = model.get("type") === CHATROOMS_TYPE;
+        const mam_jid = is_muc ? model.get("jid") : _converse.session.get("bare_jid");
+        const key = `stanza_id ${mam_jid}`;
+        const oldest_message = model.getOldestMessage();
+
+        const msg_data = {
+            before: oldest_message.get(key),
+            template_hook: "getMessageTemplate",
+            time: new Date((new Date(oldest_message.get('time'))).getTime() - 1).toISOString(),
+        };
+
+        if (model.messages.findWhere(msg_data)) {
+            log.debug("Gap placeholder already exists, not recreating.");
+            return;
+        }
+        model.messages.add(new MAMPlaceholderMessage(msg_data));
+    }
+}
+
 /**
  * Fetches messages that might have been archived *after*
  * the last archived message in our local cache.
  * @param {ChatBox|MUC} model
  */
-export function fetchNewestMessages(model) {
-    if (model.disable_mam) {
-        return;
-    }
+export async function fetchNewestMessages(model) {
+    if (model.disable_mam) return;
+
+    const is_muc = model.get("type") === CHATROOMS_TYPE;
+    const disco_jid = is_muc ? model.get("jid") : _converse.session.get("bare_jid");
+    const supported = await api.disco.supports(NS.MAM, disco_jid);
+    if (!supported) return;
+
     const most_recent_msg = model.getMostRecentMessage();
+    const should_page = api.settings.get("mam_request_all_pages") ? "backwards" : false;
 
-    // if clear_messages_on_reconnection is true, than any recent messages
-    // must have been received *after* connection and we instead must query
-    // for earlier messages
-    if (most_recent_msg && !api.settings.get('clear_messages_on_reconnection')) {
-        const should_page = api.settings.get('mam_request_all_pages');
-        if (should_page) {
-            const stanza_id = most_recent_msg.get(`stanza_id ${model.get('jid')}`);
-            if (stanza_id) {
-                fetchArchivedMessages(model, { 'after': stanza_id }, 'forwards');
-            } else {
-                fetchArchivedMessages(model, { 'start': most_recent_msg.get('time') }, 'forwards');
-            }
-        } else {
-            fetchArchivedMessages(model, { 'before': '', 'start': most_recent_msg.get('time') });
-        }
+    if (most_recent_msg) {
+        // FIXME: A race condition might happen where, where a new message comes in
+        // before we've fetched the archived messages.
+        // Can be fixed by preventing processing of new messages until MAM
+        // has been fetched and processed.
+        fetchArchivedMessages(model, { mam: { start: most_recent_msg.get("time") }, rsm: { before: "" } }, should_page);
     } else {
-        fetchArchivedMessages(model, { 'before': '' });
+        fetchArchivedMessages(model, { rsm: { before: "" } }, should_page);
     }
 }

src/headless/shared/message.js

@@ -40,8 +40,8 @@ class BaseMessage extends ModelWithVCard(ModelWithContact(ColorAwareModel(Model)
         this.lazy_load_vcard = true;
         super.initialize();
 
-        if (!this.checkValidity()) return;
         this.chatbox = this.collection?.chatbox;
+        if (!this.checkValidity()) return;
 
         if (this.get('file')) {
             this.on('change:put', () => this.uploadFile());

src/headless/shared/model-with-messages.js

@@ -128,12 +128,12 @@ export default function ModelWithMessages(BaseModel) {
             this.messages.fetched_flag = true;
             const resolve = this.messages.fetched.resolve;
             this.messages.fetch({
-                'add': true,
-                'success': () => {
+                add: true,
+                success: () => {
                     this.afterMessagesFetched();
                     resolve();
                 },
-                'error': () => {
+                error: () => {
                     this.afterMessagesFetched();
                     resolve();
                 },

src/headless/shared/rsm.js

@@ -19,13 +19,13 @@ const toNumber = (v) => Number(v);
 const toString = (v) => v.toString();
 
 export const RSM_TYPES = {
-    'after': toString,
-    'before': toString,
-    'count': toNumber,
-    'first': toString,
-    'index': toNumber,
-    'last': toString,
-    'max': toNumber
+    after: toString,
+    before: toString,
+    count: toNumber,
+    first: toString,
+    index: toNumber,
+    last: toString,
+    max: toNumber
 };
 
 const isUndefined = (x) => typeof x === 'undefined';
@@ -72,7 +72,7 @@ export class RSM {
 
     /**
      * Returns a `<set>` XML element that confirms to XEP-0059 Result Set Management.
-     * The element is constructed based on the RSMQueryParameters
+     * The element is constructed based on the RSMQueryOptions
      * that are set on this RSM instance.
      * @returns {Element}
      */

src/headless/shared/types.ts

@@ -9,6 +9,12 @@ export interface ModelOptions {
     silent?: boolean;
 }
 
+export type RSMResult = {
+    count?: string;
+    first?: string;
+    last?: string;
+};
+
 // Types for mixins.
 // -----------------
 

src/headless/types/plugins/disco/entity.d.ts

@@ -20,15 +20,15 @@ declare class DiscoEntity extends Model {
      * Returns a Promise which resolves with a map indicating
      * whether a given identity is provided by this entity.
      * @method _converse.DiscoEntity#getIdentity
-     * @param { String } category - The identity category
-     * @param { String } type - The identity type
+     * @param {String} category - The identity category
+     * @param {String} type - The identity type
      */
     getIdentity(category: string, type: string): Promise<any>;
     /**
      * Returns a Promise which resolves with a map indicating
      * whether a given feature is supported.
      * @method _converse.DiscoEntity#getFeature
-     * @param { String } feature - The feature that might be supported.
+     * @param {String} feature - The feature that might be supported.
      */
     getFeature(feature: string): Promise<this>;
     onFeatureAdded(feature: any): void;

src/headless/types/plugins/mam/api.d.ts

@@ -7,7 +7,7 @@ declare namespace _default {
          * RSM to enable easy querying between results pages.
          *
          * @method _converse.api.archive.query
-         * @param {import('./types').ArchiveQueryOptions} options - An object containing query parameters
+         * @param {import('./types').ArchiveQueryOptions} [options={}] - Optional query parameters
          * @throws {Error} An error is thrown if the XMPP server responds with an error.
          * @returns {Promise<import('./types').MAMQueryResult>}
          *
@@ -40,7 +40,7 @@ declare namespace _default {
          * // For a particular user
          * let result;
          * try {
-         *    result = await api.archive.query({'with': 'john@doe.net'});
+         *    result = await api.archive.query({ mam: { with: 'john@doe.net' }});
          * } catch (e) {
          *     // The query was not successful
          * }
@@ -48,7 +48,7 @@ declare namespace _default {
          * // For a particular room
          * let result;
          * try {
-         *    result = await api.archive.query({'with': 'discuss@conference.doglovers.net', 'groupchat': true});
+         *    result = await api.archive.query({ mam: { with: 'discuss@conference.doglovers.net' }}, is_groupchat: true });
          * } catch (e) {
          *     // The query was not successful
          * }
@@ -57,14 +57,16 @@ declare namespace _default {
          * // Requesting all archived messages before or after a certain date
          * // ===============================================================
          * //
-         * // The `start` and `end` parameters are used to query for messages
+         * // The MAM `start` and `end` parameters are used to query for messages
          * // within a certain timeframe. The passed in date values may either be ISO8601
          * // formatted date strings, or JavaScript Date objects.
          *
          *  const options = {
-         *      'with': 'john@doe.net',
-         *      'start': '2010-06-07T00:00:00Z',
-         *      'end': '2010-07-07T13:23:54Z'
+         *      mam: {
+         *          'with': 'john@doe.net',
+         *          'start': '2010-06-07T00:00:00Z',
+         *          'end': '2010-07-07T13:23:54Z'
+         *      },
          *  };
          * let result;
          * try {
@@ -83,7 +85,7 @@ declare namespace _default {
          * // Return maximum 10 archived messages
          * let result;
          * try {
-         *     result = await api.archive.query({'with': 'john@doe.net', 'max':10});
+         *     result = await api.archive.query({ mam: { with: 'john@doe.net', max:10 }});
          * } catch (e) {
          *     // The query was not successful
          * }
@@ -105,7 +107,7 @@ declare namespace _default {
          * // archived messages. Please note, when calling these methods, pass in an integer
          * // to limit your results.
          *
-         * const options = {'with': 'john@doe.net', 'max':10};
+         * const options = { mam: { with: 'john@doe.net' }, rsm: { max:10 }};
          * let result;
          * try {
          *     result = await api.archive.query(options);
@@ -117,7 +119,13 @@ declare namespace _default {
          *
          * while (!result.complete) {
          *     try {
-         *         result = await api.archive.query(Object.assign(options, rsm.next(10).query));
+         *         result = await api.archive.query({
+         *             mam: { ...options.mam },
+         *             rsm: {
+         *                 ...options.rsm,
+         *                 ...rsm.next(10).query
+         *                 }
+         *             });
          *     } catch (e) {
          *         // The query was not successful
          *     }
@@ -135,7 +143,7 @@ declare namespace _default {
          * // message, pass in the `before` parameter with an empty string value `''`.
          *
          * let result;
-         * const options = {'before': '', 'max':5};
+         * const options = { rsm: { before: '', max:5 }};
          * try {
          *     result = await api.archive.query(options);
          * } catch (e) {
@@ -146,7 +154,14 @@ declare namespace _default {
          *
          * // Now we query again, to get the previous batch.
          * try {
-         *      result = await api.archive.query(Object.assign(options, rsm.previous(5).query));
+         *     try {
+         *         result = await api.archive.query({
+         *             mam: { ...options.mam },
+         *             rsm: {
+         *                 ...options.rsm,
+         *                 ...rsm.previous(5).query
+         *                 }
+         *             });
          * } catch (e) {
          *     // The query was not successful
          * }
@@ -154,7 +169,7 @@ declare namespace _default {
          * result.messages.forEach(m => this.showMessage(m));
          *
          */
-        function query(options: import("./types").ArchiveQueryOptions): Promise<import("./types").MAMQueryResult>;
+        function query(options?: import("./types").ArchiveQueryOptions): Promise<import("./types").MAMQueryResult>;
     }
 }
 export default _default;

src/headless/types/plugins/mam/placeholder.d.ts

@@ -3,6 +3,7 @@ export default class MAMPlaceholderMessage extends Model {
         msgid: string;
         is_ephemeral: boolean;
     };
+    fetchMissingMessages(): Promise<void>;
 }
-import { Model } from '@converse/skeletor';
+import { Model } from "@converse/skeletor";
 //# sourceMappingURL=placeholder.d.ts.map

src/headless/types/plugins/mam/types.d.ts

@@ -1,26 +1,21 @@
 import { RSM } from '../../shared/rsm';
-export type MAMOptions = {
-    max?: number;
-    after?: string;
-    before?: string;
+export type MAMQueryOptions = {
     end?: string;
     start?: string;
     with?: string;
-    groupchat?: boolean;
 };
-type RSMQueryParameters = {
+type RSMQueryOptions = {
     after?: string;
     before?: string;
     index?: number;
     max?: number;
 };
-export type MAMFilterParameters = RSMQueryParameters & {
-    end?: string;
-    start?: string;
-    with?: string;
+export type FetchArchivedMessagesOptions = {
+    mam?: MAMQueryOptions;
+    rsm?: RSMQueryOptions;
 };
-export type ArchiveQueryOptions = MAMFilterParameters & {
-    groupchat?: boolean;
+export type ArchiveQueryOptions = FetchArchivedMessagesOptions & {
+    is_groupchat?: boolean;
 };
 export type MAMQueryResult = {
     messages: any[];

src/headless/types/plugins/mam/utils.d.ts

@@ -1,7 +1,8 @@
 /**
+ * @param {Element|Error} e
  * @param {Element} iq
  */
-export function onMAMError(iq: Element): Promise<void>;
+export function onMAMError(e: Element | Error, iq: Element): Promise<void>;
 /**
  * Handle returned IQ stanza containing Message Archive
  * Management (XEP-0313) preferences.
@@ -24,31 +25,36 @@ export function getMAMPrefsFromFeature(feature: Model): void;
 /**
  * @param {MUC} muc
  */
-export function preMUCJoinMAMFetch(muc: MUC): void;
+export function preMUCJoinMAMFetch(muc: MUC): Promise<void>;
 /**
  * @param {ChatBox|MUC} model
  * @param {Object} result
  * @param {Object} query
  * @param {Object} options
- * @param {('forwards'|'backwards'|null)} [should_page=null]
+ * @param {('forwards'|'backwards'|false)} [should_page=false]
  */
-export function handleMAMResult(model: ChatBox | MUC, result: any, query: any, options: any, should_page?: ("forwards" | "backwards" | null)): Promise<void>;
+export function handleMAMResult(model: ChatBox | MUC, result: any, query: any, options: any, should_page?: ("forwards" | "backwards" | false)): Promise<void>;
 /**
  * Fetch XEP-0313 archived messages based on the passed in criteria.
  * @param {ChatBox|MUC} model
- * @param {import('./types').MAMOptions} [options]
- * @param {('forwards'|'backwards'|null)} [should_page=null] - Determines whether
+ * @param {import('./types').FetchArchivedMessagesOptions} [options]
+ * @param {('forwards'|'backwards'|false)} [should_page=false] - Determines whether
  *  this function should recursively page through the entire result set if a limited
  *  number of results were returned.
  */
-export function fetchArchivedMessages(model: ChatBox | MUC, options?: import("./types").MAMOptions, should_page?: ("forwards" | "backwards" | null)): Promise<void>;
+export function fetchArchivedMessages(model: ChatBox | MUC, options?: import("./types").FetchArchivedMessagesOptions, should_page?: ("forwards" | "backwards" | false)): Promise<void>;
+/**
+ * Creates a placeholder to fetch messages at the top of the chat history.
+ * @param {ChatBox|MUC} model
+ */
+export function createScrollupPlaceholder(model: ChatBox | MUC): void;
 /**
  * Fetches messages that might have been archived *after*
  * the last archived message in our local cache.
  * @param {ChatBox|MUC} model
  */
-export function fetchNewestMessages(model: ChatBox | MUC): void;
-export type MUC = import("../muc/muc.js").default;
-export type ChatBox = import("../chat/model.js").default;
+export function fetchNewestMessages(model: ChatBox | MUC): Promise<void>;
+export type MUC = import("../muc/muc").default;
+export type ChatBox = import("../chat/model").default;
 export type Model = import("@converse/skeletor/src/types/helpers.js").Model;
 //# sourceMappingURL=utils.d.ts.map

src/headless/types/shared/parsers.d.ts

@@ -1,11 +1,11 @@
 /**
- * @param {Element|Error} stanza - The stanza to be parsed. As a convenience,
+ * @param {Element|Error|null} stanza - The stanza to be parsed. As a convenience,
  * an Error element can be passed in as well, so that this function can be
  * called in a catch block without first checking if a stanza or Error
  * element was received.
  * @returns {Promise<Error|errors.StanzaError|null>}
  */
-export function parseErrorStanza(stanza: Element | Error): Promise<Error | errors.StanzaError | null>;
+export function parseErrorStanza(stanza: Element | Error | null): Promise<Error | errors.StanzaError | null>;
 /**
  * Extract the XEP-0359 stanza IDs from the passed in stanza
  * and return a map containing them.

src/headless/types/shared/rsm.d.ts

@@ -27,7 +27,7 @@ export class RSM {
     result: {};
     /**
      * Returns a `<set>` XML element that confirms to XEP-0059 Result Set Management.
-     * The element is constructed based on the RSMQueryParameters
+     * The element is constructed based on the RSMQueryOptions
      * that are set on this RSM instance.
      * @returns {Element}
      */

src/headless/types/shared/types.d.ts

@@ -6,6 +6,11 @@ export interface ModelOptions {
     unset?: boolean;
     silent?: boolean;
 }
+export type RSMResult = {
+    count?: string;
+    first?: string;
+    last?: string;
+};
 type Constructor<T = {}> = new (...args: any[]) => T;
 export type ModelExtender = Constructor<Model>;
 type EncryptionPayloadAttrs = {

src/plugins/chatview/tests/retractions.js

@@ -1,6 +1,5 @@
 /*global mock, converse */
-
-const { Strophe, u, stx, dayjs } = converse.env;
+const { Strophe, u, stx, dayjs, sizzle } = converse.env;
 
 describe('A sent chat message', function () {
     beforeAll(() => jasmine.addMatchers({ toEqualStanza: jasmine.toEqualStanza }));
@@ -211,7 +210,7 @@ describe('A message retraction', function () {
             await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
             const stanza = await u.waitUntil(() =>
-                sent_IQs.filter((iq) => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop()
+                sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop()
             );
             const queryid = stanza.querySelector('query').getAttribute('queryid');
             const view = _converse.chatboxviews.get(contact_jid);
@@ -262,7 +261,7 @@ describe('A message retraction', function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>

src/plugins/mam-views/index.js

@@ -1,18 +1,19 @@
 /**
- * @description UI code XEP-0313 Message Archive Management
- * @copyright 2021, the Converse.js contributors
+ * @description UI code for XEP-0313 Message Archive Management
  * @license Mozilla Public License (MPLv2)
  */
 import './placeholder.js';
 import { api, converse } from '@converse/headless';
-import { fetchMessagesOnScrollUp, getPlaceholderTemplate } from './utils.js';
+import { getPlaceholderTemplate } from './utils.js';
 
 
 converse.plugins.add('converse-mam-views', {
     dependencies: ['converse-mam', 'converse-chatview', 'converse-muc-views'],
 
     initialize () {
-        api.listen.on('chatBoxScrolledUp', fetchMessagesOnScrollUp);
+        api.settings.extend({
+            auto_fill_history_gaps: true,
+        });
         api.listen.on('getMessageTemplate', getPlaceholderTemplate);
     }
 });

src/plugins/mam-views/placeholder.js

@@ -1,37 +1,50 @@
-import { api, u } from "@converse/headless";
-import { CustomElement } from 'shared/components/element.js';
-import tplPlaceholder from './templates/placeholder.js';
+import { api } from "@converse/headless";
+import { ObservableElement } from "shared/components/observable.js";
+import tplPlaceholder from "./templates/placeholder.js";
 
-import './styles/placeholder.scss';
+import "./styles/placeholder.scss";
 
+class Placeholder extends ObservableElement {
+    /**
+     * @typedef {import('shared/components/types').ObservableProperty} ObservableProperty
+     */
 
-class Placeholder extends CustomElement {
-
-    static get properties () {
+    static get properties() {
         return {
-            'model': { type: Object }
-        }
+            ...super.properties,
+            model: { type: Object },
+        };
     }
 
-    constructor () {
+    constructor() {
         super();
         this.model = null;
+        this.observable = /** @type {ObservableProperty} */ ("once");
+        this.intersectionRatio = 0.1;
     }
 
-    render () {
+    render() {
         return tplPlaceholder(this);
     }
 
-    async fetchMissingMessages (ev) {
+    /**
+     * @param {Event} [ev]
+     */
+    fetchMissingMessages(ev) {
         ev?.preventDefault?.();
-        this.model.set('fetching', true);
-        const options = {
-            'before': this.model.get('before'),
-            'start': this.model.get('start')
+        this.model.fetchMissingMessages();
+    }
+
+    /**
+     * @param {IntersectionObserverEntry} _entry
+     */
+    onVisibilityChanged(_entry) {
+        if (api.settings.get("auto_fill_history_gaps") && this.isVisible && !this.model.get("fetching")) {
+            this.fetchMissingMessages();
         }
-        await u.mam.fetchArchivedMessages(this.model.collection.chatbox, options);
-        this.model.destroy();
     }
 }
 
-api.elements.define('converse-mam-placeholder', Placeholder);
+api.elements.define("converse-mam-placeholder", Placeholder);
+
+export default Placeholder;

src/plugins/mam-views/styles/placeholder.scss

@@ -1,31 +1,39 @@
-converse-mam-placeholder {
-    .mam-placeholder {
-        position: relative;
-        height: 2em;
-        margin: 0.5em 0;
-        &:before,
-        &:after {
-            content: "";
-            display: block;
-            position: absolute;
-            left: 0;
-            right: 0;
-        }
-        &:before {
+.conversejs {
+    converse-mam-placeholder {
+        .spinner-grow {
             height: 1em;
-            top: 1em;
-            background: linear-gradient(-135deg, lightgray 0.5em, transparent 0) 0 0.5em, linear-gradient( 135deg, lightgray 0.5em, transparent 0) 0 0.5em;
-            background-position: top left;
-            background-repeat: repeat-x;
-            background-size: 1em 1em;
+            width: 1em;
+            margin: 1em;
         }
-        &:after {
-            height: 1em;
-            top: 0.75em;
-            background: linear-gradient(-135deg, var(--background-color) 0.5em, transparent 0) 0 0.5em, linear-gradient( 135deg, var(--background-color) 0.5em, transparent 0) 0 0.5em;
-            background-position: top left;
-            background-repeat: repeat-x;
-            background-size: 1em 1em;
+
+        .mam-placeholder {
+            position: relative;
+            height: 2em;
+            margin: 0.5em 0;
+            &:before,
+            &:after {
+                content: "";
+                display: block;
+                position: absolute;
+                left: 0;
+                right: 0;
+            }
+            &:before {
+                height: 1em;
+                top: 1em;
+                background: linear-gradient(-135deg, lightgray 0.5em, transparent 0) 0 0.5em, linear-gradient( 135deg, lightgray 0.5em, transparent 0) 0 0.5em;
+                background-position: top left;
+                background-repeat: repeat-x;
+                background-size: 1em 1em;
+            }
+            &:after {
+                height: 1em;
+                top: 0.75em;
+                background: linear-gradient(-135deg, var(--background-color) 0.5em, transparent 0) 0 0.5em, linear-gradient( 135deg, var(--background-color) 0.5em, transparent 0) 0 0.5em;
+                background-position: top left;
+                background-repeat: repeat-x;
+                background-size: 1em 1em;
+            }
         }
     }
 }

src/plugins/mam-views/templates/placeholder.js

@@ -1,10 +1,14 @@
-import tplSpinner from 'templates/spinner.js';
-import { __ } from 'i18n';
-import { html } from 'lit/html.js';
+import { html } from "lit/html.js";
+import { __ } from "i18n";
+import tplSpinner from "templates/spinner.js";
 
+/**
+ * @param {import('../placeholder').default} el
+ */
 export default (el) => {
-    return el.model.get('fetching') ? tplSpinner({'classes': 'hor_centered'}) :
-        html`<a @click="${(ev) => el.fetchMissingMessages(ev)}" title="${__('Click to load missing messages')}">
-            <div class="message mam-placeholder"></div>
-        </a>`;
-}
+    return el.model.get("fetching")
+        ? tplSpinner({ classes: "hor_centered" })
+        : html`<a @click="${(ev) => el.fetchMissingMessages(ev)}" title="${__('Click to load missing messages')}">
+              <div class="message mam-placeholder"></div>
+          </a>`;
+};

src/plugins/mam-views/tests/placeholder.js

@@ -1,24 +1,27 @@
 /*global mock, converse */
-
-const { Strophe, u } = converse.env;
+const { Strophe, sizzle, u } = converse.env;
 
 describe("Message Archive Management", function () {
 
+    beforeAll(() => jasmine.addMatchers({ toEqualStanza: jasmine.toEqualStanza }));
+
     describe("A placeholder message", function () {
 
         it("is created to indicate a gap in the history",
             mock.initConverse(
                 ['discoInitialized'],
                 {
-                    'archived_messages_page_size': 2,
-                    'persistent_store': 'localStorage',
-                    'mam_request_all_pages': false
+                    auto_fill_history_gaps: false,
+                    archived_messages_page_size: 2,
+                    persistent_store: 'localStorage',
+                    mam_request_all_pages: false
                 },
                 async function (_converse) {
 
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
             const muc_jid = 'orchard@chat.shakespeare.lit';
             const msgid = u.getUniqueId();
+            const conn = _converse.api.connection.get();
 
             // We put an already cached message in localStorage
             const key_prefix = `converse-test-persistent/${_converse.bare_jid}`;
@@ -49,12 +52,12 @@ describe("Message Archive Management", function () {
             await mock.openAndEnterMUC(_converse, muc_jid, 'romeo');
             const view = _converse.chatboxviews.get(muc_jid);
 
-            let iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
-            const first_msg_id = _converse.api.connection.get().getUniqueId();
-            const second_msg_id = _converse.api.connection.get().getUniqueId();
-            const third_msg_id = _converse.api.connection.get().getUniqueId();
-            let message = u.toStanza(
-                `<message xmlns="jabber:client"
+            let iq_get = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
+            const first_msg_id = conn.getUniqueId();
+            const second_msg_id = conn.getUniqueId();
+            const third_msg_id = conn.getUniqueId();
+            let message = stx`
+                <message xmlns="jabber:client"
                         to="romeo@montague.lit/orchard"
                         from="${muc_jid}">
                     <result xmlns="urn:xmpp:mam:2" queryid="${iq_get.querySelector('query').getAttribute('queryid')}" id="${second_msg_id}">
@@ -65,11 +68,11 @@ describe("Message Archive Management", function () {
                             </message>
                         </forwarded>
                     </result>
-                </message>`);
-            _converse.api.connection.get()._dataRecv(mock.createRequest(message));
+                </message>`;
+            conn._dataRecv(mock.createRequest(message));
 
-            message = u.toStanza(
-                `<message xmlns="jabber:client"
+            message = stx`
+                <message xmlns="jabber:client"
                         to="romeo@montague.lit/orchard"
                         from="${muc_jid}">
                     <result xmlns="urn:xmpp:mam:2" queryid="${iq_get.querySelector('query').getAttribute('queryid')}" id="${third_msg_id}">
@@ -80,14 +83,14 @@ describe("Message Archive Management", function () {
                             </message>
                         </forwarded>
                     </result>
-                </message>`);
-            _converse.api.connection.get()._dataRecv(mock.createRequest(message));
+                </message>`;
+            conn._dataRecv(mock.createRequest(message));
 
             // Clear so that we don't match the older query
             while (sent_IQs.length) { sent_IQs.pop(); }
 
-            let result = u.toStanza(
-                `<iq type='result' id='${iq_get.getAttribute('id')}'>
+            let result = stx`
+                <iq type='result' id='${iq_get.getAttribute('id')}' xmlns="jabber:client">
                     <fin xmlns='urn:xmpp:mam:2'>
                         <set xmlns='http://jabber.org/protocol/rsm'>
                             <first index='0'>${second_msg_id}</first>
@@ -95,34 +98,40 @@ describe("Message Archive Management", function () {
                             <count>3</count>
                         </set>
                     </fin>
-                </iq>`);
-            _converse.api.connection.get()._dataRecv(mock.createRequest(result));
-            await u.waitUntil(() => view.model.messages.length === 4);
-
-            const msg = view.model.messages.at(1);
-            expect(msg instanceof _converse.MAMPlaceholderMessage).toBe(true);
-            expect(msg.get('time')).toBe('2021-06-15T11:18:22.999Z');
-
-            const placeholder_el = view.querySelector('converse-mam-placeholder');
+                </iq>`;
+            conn._dataRecv(mock.createRequest(result));
+            await u.waitUntil(() => view.model.messages.length === 5);
+
+            expect(view.model.messages.at(0) instanceof _converse.MAMPlaceholderMessage).toBe(true);
+            expect(view.model.messages.at(0).get('time')).toBe('2021-06-15T11:17:15.423Z');
+            expect(view.model.messages.at(1).get('body')).toBe('existing cached message');
+            expect(view.model.messages.at(2) instanceof _converse.MAMPlaceholderMessage).toBe(true);
+            expect(view.model.messages.at(2).get('time')).toBe('2021-06-15T11:18:22.999Z');
+            expect(view.model.messages.at(3).get('body')).toBe('2nd MAM Message');
+            expect(view.model.messages.at(4).get('body')).toBe('3rd MAM Message');
+
+            const placeholder_el = [...view.querySelectorAll('converse-mam-placeholder')].pop();
             placeholder_el.firstElementChild.click();
+
             await u.waitUntil(() => view.querySelector('converse-mam-placeholder .spinner-grow'));
 
-            iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
-            expect(Strophe.serialize(iq_get)).toBe(
-                `<iq id="${iq_get.getAttribute('id')}" to="${muc_jid}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${iq_get.querySelector('query').getAttribute('queryid')}" xmlns="urn:xmpp:mam:2">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE"><value>urn:xmpp:mam:2</value></field>`+
-                            `<field var="start"><value>2021-06-15T11:17:15.424Z</value></field>`+
-                        `</x>`+
-                        `<set xmlns="http://jabber.org/protocol/rsm"><before>${view.model.messages.at(2).get(`stanza_id ${muc_jid}`)}</before>`+
-                        `<max>2</max>`+
-                    `</set>`+
-                    `</query>`+
-                `</iq>`);
-
-            message = u.toStanza(
-                `<message xmlns="jabber:client"
+            iq_get = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`iq query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
+            expect(iq_get).toEqualStanza(
+                stx`<iq id="${iq_get.getAttribute('id')}" to="${muc_jid}" type="set" xmlns="jabber:client">
+                    <query queryid="${iq_get.querySelector('query').getAttribute('queryid')}" xmlns="urn:xmpp:mam:2">
+                        <x xmlns="jabber:x:data" type="submit">
+                            <field type="hidden" var="FORM_TYPE"><value>urn:xmpp:mam:2</value></field>
+                            <field var="start"><value>2021-06-15T11:17:15.424Z</value></field>
+                        </x>
+                        <set xmlns="http://jabber.org/protocol/rsm">
+                            <before>${second_msg_id}</before>
+                            <max>2</max>
+                        </set>
+                    </query>
+                </iq>`);
+
+            message = stx`
+                <message xmlns="jabber:client"
                         to="romeo@montague.lit/orchard"
                         from="${muc_jid}">
                     <result xmlns="urn:xmpp:mam:2" queryid="${iq_get.querySelector('query').getAttribute('queryid')}" id="${first_msg_id}">
@@ -133,14 +142,14 @@ describe("Message Archive Management", function () {
                             </message>
                         </forwarded>
                     </result>
-                </message>`);
-            _converse.api.connection.get()._dataRecv(mock.createRequest(message));
+                </message>`;
+            conn._dataRecv(mock.createRequest(message));
 
             // Clear so that we don't match the older query
             while (sent_IQs.length) { sent_IQs.pop(); }
 
-            result = u.toStanza(
-                `<iq type='result' id='${iq_get.getAttribute('id')}'>
+            result = stx`
+                <iq type='result' id='${iq_get.getAttribute('id')}' xmlns="jabber:client">
                     <fin xmlns='urn:xmpp:mam:2' complete='true'>
                         <set xmlns='http://jabber.org/protocol/rsm'>
                             <first index='0'>${first_msg_id}</first>
@@ -148,13 +157,13 @@ describe("Message Archive Management", function () {
                             <count>1</count>
                         </set>
                     </fin>
-                </iq>`);
-            _converse.api.connection.get()._dataRecv(mock.createRequest(result));
-            await u.waitUntil(() => view.model.messages.length === 4);
-            await u.waitUntil(() => view.querySelector('converse-mam-placeholder') === null);
+                </iq>`;
+            conn._dataRecv(mock.createRequest(result));
+            await u.waitUntil(() => view.model.messages.length === 5);
+            await u.waitUntil(() => view.querySelectorAll('converse-mam-placeholder').length === 1);
         }));
 
-        it("is not created when there isn't a gap because the cached history is empty",
+        it("is not created when the full RSM result set is returned",
                 mock.initConverse(['discoInitialized'], {'archived_messages_page_size': 2},
                 async function (_converse) {
 
@@ -162,56 +171,55 @@ describe("Message Archive Management", function () {
             const muc_jid = 'orchard@chat.shakespeare.lit';
             await mock.openAndEnterMUC(_converse, muc_jid, 'romeo');
             const view = _converse.chatboxviews.get(muc_jid);
-            const iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const iq_get = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
 
             const first_msg_id = _converse.api.connection.get().getUniqueId();
             const last_msg_id = _converse.api.connection.get().getUniqueId();
-            let message = u.toStanza(
-                `<message xmlns="jabber:client"
+            let message = stx`
+                <message xmlns="jabber:client"
                         to="romeo@montague.lit/orchard"
                         from="${muc_jid}">
                     <result xmlns="urn:xmpp:mam:2" queryid="${iq_get.querySelector('query').getAttribute('queryid')}" id="${first_msg_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2018-01-09T06:15:23Z"/>
                             <message from="${muc_jid}/some1" type="groupchat">
-                                <body>2nd Message</body>
+                                <body>First Message</body>
                             </message>
                         </forwarded>
                     </result>
-                </message>`);
+                </message>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(message));
 
-            message = u.toStanza(
-                `<message xmlns="jabber:client"
+            message = stx`
+                <message xmlns="jabber:client"
                         to="romeo@montague.lit/orchard"
                         from="${muc_jid}">
                     <result xmlns="urn:xmpp:mam:2" queryid="${iq_get.querySelector('query').getAttribute('queryid')}" id="${last_msg_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2018-01-09T06:16:23Z"/>
                             <message from="${muc_jid}/some1" type="groupchat">
-                                <body>3rd Message</body>
+                                <body>Second Message</body>
                             </message>
                         </forwarded>
                     </result>
-                </message>`);
+                </message>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(message));
 
             // Clear so that we don't match the older query
             while (sent_IQs.length) { sent_IQs.pop(); }
 
-            const result = u.toStanza(
-                `<iq type='result' id='${iq_get.getAttribute('id')}'>
-                    <fin xmlns='urn:xmpp:mam:2'>
+            const result = stx`
+                <iq type='result' id='${iq_get.getAttribute('id')}' xmlns="jabber:client">
+                    <fin xmlns='urn:xmpp:mam:2' complete='true'>
                         <set xmlns='http://jabber.org/protocol/rsm'>
                             <first index='0'>${first_msg_id}</first>
                             <last>${last_msg_id}</last>
-                            <count>3</count>
                         </set>
                     </fin>
-                </iq>`);
+                </iq>`;
             _converse.api.connection.get()._dataRecv(mock.createRequest(result));
             await u.waitUntil(() => view.model.messages.length === 2);
-            expect(true).toBe(true);
+            expect(view.model.messages.pluck('id').join(',')).toBe(`${first_msg_id},${last_msg_id}`);
         }));
     });
 });

src/plugins/muc-views/tests/deprecated-retractions.js

@@ -1,5 +1,5 @@
 /*global mock, converse */
-const { Strophe, u, stx } = converse.env;
+const { Strophe, u, stx, sizzle } = converse.env;
 
 describe("Deprecated Message Retractions", function () {
     beforeAll(() => jasmine.addMatchers({ toEqualStanza: jasmine.toEqualStanza }));
@@ -376,7 +376,7 @@ describe("Deprecated Message Retractions", function () {
             await mock.openChatBoxFor(_converse, contact_jid);
             await mock.waitUntilDiscoConfirmed(_converse, _converse.bare_jid, null, [Strophe.NS.MAM]);
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-            const stanza = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const stanza = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
             const queryid = stanza.querySelector('query').getAttribute('queryid');
             const view = _converse.chatboxviews.get(contact_jid);
             const first_id = u.getUniqueId();
@@ -428,7 +428,7 @@ describe("Deprecated Message Retractions", function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>
@@ -465,13 +465,13 @@ describe("Deprecated Message Retractions", function () {
             const view = _converse.chatboxviews.get(muc_jid);
 
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-            const stanza = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const stanza = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
             const queryid = stanza.querySelector('query').getAttribute('queryid');
 
             const first_id = u.getUniqueId();
             const tombstone = stx`
                 <message id="${u.getUniqueId()}" to="${_converse.jid}" from="${muc_jid}" xmlns="jabber:client">
-                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="stanza-id">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="${first_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}/eve" to="${_converse.bare_jid}" id="message-id-1">
@@ -502,7 +502,7 @@ describe("Deprecated Message Retractions", function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>
@@ -542,13 +542,13 @@ describe("Deprecated Message Retractions", function () {
             const view = _converse.chatboxviews.get(muc_jid);
 
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-            const stanza = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const stanza = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
             const queryid = stanza.querySelector('query').getAttribute('queryid');
 
             const first_id = u.getUniqueId();
             const tombstone = stx`
                 <message id="${u.getUniqueId()}" to="${_converse.jid}" from="${muc_jid}" xmlns="jabber:client">
-                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="stanza-id">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="${first_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}/eve" to="${_converse.bare_jid}" id="message-id-1">
@@ -570,7 +570,7 @@ describe("Deprecated Message Retractions", function () {
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}" to="${_converse.bare_jid}" id="retract-message-1">
-                                <apply-to id="stanza-id" xmlns="urn:xmpp:fasten:0">
+                                <apply-to id="${first_id}" xmlns="urn:xmpp:fasten:0">
                                     <moderated by="${muc_jid}/bob" xmlns='urn:xmpp:message-moderate:0'>
                                         <retract xmlns="urn:xmpp:message-retract:0"/>
                                         <reason>This message contains inappropriate content</reason>
@@ -584,7 +584,7 @@ describe("Deprecated Message Retractions", function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>

src/plugins/muc-views/tests/mam.js

@@ -1,13 +1,97 @@
 /*global mock, converse */
+const { Strophe, stx, u, sizzle } = converse.env;
 
-const { Strophe, stx } = converse.env;
-const u = converse.env.utils;
+describe("MAM archived messages", function () {
 
-describe("A MAM archived message", function () {
+    beforeAll(() => jasmine.addMatchers({ toEqualStanza: jasmine.toEqualStanza }));
 
-    it("will appear in the correct order",
-            mock.initConverse([], {}, async function (_converse) {
+    it("will be fetched newest first and will automatically fetch again if the placeholder message becomes visible",
+
+    mock.initConverse([], {
+        archived_messages_page_size: 3,
+    }, async function (_converse) {
+        const { api } = _converse;
+        const nick = 'romeo';
+        const muc_jid = 'room@muc.example.com';
+        await mock.openAndEnterMUC(_converse, muc_jid, nick);
+
+        const conn = api.connection.get();
+        let iq = await u.waitUntil(() => conn.IQ_stanzas.filter((iq) => sizzle('query[xmlns="urn:xmpp:mam:2"]', iq).length).pop());
+        let iq_id = iq.getAttribute('id');
+        let query_id = iq.querySelector('query').getAttribute('queryid');
+        expect(iq).toEqualStanza(
+            stx`<iq type="set" to="${muc_jid}" xmlns="jabber:client" id="${iq_id}">
+                <query xmlns="urn:xmpp:mam:2" queryid="${query_id}">
+                    <set xmlns="http://jabber.org/protocol/rsm"><before></before><max>3</max></set>
+                </query>
+            </iq>`);
+
+        [
+            stx`<message to="${conn.jid}" from="${muc_jid}" xmlns="jabber:server">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${query_id}" id="9fe1a9d9-c979-488c-93a4-8a3c4dcbc63e">
+                        <forwarded xmlns="urn:xmpp:forward:0">
+                            <delay xmlns="urn:xmpp:delay" stamp="2024-10-13T17:51:20Z"/>
+                            <message xmlns="jabber:client" xml:lang="en" from="${muc_jid}/dadmin" type="groupchat" id="bc4caee0-380a-4f08-b20b-9015177a95bb">
+                                <body>first message</body>
+                            </message>
+                        </forwarded>
+                    </result>
+                </message>`,
+
+            stx`<message to="${conn.jid}" from="${muc_jid}" xmlns="jabber:server">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${query_id}" id="64f68d52-76e6-4fa6-93ef-9fbf96bb237b">
+                        <forwarded xmlns="urn:xmpp:forward:0">
+                            <delay xmlns="urn:xmpp:delay" stamp="2024-10-13T17:51:25Z"/>
+                            <message xmlns="jabber:client" xml:lang="en" from="${muc_jid}/dadmin" type="groupchat" id="7aae4842-6a8b-4a10-a9c4-47cc408650ef">
+                                <body>2nd message</body>
+                            </message>
+                        </forwarded>
+                    </result>
+                </message>`,
+
+            stx`<message to="${conn.jid}" from="${muc_jid}" xmlns="jabber:server">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${query_id}" id="c2c07703-b285-4529-a4b4-12594f749c58">
+                        <forwarded xmlns="urn:xmpp:forward:0">
+                            <delay xmlns="urn:xmpp:delay" stamp="2024-10-13T17:52:34Z"/>
+                            <message xmlns="jabber:client" from="${muc_jid}/dadmin" type="groupchat" id="hDs1J0QHfimjggw2">
+                                <body>3rd message</body>
+                            </message>
+                        </forwarded>
+                    </result>
+                </message>`,
+        ].forEach((stanza) => conn._dataRecv(mock.createRequest(stanza)));
+
+        while (conn.IQ_stanzas.length) conn.IQ_stanzas.pop();
+
+        conn._dataRecv(mock.createRequest(stx`
+            <iq type="result" id="${iq_id}" xmlns="jabber:client">
+                <fin xmlns="urn:xmpp:mam:2">
+                    <set xmlns="http://jabber.org/protocol/rsm">
+                        <first index="0">9fe1a9d9-c979-488c-93a4-8a3c4dcbc63e</first>
+                        <last>c2c07703-b285-4529-a4b4-12594f749c58</last>
+                    </set>
+                </fin>
+            </iq>`));
+
+        await u.waitUntil(() => _converse.chatboxes.get(muc_jid).messages.length === 4);
+        const messages = [..._converse.chatboxes.get(muc_jid).messages];
+        expect(messages.shift() instanceof _converse.exports.MAMPlaceholderMessage).toBe(true);
+
+        iq = await u.waitUntil(() => conn.IQ_stanzas.filter((iq) => sizzle('query[xmlns="urn:xmpp:mam:2"]', iq).length).pop());
+        iq_id = iq.getAttribute('id');
+        query_id = iq.querySelector('query').getAttribute('queryid');
+        expect(iq).toEqualStanza(
+            stx`<iq type="set" to="${muc_jid}" xmlns="jabber:client" id="${iq_id}">
+                <query xmlns="urn:xmpp:mam:2" queryid="${query_id}">
+                    <set xmlns="http://jabber.org/protocol/rsm">
+                        <before>9fe1a9d9-c979-488c-93a4-8a3c4dcbc63e</before>
+                        <max>3</max>
+                    </set>
+                </query>
+            </iq>`);
+    }));
 
+    it("will appear in the correct order", mock.initConverse([], {}, async function (_converse) {
         const nick = 'romeo';
         const muc_jid = 'room@muc.example.com';
         const model = await mock.openAndEnterMUC(_converse, muc_jid, nick);

src/plugins/muc-views/tests/muc.js

@@ -147,9 +147,9 @@ describe("Groupchats", function () {
 
         it("maintains its state across reloads",
             mock.initConverse([], {
-                    'clear_messages_on_reconnection': true,
-                    'enable_smacks': false
-                }, async function (_converse) {
+                clear_messages_on_reconnection: true,
+                enable_smacks: false
+            }, async function (_converse) {
 
             const { api } = _converse;
             const nick = 'romeo';
@@ -157,7 +157,14 @@ describe("Groupchats", function () {
             const muc_jid = 'lounge@montague.lit'
             await mock.openAndEnterMUC(_converse, muc_jid, nick, [], []);
             const view = _converse.chatboxviews.get(muc_jid);
-            let iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            let iq_get = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
+            expect(iq_get).toEqualStanza(stx`
+                <iq id="${iq_get.getAttribute('id')}" to="${muc_jid}" type="set" xmlns="jabber:client">
+                    <query queryid="${iq_get.querySelector('query').getAttribute('queryid')}" xmlns="${Strophe.NS.MAM}">
+                        <set xmlns="http://jabber.org/protocol/rsm"><before></before><max>50</max></set>
+                    </query>
+                </iq>`);
+
             const first_msg_id = _converse.api.connection.get().getUniqueId();
             const last_msg_id = _converse.api.connection.get().getUniqueId();
             _converse.api.connection.get()._dataRecv(mock.createRequest(
@@ -189,7 +196,7 @@ describe("Groupchats", function () {
             _converse.api.connection.get()._dataRecv(mock.createRequest(message));
 
             const result = stx`<iq type='result' id='${iq_get.getAttribute('id')}' xmlns="jabber:client">
-                    <fin xmlns='urn:xmpp:mam:2'>
+                    <fin xmlns='urn:xmpp:mam:2' complete="true">
                         <set xmlns='http://jabber.org/protocol/rsm'>
                             <first index='0'>${first_msg_id}</first>
                             <last>${last_msg_id}</last>
@@ -233,16 +240,17 @@ describe("Groupchats", function () {
             const all_affiliations = Array.isArray(affs) ? affs :  (affs ? ['member', 'admin', 'owner'] : []);
             await mock.returnMemberLists(_converse, muc_jid, [], all_affiliations);
 
-            iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
-            expect(Strophe.serialize(iq_get)).toBe(
-                `<iq id="${iq_get.getAttribute('id')}" to="${muc_jid}" type="set" xmlns="jabber:client">`+
-                    `<query queryid="${iq_get.querySelector('query').getAttribute('queryid')}" xmlns="${Strophe.NS.MAM}">`+
-                        `<x type="submit" xmlns="jabber:x:data">`+
-                            `<field type="hidden" var="FORM_TYPE"><value>urn:xmpp:mam:2</value></field>`+
-                        `</x>`+
-                        `<set xmlns="http://jabber.org/protocol/rsm"><before></before><max>50</max></set>`+
-                    `</query>`+
-                `</iq>`);
+            iq_get = await u.waitUntil(() => sent_IQs.filter(iq => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
+            expect(iq_get).toEqualStanza(stx`
+                <iq id="${iq_get.getAttribute('id')}" to="${muc_jid}" type="set" xmlns="jabber:client">
+                    <query queryid="${iq_get.querySelector('query').getAttribute('queryid')}" xmlns="${Strophe.NS.MAM}">
+                        <x xmlns="jabber:x:data" type="submit">
+                            <field type="hidden" var="FORM_TYPE"><value>urn:xmpp:mam:2</value></field>
+                            <field var="start"><value>2020-07-14T17:46:47.000Z</value></field>
+                        </x>
+                        <set xmlns="http://jabber.org/protocol/rsm"><before></before><max>50</max></set>
+                    </query>
+                </iq>`);
         }));
 
         it("shows a new messages indicator when you're scrolled up",

src/plugins/muc-views/tests/rai.js

@@ -1,6 +1,5 @@
 /*global mock, converse */
-
-const { Strophe, u, stx } = converse.env;
+const { Strophe, u, stx, sizzle } = converse.env;
 
 // See: https://xmpp.org/rfcs/rfc3921.html
 
@@ -22,7 +21,7 @@ describe("XEP-0437 Room Activity Indicators", function () {
         expect(view.model.get('hidden')).toBe(false);
 
         const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-        const iq_get = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+        const iq_get = await u.waitUntil(() => sent_IQs.filter(iq => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
         const first_msg_id = _converse.api.connection.get().getUniqueId();
         const last_msg_id = _converse.api.connection.get().getUniqueId();
         let message =
@@ -58,7 +57,7 @@ describe("XEP-0437 Room Activity Indicators", function () {
             stx`<iq type="result"
                     id="${iq_get.getAttribute("id")}"
                     xmlns="jabber:client">
-                <fin xmlns="urn:xmpp:mam:2">
+                <fin xmlns="urn:xmpp:mam:2" complete="true">
                     <set xmlns="http://jabber.org/protocol/rsm">
                         <first index="0">${first_msg_id}</first>
                         <last>${last_msg_id}</last>

src/plugins/muc-views/tests/retractions.js

@@ -1,6 +1,5 @@
 /*global mock, converse */
-
-const { Strophe, u, stx } = converse.env;
+const { Strophe, u, stx, sizzle } = converse.env;
 
 async function sendAndThenRetractMessage (_converse, view) {
     view.model.sendMessage({'body': 'hello world'});
@@ -816,13 +815,13 @@ describe("Message Retractions", function () {
             const view = _converse.chatboxviews.get(muc_jid);
 
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-            const stanza = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const stanza = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
             const queryid = stanza.querySelector('query').getAttribute('queryid');
 
             const first_id = u.getUniqueId();
             const tombstone = stx`
                 <message id="${u.getUniqueId()}" to="${_converse.jid}" from="${muc_jid}" xmlns="jabber:client">
-                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="stanza-id">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="${first_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}/eve" to="${_converse.bare_jid}" id="message-id-1">
@@ -852,7 +851,7 @@ describe("Message Retractions", function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>
@@ -892,13 +891,13 @@ describe("Message Retractions", function () {
             const view = _converse.chatboxviews.get(muc_jid);
 
             const sent_IQs = _converse.api.connection.get().IQ_stanzas;
-            const stanza = await u.waitUntil(() => sent_IQs.filter(iq => iq.querySelector(`iq[type="set"] query[xmlns="${Strophe.NS.MAM}"]`)).pop());
+            const stanza = await u.waitUntil(() => sent_IQs.filter((iq) => sizzle(`query[xmlns="${Strophe.NS.MAM}"]`, iq).length).pop());
             const queryid = stanza.querySelector('query').getAttribute('queryid');
 
             const first_id = u.getUniqueId();
             const tombstone = stx`
                 <message id="${u.getUniqueId()}" to="${_converse.jid}" from="${muc_jid}" xmlns="jabber:client">
-                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="stanza-id">
+                    <result xmlns="urn:xmpp:mam:2" queryid="${queryid}" id="${first_id}">
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}/eve" to="${_converse.bare_jid}" id="message-id-1">
@@ -920,7 +919,7 @@ describe("Message Retractions", function () {
                         <forwarded xmlns="urn:xmpp:forward:0">
                             <delay xmlns="urn:xmpp:delay" stamp="2019-09-20T23:08:25Z"/>
                             <message type="groupchat" from="${muc_jid}" to="${_converse.bare_jid}" id="retract-message-1">
-                                <retract id="stanza-id" xmlns='urn:xmpp:message-retract:1'>
+                                <retract id="${first_id}" xmlns='urn:xmpp:message-retract:1'>
                                     <moderated by='room@muc.example.com/macbeth' xmlns='urn:xmpp:message-moderate:1'/>
                                     <reason>This message contains inappropriate content</reason>
                                 </retract>
@@ -932,7 +931,7 @@ describe("Message Retractions", function () {
 
             const iq_result = stx`
                 <iq type="result" id="${stanza.getAttribute('id')}" xmlns="jabber:client">
-                    <fin xmlns="urn:xmpp:mam:2">
+                    <fin xmlns="urn:xmpp:mam:2" complete="true">
                         <set xmlns="http://jabber.org/protocol/rsm">
                             <first index="0">${first_id}</first>
                             <last>${last_id}</last>

src/plugins/muc-views/tests/unfurls.js

@@ -1,6 +1,5 @@
 /*global mock, converse */
-
-const { Strophe, u, stx } = converse.env;
+const { Strophe, u, stx, sizzle } = converse.env;
 
 describe("A Groupchat Message", function () {
 
@@ -423,17 +422,18 @@ describe("A Groupchat Message", function () {
         const message_form = view.querySelector('converse-muc-message-form');
         textarea.value = unfurl_url;
         const enter_event = {
-            'target': textarea,
-            'preventDefault': function preventDefault () {},
-            'stopPropagation': function stopPropagation () {},
-            'keyCode': 13 // Enter
+            target: textarea,
+            preventDefault: function preventDefault () {},
+            stopPropagation: function stopPropagation () {},
+            keyCode: 13 // Enter
         }
         message_form.onKeyDown(enter_event);
 
         await u.waitUntil(() => view.querySelectorAll('.chat-msg').length === 1);
         expect(view.querySelector('.chat-msg__text').textContent).toBe(unfurl_url);
 
-        let msg = _converse.api.connection.get().send.calls.all()[1].args[0];
+        const sent_stanzas = _converse.api.connection.get().sent_stanzas;
+        let msg = await u.waitUntil(() => sent_stanzas.filter(s => s.matches('message')).pop());
         expect(msg).toEqualStanza(stx`
             <message from="${own_jid}" id="${msg.getAttribute('id')}" to="${muc_jid}" type="groupchat" xmlns="jabber:client">
                 <body>${unfurl_url}</body>

src/shared/components/observable.js

@@ -1,11 +1,16 @@
-import { api } from '@converse/headless';
+import { api } from "@converse/headless";
 import { CustomElement } from "./element";
 
+/**
+ * An element which triggers a global `visibilityChanged` event when
+ * it becomes visible in the viewport. The `observable` property needs to be set.
+ */
 export class ObservableElement extends CustomElement {
     static get properties() {
         return {
             ...super.properties,
             observable: { type: String },
+            intersectionRatio: { type: Number },
         };
     }
 
@@ -13,8 +18,6 @@ export class ObservableElement extends CustomElement {
         super();
         this.model = null;
 
-        // Related to IntersectionObserver
-        this.isVisible = false;
         /**
          * The observable property determines the observability of this element.
          * - 'once': an event will be triggered once when the element becomes visible.
@@ -22,9 +25,11 @@ export class ObservableElement extends CustomElement {
          * @type {import('./types').ObservableProperty}
          */
         this.observable = null;
+
+        this.isVisible = false;
         this.observableThresholds = [0.0, 0.25, 0.5, 0.75, 1.0]; // thresholds to check for, every 25%
         this.observableMargin = "0px"; // margin from root element
-        this.observableRatio = 0.5; // wait till at least 50% of the item is visible
+        this.intersectionRatio = 0.5; // wait till at least 50% of the item is visible
         this.observableDelay = 100;
     }
 
@@ -56,20 +61,34 @@ export class ObservableElement extends CustomElement {
         }
     }
 
+    alreadyHandled() {
+        return (this.observable === "once" && this.isVisible);
+    }
+
     /**
      * @param {IntersectionObserverEntry[]} entries
      */
     handleIntersectionCallback(entries) {
+        if (this.alreadyHandled()) return;
+
         for (const entry of entries) {
             const ratio = Number(entry.intersectionRatio.toFixed(2));
-            if (ratio >= this.observableRatio) {
-                this.isVisible = true;
-                api.trigger('visibilityChanged', { el: this, entry });
-
+            if (ratio >= this.intersectionRatio && !this.alreadyHandled()) {
                 if (this.observable === "once") {
                     this.intersectionObserver.disconnect();
                 }
+                this.isVisible = true;
+
+                api.trigger("visibilityChanged", { el: this, entry });
+                this.onVisibilityChanged(entry);
             }
         }
     }
+
+    /**
+     * @param {IntersectionObserverEntry} _entry
+     */
+    onVisibilityChanged(_entry) {
+        // override this method in your subclass
+    }
 }

src/types/plugins/mam-views/placeholder.d.ts

@@ -1,2 +1,24 @@
-export {};
+export default Placeholder;
+declare class Placeholder extends ObservableElement {
+    /**
+     * @typedef {import('shared/components/types').ObservableProperty} ObservableProperty
+     */
+    static get properties(): {
+        model: {
+            type: ObjectConstructor;
+        };
+        observable: {
+            type: StringConstructor;
+        };
+        intersectionRatio: {
+            type: NumberConstructor;
+        };
+    };
+    render(): import("lit").TemplateResult<1>;
+    /**
+     * @param {Event} [ev]
+     */
+    fetchMissingMessages(ev?: Event): void;
+}
+import { ObservableElement } from "shared/components/observable.js";
 //# sourceMappingURL=placeholder.d.ts.map

src/types/plugins/mam-views/templates/placeholder.d.ts

@@ -1,3 +1,3 @@
-declare function _default(el: any): import("lit/html.js").TemplateResult<1>;
+declare function _default(el: import("../placeholder").default): import("lit/html.js").TemplateResult<1>;
 export default _default;
 //# sourceMappingURL=placeholder.d.ts.map

src/types/plugins/muc-views/sidebar-occupant.d.ts

@@ -9,6 +9,9 @@ export default class MUCOccupantListItem extends ObservableElement {
         observable: {
             type: StringConstructor;
         };
+        intersectionRatio: {
+            type: NumberConstructor;
+        };
     };
     muc: any;
     initialize(): Promise<void>;

src/types/plugins/rosterview/contactview.d.ts

@@ -6,6 +6,9 @@ export default class RosterContact extends ObservableElement {
         observable: {
             type: StringConstructor;
         };
+        intersectionRatio: {
+            type: NumberConstructor;
+        };
     };
     initialize(): void;
     render(): import("lit").TemplateResult<1>;

src/types/shared/chat/message.d.ts

@@ -9,6 +9,9 @@ export default class Message extends ObservableElement {
         observable: {
             type: StringConstructor;
         };
+        intersectionRatio: {
+            type: NumberConstructor;
+        };
     };
     model_with_messages: any;
     initialize(): Promise<void>;

src/types/shared/components/observable.d.ts

@@ -1,11 +1,17 @@
+/**
+ * An element which triggers a global `visibilityChanged` event when
+ * it becomes visible in the viewport. The `observable` property needs to be set.
+ */
 export class ObservableElement extends CustomElement {
     static get properties(): {
         observable: {
             type: StringConstructor;
         };
+        intersectionRatio: {
+            type: NumberConstructor;
+        };
     };
     model: any;
-    isVisible: boolean;
     /**
      * The observable property determines the observability of this element.
      * - 'once': an event will be triggered once when the element becomes visible.
@@ -13,16 +19,22 @@ export class ObservableElement extends CustomElement {
      * @type {import('./types').ObservableProperty}
      */
     observable: import("./types").ObservableProperty;
+    isVisible: boolean;
     observableThresholds: number[];
     observableMargin: string;
-    observableRatio: number;
+    intersectionRatio: number;
     observableDelay: number;
     initIntersectionObserver(): void;
     intersectionObserver: IntersectionObserver;
+    alreadyHandled(): boolean;
     /**
      * @param {IntersectionObserverEntry[]} entries
      */
     handleIntersectionCallback(entries: IntersectionObserverEntry[]): void;
+    /**
+     * @param {IntersectionObserverEntry} _entry
+     */
+    onVisibilityChanged(_entry: IntersectionObserverEntry): void;
 }
 import { CustomElement } from "./element";
 //# sourceMappingURL=observable.d.ts.map