From be74ee004ff75101b4d2c203ab1b15288f13f85c Mon Sep 17 00:00:00 2001 From: Sven Neumann Date: Sun, 12 Apr 2026 13:02:33 +0200 Subject: [PATCH 1/2] refactor(structures): optimize message loading with msgFindBefore Builds on #201705 by JuniorRibas. Replaces WAWebChatLoadMessages.loadEarlierMsgs with WAWebDBMessageFindLocal.msgFindBefore when paginating messages until reaching searchOptions.limit in both Chat and Channel, resolving models through MsgStore when the returned entries are not yet serialized. Follow-up fix: in the !fromMeFilter && finite branch of both Chat.fetchMessages and Channel.fetchMessages, include the already-loaded in-memory msgs when constructing merged. The original PR discarded them, causing chats whose requested window consists mostly of recent in-memory messages (with nothing older resolvable via msgFindBefore) to return only the anchor message. dedupeByMsgId keeps the result unique. --- src/structures/Channel.js | 173 +++++++++++++++++++++++++++++++++++--- src/structures/Chat.js | 171 ++++++++++++++++++++++++++++++++++--- 2 files changed, 324 insertions(+), 20 deletions(-) diff --git a/src/structures/Channel.js b/src/structures/Channel.js index 7f4bee9676..f21b8c54e2 100644 --- a/src/structures/Channel.js +++ b/src/structures/Channel.js @@ -350,17 +350,170 @@ class Channel extends Base { let msgs = channel.msgs.getModelsArray().filter(msgFilter); if (searchOptions && searchOptions.limit > 0) { - while (msgs.length < searchOptions.limit) { - const loadedMessages = await window - .require('WAWebChatLoadMessages') - .loadEarlierMsgs(channel); - if (!loadedMessages || !loadedMessages.length) break; - msgs = [...loadedMessages.filter(msgFilter), ...msgs]; - } - - if (msgs.length > searchOptions.limit) { + const msgFindLocal = window.require( + 'WAWebDBMessageFindLocal', + ); + const WAWebMsgKey = window.require('WAWebMsgKey'); + const MsgStore = window.require('WAWebCollections').Msg; + + const findBefore = async (anchorKey, count) => { + if ( + typeof msgFindLocal.msgFindByDirection === + 'function' + ) { + return await msgFindLocal.msgFindByDirection({ + anchor: anchorKey, + count, + direction: 'before', + }); + } + return await msgFindLocal.msgFindBefore({ + anchor: anchorKey, + count, + }); + }; + + const toMsgKey = (id) => { + if (!id) return null; + if (id instanceof WAWebMsgKey) return id; + const s = + typeof id === 'string' + ? id + : id._serialized || id?.toString?.(); + return s ? WAWebMsgKey.fromString(s) : null; + }; + + const toMsgModels = (rawMessages) => { + const out = []; + for (const m of rawMessages) { + if (m && typeof m.serialize === 'function') { + out.push(m); + continue; + } + const serialized = + m?.id?._serialized || + (typeof m === 'string' ? m : null); + let model = + (serialized && MsgStore.get(serialized)) || + (m?.id && + MsgStore.get(m.id._serialized || m.id)) || + null; + if (!model && m && MsgStore.modelClass) { + try { + model = new MsgStore.modelClass(m); + } catch (e) { + model = null; + } + } + if (model) out.push(model); + } + return out; + }; + + const dedupeByMsgId = (arr) => { + const seen = new Set(); + return arr.filter((m) => { + const key = m.id?._serialized; + if (!key || seen.has(key)) return false; + seen.add(key); + return true; + }); + }; + + const limit = searchOptions.limit; + const finite = Number.isFinite(limit); + const fromMeFilter = + searchOptions && searchOptions.fromMe !== undefined; + + if (!fromMeFilter && finite) { + const anchorSerialized = + channel.lastReceivedKey?.toString(); + if (!anchorSerialized) { + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + msgs = msgs.slice(-Math.min(limit, msgs.length)); + } else { + const fetchCount = Math.max(0, limit - 1); + const anchorKey = toMsgKey(anchorSerialized); + const result = await findBefore( + anchorKey, + fetchCount, + ); + const rawMessages = Array.isArray(result) + ? result + : result?.messages || []; + if ( + result?.status === 404 && + (!rawMessages || !rawMessages.length) + ) { + msgs = []; + } else { + let loaded = toMsgModels(rawMessages); + const anchorMsg = + MsgStore.get(anchorSerialized); + let merged = [ + ...loaded, + ...(anchorMsg ? [anchorMsg] : []), + ...msgs, + ]; + merged = merged.filter( + (m) => + !m.isNotification && + m.type !== 'newsletter_notification', + ); + merged.sort((a, b) => (a.t > b.t ? 1 : -1)); + merged = dedupeByMsgId(merged); + msgs = merged.filter(msgFilter); + if (msgs.length > limit) { + msgs = msgs.slice(-limit); + } + } + } + } else { msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); - msgs = msgs.splice(msgs.length - searchOptions.limit); + const batchCap = finite ? limit : 100; + while (msgs.length < limit || !finite) { + const anchor = + msgs[0]?.id || + channel.msgs.getModelsArray()[0]?.id || + channel.lastReceivedKey; + if (!anchor) break; + + const anchorKey = toMsgKey(anchor); + if (!anchorKey) break; + + const need = finite + ? Math.min(batchCap, limit - msgs.length) + : batchCap; + if (need <= 0) break; + + const result = await findBefore(anchorKey, need); + const rawMessages = Array.isArray(result) + ? result + : result?.messages || []; + if (result?.status === 404 || !rawMessages.length) { + break; + } + + const loadedMessages = toMsgModels(rawMessages); + if (!loadedMessages.length) break; + + const prevLen = msgs.length; + msgs = dedupeByMsgId([ + ...loadedMessages.filter(msgFilter), + ...msgs, + ]); + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + + if (msgs.length === prevLen) break; + + if (!finite && loadedMessages.length < need) { + break; + } + } + + if (finite && msgs.length > limit) { + msgs = msgs.slice(-limit); + } } } diff --git a/src/structures/Chat.js b/src/structures/Chat.js index 8cea7c134e..075404a168 100644 --- a/src/structures/Chat.js +++ b/src/structures/Chat.js @@ -223,17 +223,168 @@ class Chat extends Base { let msgs = chat.msgs.getModelsArray().filter(msgFilter); if (searchOptions && searchOptions.limit > 0) { - while (msgs.length < searchOptions.limit) { - const loadedMessages = await window - .require('WAWebChatLoadMessages') - .loadEarlierMsgs(chat, chat.msgs); - if (!loadedMessages || !loadedMessages.length) break; - msgs = [...loadedMessages.filter(msgFilter), ...msgs]; - } - - if (msgs.length > searchOptions.limit) { + const msgFindLocal = window.require( + 'WAWebDBMessageFindLocal', + ); + const WAWebMsgKey = window.require('WAWebMsgKey'); + const MsgStore = window.require('WAWebCollections').Msg; + + const findBefore = async (anchorKey, count) => { + if ( + typeof msgFindLocal.msgFindByDirection === + 'function' + ) { + return await msgFindLocal.msgFindByDirection({ + anchor: anchorKey, + count, + direction: 'before', + }); + } + return await msgFindLocal.msgFindBefore({ + anchor: anchorKey, + count, + }); + }; + + const toMsgKey = (id) => { + if (!id) return null; + if (id instanceof WAWebMsgKey) return id; + const s = + typeof id === 'string' + ? id + : id._serialized || id?.toString?.(); + return s ? WAWebMsgKey.fromString(s) : null; + }; + + const toMsgModels = (rawMessages) => { + const out = []; + for (const m of rawMessages) { + if (m && typeof m.serialize === 'function') { + out.push(m); + continue; + } + const serialized = + m?.id?._serialized || + (typeof m === 'string' ? m : null); + let model = + (serialized && MsgStore.get(serialized)) || + (m?.id && + MsgStore.get(m.id._serialized || m.id)) || + null; + if (!model && m && MsgStore.modelClass) { + try { + model = new MsgStore.modelClass(m); + } catch (e) { + model = null; + } + } + if (model) out.push(model); + } + return out; + }; + + const dedupeByMsgId = (arr) => { + const seen = new Set(); + return arr.filter((m) => { + const key = m.id?._serialized; + if (!key || seen.has(key)) return false; + seen.add(key); + return true; + }); + }; + + const limit = searchOptions.limit; + const finite = Number.isFinite(limit); + const fromMeFilter = + searchOptions && searchOptions.fromMe !== undefined; + + if (!fromMeFilter && finite) { + const anchorSerialized = + chat.lastReceivedKey?.toString(); + if (!anchorSerialized) { + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + msgs = msgs.slice(-Math.min(limit, msgs.length)); + } else { + const fetchCount = Math.max(0, limit - 1); + const anchorKey = toMsgKey(anchorSerialized); + const result = await findBefore( + anchorKey, + fetchCount, + ); + const rawMessages = Array.isArray(result) + ? result + : result?.messages || []; + if ( + result?.status === 404 && + (!rawMessages || !rawMessages.length) + ) { + msgs = []; + } else { + let loaded = toMsgModels(rawMessages); + const anchorMsg = + MsgStore.get(anchorSerialized); + let merged = [ + ...loaded, + ...(anchorMsg ? [anchorMsg] : []), + ...msgs, + ]; + merged = merged.filter( + (m) => !m.isNotification, + ); + merged.sort((a, b) => (a.t > b.t ? 1 : -1)); + merged = dedupeByMsgId(merged); + msgs = merged.filter(msgFilter); + if (msgs.length > limit) { + msgs = msgs.slice(-limit); + } + } + } + } else { msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); - msgs = msgs.splice(msgs.length - searchOptions.limit); + const batchCap = finite ? limit : 100; + while (msgs.length < limit || !finite) { + const anchor = + msgs[0]?.id || + chat.msgs.getModelsArray()[0]?.id || + chat.lastReceivedKey; + if (!anchor) break; + + const anchorKey = toMsgKey(anchor); + if (!anchorKey) break; + + const need = finite + ? Math.min(batchCap, limit - msgs.length) + : batchCap; + if (need <= 0) break; + + const result = await findBefore(anchorKey, need); + const rawMessages = Array.isArray(result) + ? result + : result?.messages || []; + if (result?.status === 404 || !rawMessages.length) { + break; + } + + const loadedMessages = toMsgModels(rawMessages); + if (!loadedMessages.length) break; + + const prevLen = msgs.length; + msgs = dedupeByMsgId([ + ...loadedMessages.filter(msgFilter), + ...msgs, + ]); + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + + if (msgs.length === prevLen) break; + + if (!finite && loadedMessages.length < need) { + break; + } + } + + if (finite && msgs.length > limit) { + msgs = msgs.slice(-limit); + } } } From f7a7582ba01da54b856e33d42588713acb968870 Mon Sep 17 00:00:00 2001 From: Sven Neumann Date: Sun, 12 Apr 2026 13:31:25 +0200 Subject: [PATCH 2/2] fix(structures): preserve in-memory msgs on msgFindBefore 404 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When msgFindBefore returns status 404 (anchor message not in local DB), the fast-path in Chat/Channel.fetchMessages was discarding the already-filtered in-memory msgs. Mirror the !anchorSerialized fallback directly above: sort by t and slice to the limit, keeping in-memory messages rather than returning an empty array. Also adds short inline comments to the non-obvious decisions introduced by #201705: the msgFindByDirection fallback, the in-memory msgs merge, and the new 404 fallback — aimed at making upstream review easier. --- src/structures/Channel.js | 8 +++++++- src/structures/Chat.js | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/structures/Channel.js b/src/structures/Channel.js index f21b8c54e2..fc8cf67b26 100644 --- a/src/structures/Channel.js +++ b/src/structures/Channel.js @@ -356,6 +356,7 @@ class Channel extends Base { const WAWebMsgKey = window.require('WAWebMsgKey'); const MsgStore = window.require('WAWebCollections').Msg; + // msgFindByDirection is the newer API; fall back to msgFindBefore on older WA Web versions const findBefore = async (anchorKey, count) => { if ( typeof msgFindLocal.msgFindByDirection === @@ -445,7 +446,11 @@ class Channel extends Base { result?.status === 404 && (!rawMessages || !rawMessages.length) ) { - msgs = []; + // anchor not in local DB — fall back to in-memory msgs (same as the !anchorSerialized branch above) + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + msgs = msgs.slice( + -Math.min(limit, msgs.length), + ); } else { let loaded = toMsgModels(rawMessages); const anchorMsg = @@ -453,6 +458,7 @@ class Channel extends Base { let merged = [ ...loaded, ...(anchorMsg ? [anchorMsg] : []), + // include in-memory msgs so recent (not-yet-persisted) messages aren't dropped ...msgs, ]; merged = merged.filter( diff --git a/src/structures/Chat.js b/src/structures/Chat.js index 075404a168..48a885530d 100644 --- a/src/structures/Chat.js +++ b/src/structures/Chat.js @@ -229,6 +229,7 @@ class Chat extends Base { const WAWebMsgKey = window.require('WAWebMsgKey'); const MsgStore = window.require('WAWebCollections').Msg; + // msgFindByDirection is the newer API; fall back to msgFindBefore on older WA Web versions const findBefore = async (anchorKey, count) => { if ( typeof msgFindLocal.msgFindByDirection === @@ -318,7 +319,11 @@ class Chat extends Base { result?.status === 404 && (!rawMessages || !rawMessages.length) ) { - msgs = []; + // anchor not in local DB — fall back to in-memory msgs (same as the !anchorSerialized branch above) + msgs.sort((a, b) => (a.t > b.t ? 1 : -1)); + msgs = msgs.slice( + -Math.min(limit, msgs.length), + ); } else { let loaded = toMsgModels(rawMessages); const anchorMsg = @@ -326,6 +331,7 @@ class Chat extends Base { let merged = [ ...loaded, ...(anchorMsg ? [anchorMsg] : []), + // include in-memory msgs so recent (not-yet-persisted) messages aren't dropped ...msgs, ]; merged = merged.filter(