Fix messaging pagination and scrolling

backend/api/src/get-private-messages.ts

@@ -100,10 +100,10 @@ export async function getChannelMessages(props: {
   channelId: number
   limit?: number
   id?: number | undefined
+  beforeId?: number | undefined
   userId: string
 }) {
-  // console.log('initial message request', props)
-  const {channelId, limit, id, userId} = props
+  const {channelId, limit, id, beforeId, userId} = props
   const pg = createSupabaseDirectClient()
   const {data, error} = await tryCatch(
     pg.map(
@@ -115,11 +115,12 @@ export async function getChannelMessages(props: {
                    where pumcm.user_id = $2
                      and pumcm.channel_id = $1)
        and ($4 is null or id > $4)
+       and ($5 is null or id < $5)
        and not visibility = 'system_status'
      order by created_time desc
          ${limit ? 'limit $3' : ''}
     `,
-      [channelId, userId, limit, id],
+      [channelId, userId, limit, id, beforeId],
       convertPrivateChatMessage,
     ),
   )

common/src/api/schema.ts

@@ -866,6 +866,7 @@ export const API = (_apiTypeCheck = {
       channelId: z.coerce.number(),
       limit: z.coerce.number(),
       id: z.coerce.number().optional(),
+      beforeId: z.coerce.number().optional(),
     }),
     returns: [] as PrivateChatMessage[],
     summary: 'Retrieve messages for a private channel',

web/hooks/use-private-messages.ts

@@ -21,23 +21,32 @@ export function usePrivateMessages(channelId: number, limit: number, userId: str
     key,
   )
 
-  const fetchMessages = async (id?: number) => {
+  const fetchMessages = async (id?: number, beforeId?: number) => {
     const data = {
       channelId,
       limit,
-      // id filter is useful to pull up new messages (later than the last message in messages),
-      // but since messages can be deleted or edited, we can't rely on the id filter anymore (at least not in the same fashion)
-      // id: id ?? (messages?.length ? max(messages.map((m) => m.id)) : undefined),
+      id,
+      beforeId,
     }
     const newMessages = await api('get-channel-messages', data)
     // console.debug(key, {newMessages, messages, data})
-    setMessages((prevMessages) =>
-      orderBy(
-        uniqBy([...newMessages, ...(prevMessages && id ? prevMessages : [])], (m) => m.id),
-        'createdTime',
-        'desc',
-      ),
-    )
+    if (beforeId) {
+      setMessages((prevMessages) =>
+        orderBy(
+          uniqBy([...(prevMessages ?? []), ...newMessages], (m) => m.id),
+          'createdTime',
+          'desc',
+        ),
+      )
+    } else {
+      setMessages((prevMessages) =>
+        orderBy(
+          uniqBy([...newMessages, ...(prevMessages && id ? prevMessages : [])], (m) => m.id),
+          'createdTime',
+          'desc',
+        ),
+      )
+    }
   }
 
   useEffect(() => {

web/lib/supabase/chat-messages.ts

@@ -1,4 +1,5 @@
 import {ChatMessage, PrivateChatMessage} from 'common/chat-message'
+import {debug} from 'common/logger'
 import {richTextToString} from 'common/util/parse'
 import {HOUR_MS, MINUTE_MS} from 'common/util/time'
 import {forEach, last, uniq} from 'lodash'
@@ -38,54 +39,59 @@ export function updateReactionUI(
 }
 
 export const usePaginatedScrollingMessages = (
-  realtimeMessages: ChatMessage[] | undefined,
-  heightFromTop: number,
+  messages: ChatMessage[] | undefined,
   userId: string | undefined,
+  loadMore?: (oldestMessageId: number) => void,
 ) => {
-  const messagesPerPage = 50
-  const initialScroll = useRef(realtimeMessages === undefined)
+  messages = messages ?? []
+  const initialScroll = useRef(messages === undefined)
   const outerDiv = useRef<HTMLDivElement | null>(null)
   const innerDiv = useRef<HTMLDivElement | null>(null)
   const scrollToOldTop = useRef(false)
-  const [page, setPage] = useState(1)
+  const isLoadingMore = useRef(false)
   const [prevInnerDivHeight, setPrevInnerDivHeight] = useState<number>()
+  const expectedLengthAfterLoad = useRef<number>(0)
   const {ref: topVisibleRef} = useIsVisible(() => {
+    if (loadMore && messages && messages.length > 0 && !isLoadingMore.current) {
+      isLoadingMore.current = true
+      loadMore(messages[messages.length - 1].id)
+    }
     scrollToOldTop.current = true
-    setPage(page + 1)
+    expectedLengthAfterLoad.current = messages.length + 5
   })
 
   const [showMessages, setShowMessages] = useState(false)
-  const messages = useMemo(
-    () => (realtimeMessages ?? []).slice(0, messagesPerPage * page).reverse(),
-    [JSON.stringify(realtimeMessages), page],
-  )
+  useEffect(() => {
+    isLoadingMore.current = false
+  }, [messages?.length])
   useEffect(() => {
     const outerDivHeight = outerDiv?.current?.clientHeight ?? 0
     const innerDivHeight = innerDiv?.current?.clientHeight ?? 0
     const outerDivScrollTop = outerDiv?.current?.scrollTop ?? 0
-    // For the private messages page a tolerance of 0 suffices, but for some reason the tv page requires a tolerance of 43
-    const tolerance = 43
     const difference = prevInnerDivHeight
       ? prevInnerDivHeight - outerDivHeight - outerDivScrollTop
       : 0
-    const isScrolledToBottom = difference <= tolerance
-    if ((!prevInnerDivHeight || isScrolledToBottom || initialScroll.current) && realtimeMessages) {
-      outerDiv?.current?.scrollTo({
-        top: innerDivHeight! - outerDivHeight!,
-        left: 0,
-        behavior: prevInnerDivHeight ? 'smooth' : 'auto',
-      })
-      setShowMessages(true)
-      initialScroll.current = false
-    } else if (scrollToOldTop.current) {
-      // Loaded more messages, scroll to old top
-      const height = innerDivHeight! - prevInnerDivHeight! + heightFromTop
+    const isScrolledToBottom = difference <= 0
+
+    if (scrollToOldTop.current && messages?.length > expectedLengthAfterLoad.current) {
+      // Loaded more messages, scroll to old top position
+      const height = innerDivHeight! - prevInnerDivHeight!
       outerDiv?.current?.scrollTo({
         top: height,
         left: 0,
         behavior: 'auto',
       })
       scrollToOldTop.current = false
+    } else if (!prevInnerDivHeight || isScrolledToBottom || initialScroll.current) {
+      if (messages) {
+        outerDiv?.current?.scrollTo({
+          top: innerDivHeight! - outerDivHeight!,
+          left: 0,
+          behavior: prevInnerDivHeight ? 'smooth' : 'auto',
+        })
+        setShowMessages(true)
+        initialScroll.current = false
+      }
     } else if (last(messages)?.userId === userId) {
       // Sent a message, scroll to bottom
       outerDiv?.current?.scrollTo({
@@ -97,12 +103,14 @@ export const usePaginatedScrollingMessages = (
 
     setPrevInnerDivHeight(innerDivHeight)
   }, [messages])
-  return {topVisibleRef, showMessages, messages, outerDiv, innerDiv}
+  return {topVisibleRef, showMessages, outerDiv, innerDiv}
 }
 
-export const useGroupedMessages = (messages: ChatMessage[]) => {
+export const useGroupedMessages = (messages: ChatMessage[] | undefined) => {
+  messages = messages ?? []
+  messages = messages.slice().reverse()
   // Create a string key that changes when any message's content or reactions change
-  console.log('messages in useGroupedMessages', messages[0]?.reactions)
+  debug('messages in useGroupedMessages', messages[0]?.reactions)
 
   return useMemo(() => {
     // Group messages created within a short time of each other.

web/pages/messages/[channelId].tsx

@@ -109,16 +109,33 @@ export const PrivateChat = (props: {
 
   const totalMessagesToLoad = 100
   const {
-    messages: realtimeMessages,
+    messages: _messages,
     setMessages,
     fetchMessages,
   } = usePrivateMessages(channelId, totalMessagesToLoad, user.id)
-  // console.log('realtimeMessages', realtimeMessages)
+
+  const messages =
+    _messages?.map(
+      (m) =>
+        ({
+          ...m,
+          id: m.id,
+        }) as ChatMessage,
+    ) ?? []
+
+  console.log(messages)
+
+  const loadMoreMessages = useCallback(
+    (beforeId: number) => {
+      fetchMessages(undefined, beforeId)
+    },
+    [fetchMessages],
+  )
 
   const [showUsers, setShowUsers] = useState(false)
   const maxUsersToGet = 100
   const messageUserIds = uniq(
-    (realtimeMessages ?? [])
+    (messages ?? [])
       .filter((message) => message.userId !== user.id)
       .map((message) => message.userId),
   )
@@ -133,16 +150,10 @@ export const PrivateChat = (props: {
   const members = filterDefined(otherUsers?.filter((user) => memberIds.includes(user.id)) ?? [])
   const router = useRouter()
 
-  const {topVisibleRef, showMessages, messages, innerDiv, outerDiv} = usePaginatedScrollingMessages(
-    realtimeMessages?.map(
-      (m) =>
-        ({
-          ...m,
-          id: m.id,
-        }) as ChatMessage,
-    ),
-    200,
+  const {topVisibleRef, showMessages, innerDiv, outerDiv} = usePaginatedScrollingMessages(
+    messages,
     user?.id,
+    loadMoreMessages,
   )
 
   const [editingMessage, setEditingMessage] = useState<ChatMessage | null>(null)
@@ -239,7 +250,7 @@ export const PrivateChat = (props: {
     editor?.commands.focus()
   }, [editor])
 
-  const heightFromTop = 200
+  const SENTINEL_PREFETCH_PX = 1000 // how early to start loading
 
   const [replyToUserInfo, setReplyToUserInfo] = useState<any>()
 
@@ -366,11 +377,15 @@ export const PrivateChat = (props: {
               opacity: showMessages ? 1 : 0,
             }}
           >
-            {realtimeMessages === undefined ? (
+            {messages === undefined ? (
               <CompassLoadingIndicator />
             ) : (
               <>
-                <div className={'absolute h-1 '} ref={topVisibleRef} style={{top: heightFromTop}} />
+                <div
+                  className={'absolute h-1 '}
+                  ref={topVisibleRef}
+                  style={{top: SENTINEL_PREFETCH_PX}}
+                />
                 {groupedMessages.map((messages, i) => {
                   const firstMessage = messages[0]
                   if (firstMessage.visibility === 'system_status') {
@@ -407,7 +422,7 @@ export const PrivateChat = (props: {
                 })}
               </>
             )}
-            {realtimeMessages && messages.length === 0 && (
+            {messages && messages.length === 0 && (
               <div className="text-ink-500 dark:text-ink-600 p-2">
                 {t('messages.empty', 'No messages yet.')}
               </div>