feat: collab hocuspocus v4 upgrade (#2351)

* WIP 1

* complete v4 migration

* add flushDelay

* feat: multiplexing

* fix: survive hocuspocus v4 message timeout

* fix: searchTerm type error
This commit is contained in:
Philip Okugbe
2026-07-20 20:34:09 +01:00
committed by GitHub
parent ce8fbb86ff
commit a55057db37
13 changed files with 876 additions and 787 deletions
@@ -0,0 +1,46 @@
import {
HocuspocusProviderWebsocket,
WebSocketStatus,
} from "@hocuspocus/provider";
import { getCollaborationUrl } from "@/lib/config.ts";
const RELEASE_GRACE_MS = 5000;
let socket: HocuspocusProviderWebsocket | null = null;
let editorCount = 0;
let releaseTimer: ReturnType<typeof setTimeout> | null = null;
export function getCollabSocket(): HocuspocusProviderWebsocket {
if (!socket) {
socket = new HocuspocusProviderWebsocket({
url: getCollaborationUrl(),
autoConnect: false,
});
}
return socket;
}
export function acquireCollabSocket(): void {
editorCount++;
if (releaseTimer) {
clearTimeout(releaseTimer);
releaseTimer = null;
}
const collabSocket = getCollabSocket();
collabSocket.shouldConnect = true;
if (collabSocket.status === WebSocketStatus.Disconnected) {
collabSocket.connect();
}
}
export function releaseCollabSocket(): void {
editorCount--;
if (editorCount > 0) return;
if (releaseTimer) clearTimeout(releaseTimer);
releaseTimer = setTimeout(() => {
releaseTimer = null;
if (editorCount === 0) {
socket?.disconnect();
}
}, RELEASE_GRACE_MS);
}
@@ -1,7 +0,0 @@
import { getCollaborationUrl } from "@/lib/config.ts";
const useCollaborationURL = (): string => {
return getCollaborationUrl();
};
export default useCollaborationURL;
+161 -158
View File
@@ -7,15 +7,16 @@ import React, {
useState, useState,
} from "react"; } from "react";
import { IndexeddbPersistence } from "y-indexeddb"; import { IndexeddbPersistence } from "y-indexeddb";
import * as Y from "yjs";
import { import {
HocuspocusProvider,
onStatusParameters,
WebSocketStatus, WebSocketStatus,
HocuspocusProviderWebsocket,
onSyncedParameters,
onStatelessParameters, onStatelessParameters,
} from "@hocuspocus/provider"; } from "@hocuspocus/provider";
import {
HocuspocusProviderWebsocketComponent,
HocuspocusRoom,
useHocuspocusEvent,
useHocuspocusProvider,
} from "@hocuspocus/provider-react";
import { import {
Editor, Editor,
EditorContent, EditorContent,
@@ -28,7 +29,6 @@ import {
mainExtensions, mainExtensions,
} from "@/features/editor/extensions/extensions"; } from "@/features/editor/extensions/extensions";
import { useAtom, useAtomValue } from "jotai"; import { useAtom, useAtomValue } from "jotai";
import useCollaborationUrl from "@/features/editor/hooks/use-collaboration-url";
import { currentUserAtom } from "@/features/user/atoms/current-user-atom"; import { currentUserAtom } from "@/features/user/atoms/current-user-atom";
import { import {
currentPageEditModeAtom, currentPageEditModeAtom,
@@ -76,6 +76,11 @@ import { EditorLinkMenu } from "@/features/editor/components/link/link-menu";
import ColumnsMenu from "@/features/editor/components/columns/columns-menu.tsx"; import ColumnsMenu from "@/features/editor/components/columns/columns-menu.tsx";
import { TransclusionLookupProvider } from "@/features/editor/components/transclusion/transclusion-lookup-context"; import { TransclusionLookupProvider } from "@/features/editor/components/transclusion/transclusion-lookup-context";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import {
acquireCollabSocket,
getCollabSocket,
releaseCollabSocket,
} from "@/features/editor/collab-socket";
interface PageEditorProps { interface PageEditorProps {
pageId: string; pageId: string;
@@ -91,7 +96,78 @@ export default function PageEditor({
canComment, canComment,
}: PageEditorProps) { }: PageEditorProps) {
const { t } = useTranslation(); const { t } = useTranslation();
const collaborationURL = useCollaborationUrl(); const { data: collabQuery, refetch: refetchCollabToken } = useCollabToken();
const { pageSlug } = useParams();
const slugId = extractPageSlugId(pageSlug);
const [socket] = useState(getCollabSocket);
useEffect(() => {
acquireCollabSocket();
return () => releaseCollabSocket();
}, []);
const handleStateless = ({ payload }: onStatelessParameters) => {
try {
const message = JSON.parse(payload);
if (message?.type !== "page.updated" || !message.updatedAt) return;
const pageData = queryClient.getQueryData<IPage>(["pages", slugId]);
if (pageData) {
queryClient.setQueryData(["pages", slugId], {
...pageData,
updatedAt: message.updatedAt,
...(message.lastUpdatedBy && {
lastUpdatedBy: message.lastUpdatedBy,
}),
});
}
} catch {
// ignore unrelated stateless messages
}
};
const handleAuthenticationFailed = () => {
const payload = jwtDecode(collabQuery?.token);
const now = Date.now().valueOf() / 1000;
const isTokenExpired = now >= payload.exp;
if (isTokenExpired) {
refetchCollabToken();
}
};
return (
<TransclusionLookupProvider>
{collabQuery?.token ? (
<HocuspocusProviderWebsocketComponent websocketProvider={socket}>
<HocuspocusRoom
name={`page.${pageId}`}
token={collabQuery.token}
flushDelay={500}
onStateless={handleStateless}
onAuthenticationFailed={handleAuthenticationFailed}
>
<CollabPageEditor
pageId={pageId}
editable={editable}
content={content}
canComment={canComment}
/>
</HocuspocusRoom>
</HocuspocusProviderWebsocketComponent>
) : (
<StaticPageEditor content={content} ariaLabel={t("Page content")} />
)}
</TransclusionLookupProvider>
);
}
function CollabPageEditor({
pageId,
editable,
content,
canComment,
}: PageEditorProps) {
const { t } = useTranslation();
const provider = useHocuspocusProvider();
const isComponentMounted = useRef(false); const isComponentMounted = useRef(false);
const editorRef = useRef<Editor | null>(null); const editorRef = useRef<Editor | null>(null);
@@ -112,7 +188,6 @@ export default function PageEditor({
); );
const [, setYjsSynced] = useAtom(yjsSyncedAtom); const [, setYjsSynced] = useAtom(yjsSyncedAtom);
const menuContainerRef = useRef(null); const menuContainerRef = useRef(null);
const { data: collabQuery, refetch: refetchCollabToken } = useCollabToken();
const { isIdle, resetIdle } = useIdle(FIVE_MINUTES, { initialState: false }); const { isIdle, resetIdle } = useIdle(FIVE_MINUTES, { initialState: false });
const documentState = useDocumentVisibility(); const documentState = useDocumentVisibility();
const { pageSlug } = useParams(); const { pageSlug } = useParams();
@@ -123,95 +198,24 @@ export default function PageEditor({
[isComponentMounted], [isComponentMounted],
); );
const { handleScrollTo } = useEditorScroll({ canScroll }); const { handleScrollTo } = useEditorScroll({ canScroll });
// Providers only created once per pageId
const providersRef = useRef<{
local: IndexeddbPersistence;
remote: HocuspocusProvider;
socket: HocuspocusProviderWebsocket;
} | null>(null);
const [providersReady, setProvidersReady] = useState(false);
useEffect(() => { useEffect(() => {
if (!providersRef.current) { const local = new IndexeddbPersistence(
const documentName = `page.${pageId}`; provider.configuration.name,
const ydoc = new Y.Doc(); provider.document,
const local = new IndexeddbPersistence(documentName, ydoc); );
const socket = new HocuspocusProviderWebsocket({ local.on("synced", () => setIsLocalSynced(true));
url: collaborationURL,
});
const onLocalSyncedHandler = () => {
setIsLocalSynced(true);
};
const onStatusHandler = (event: onStatusParameters) => {
setYjsConnectionStatus(event.status);
};
const onSyncedHandler = (event: onSyncedParameters) => {
setIsRemoteSynced(event.state);
};
const onStatelessHandler = ({ payload }: onStatelessParameters) => {
try {
const message = JSON.parse(payload);
if (message?.type !== "page.updated" || !message.updatedAt) return;
const pageData = queryClient.getQueryData<IPage>(["pages", slugId]);
if (pageData) {
queryClient.setQueryData(["pages", slugId], {
...pageData,
updatedAt: message.updatedAt,
...(message.lastUpdatedBy && {
lastUpdatedBy: message.lastUpdatedBy,
}),
});
}
} catch {
// ignore unrelated stateless messages
}
};
const onAuthenticationFailedHandler = () => {
const payload = jwtDecode(collabQuery?.token);
const now = Date.now().valueOf() / 1000;
const isTokenExpired = now >= payload.exp;
if (isTokenExpired) {
refetchCollabToken().then((result) => {
if (result.data?.token) {
socket.disconnect();
setTimeout(() => {
remote.configuration.token = result.data.token;
socket.connect();
}, 100);
}
});
}
};
const remote = new HocuspocusProvider({
websocketProvider: socket,
name: documentName,
document: ydoc,
token: collabQuery?.token,
onAuthenticationFailed: onAuthenticationFailedHandler,
onStatus: onStatusHandler,
onSynced: onSyncedHandler,
onStateless: onStatelessHandler,
});
local.on("synced", onLocalSyncedHandler);
providersRef.current = { socket, local, remote };
setProvidersReady(true);
} else {
setProvidersReady(true);
}
// Only destroy on final unmount
return () => { return () => {
providersRef.current?.socket.destroy(); local.destroy();
providersRef.current?.remote.destroy();
providersRef.current?.local.destroy();
providersRef.current = null;
}; };
}, [pageId]); }, [provider]);
useHocuspocusEvent("synced", ({ state }) => setIsRemoteSynced(state));
useHocuspocusEvent("status", ({ status }) => setYjsConnectionStatus(status));
// Only connect/disconnect on tab/idle, not destroy // Only connect/disconnect on tab/idle, not destroy
useEffect(() => { useEffect(() => {
if (!providersReady || !providersRef.current) return; const socket = provider.configuration.websocketProvider;
const socket = providersRef.current.socket;
if ( if (
isIdle && isIdle &&
@@ -228,23 +232,15 @@ export default function PageEditor({
resetIdle(); resetIdle();
socket.connect(); socket.connect();
} }
}, [isIdle, documentState, providersReady, resetIdle]); }, [isIdle, documentState, provider, resetIdle]);
// Attach here, to make sure the connection gets properly established
providersRef.current?.remote.attach();
const extensions = useMemo(() => { const extensions = useMemo(() => {
if (!providersReady || !providersRef.current || !currentUser?.user) { if (!currentUser?.user) {
return mainExtensions; return mainExtensions;
} }
const remoteProvider = providersRef.current.remote; return [...mainExtensions, ...collabExtensions(provider, currentUser.user)];
}, [provider, currentUser?.user]);
return [
...mainExtensions,
...collabExtensions(remoteProvider, currentUser?.user),
];
}, [providersReady, currentUser?.user]);
const editor = useEditor( const editor = useEditor(
{ {
@@ -416,65 +412,72 @@ export default function PageEditor({
} }
}, [yjsConnectionStatus, isSynced]); }, [yjsConnectionStatus, isSynced]);
if (showStatic) {
return <StaticPageEditor content={content} ariaLabel={t("Page content")} />;
}
return ( return (
<TransclusionLookupProvider> <div className="editor-container" style={{ position: "relative" }}>
{showStatic ? ( <div ref={menuContainerRef}>
<EditorProvider <EditorContent editor={editor} />
editable={false}
immediatelyRender={true}
extensions={mainExtensions}
content={content}
editorProps={{
attributes: {
"aria-label": t("Page content"),
},
}}
/>
) : (
<div className="editor-container" style={{ position: "relative" }}>
<div ref={menuContainerRef}>
<EditorContent editor={editor} />
{editor && ( {editor && (
<SearchAndReplaceDialog editor={editor} editable={editable} /> <SearchAndReplaceDialog editor={editor} editable={editable} />
)} )}
{editor && editorIsEditable && ( {editor && editorIsEditable && (
<div> <div>
<EditorAiMenu editor={editor} /> <EditorAiMenu editor={editor} />
<EditorLinkMenu editor={editor} /> <EditorLinkMenu editor={editor} />
<EditorBubbleMenu editor={editor} /> <EditorBubbleMenu editor={editor} />
<TableMenu editor={editor} /> <TableMenu editor={editor} />
<TableHandlesLayer editor={editor} /> <TableHandlesLayer editor={editor} />
<ImageMenu editor={editor} /> <ImageMenu editor={editor} />
<VideoMenu editor={editor} /> <VideoMenu editor={editor} />
<PdfMenu editor={editor} /> <PdfMenu editor={editor} />
<CalloutMenu editor={editor} /> <CalloutMenu editor={editor} />
<SubpagesMenu editor={editor} /> <SubpagesMenu editor={editor} />
<ExcalidrawMenu editor={editor} /> <ExcalidrawMenu editor={editor} />
<DrawioMenu editor={editor} /> <DrawioMenu editor={editor} />
<ColumnsMenu editor={editor} /> <ColumnsMenu editor={editor} />
</div>
)}
{editor &&
!editorIsEditable &&
(editable || canComment) &&
providersRef.current && <ReadonlyBubbleMenu editor={editor} />}
{showCommentPopup && (
<CommentDialog editor={editor} pageId={pageId} />
)}
{showReadOnlyCommentPopup && (
<CommentDialog editor={editor} pageId={pageId} readOnly />
)}
</div> </div>
<div )}
onClick={() => { {editor && !editorIsEditable && (editable || canComment) && (
if (editor && !editor.isDestroyed) editor.commands.focus("end"); <ReadonlyBubbleMenu editor={editor} />
}} )}
style={{ paddingBottom: "20vh" }} {showCommentPopup && <CommentDialog editor={editor} pageId={pageId} />}
></div> {showReadOnlyCommentPopup && (
</div> <CommentDialog editor={editor} pageId={pageId} readOnly />
)} )}
</TransclusionLookupProvider> </div>
<div
onClick={() => {
if (editor && !editor.isDestroyed) editor.commands.focus("end");
}}
style={{ paddingBottom: "20vh" }}
></div>
</div>
);
}
function StaticPageEditor({
content,
ariaLabel,
}: {
content: any;
ariaLabel: string;
}) {
return (
<EditorProvider
editable={false}
immediatelyRender={true}
extensions={mainExtensions}
content={content}
editorProps={{
attributes: {
"aria-label": ariaLabel,
},
}}
/>
); );
} }
-2
View File
@@ -88,7 +88,6 @@
"kysely-migration-cli": "0.4.2", "kysely-migration-cli": "0.4.2",
"kysely-postgres-js": "3.0.0", "kysely-postgres-js": "3.0.0",
"ldapts": "8.1.7", "ldapts": "8.1.7",
"lib0": "0.2.117",
"mammoth": "1.12.0", "mammoth": "1.12.0",
"mime-types": "3.0.2", "mime-types": "3.0.2",
"msgpackr": "^1.11.9", "msgpackr": "^1.11.9",
@@ -118,7 +117,6 @@
"stripe": "^17.7.0", "stripe": "^17.7.0",
"tlds": "1.261.0", "tlds": "1.261.0",
"tmp-promise": "3.0.3", "tmp-promise": "3.0.3",
"tseep": "1.3.1",
"typesense": "3.0.5", "typesense": "3.0.5",
"undici": "7.28.0", "undici": "7.28.0",
"ws": "8.21.0", "ws": "8.21.0",
@@ -15,6 +15,7 @@ import {
RedisSyncExtension, RedisSyncExtension,
SerializedHTTPRequest, SerializedHTTPRequest,
} from './extensions/redis-sync'; } from './extensions/redis-sync';
import { toWebRequest } from './extensions/redis-sync/redis-sync.types';
import { WsSocketWrapper } from './extensions/redis-sync/ws-socket-wrapper'; import { WsSocketWrapper } from './extensions/redis-sync/ws-socket-wrapper';
import RedisClient from 'ioredis'; import RedisClient from 'ioredis';
import { pack, unpack } from 'msgpackr'; import { pack, unpack } from 'msgpackr';
@@ -98,34 +99,36 @@ export class CollaborationGateway {
const serializedHTTPRequest = this.serializeRequest(request); const serializedHTTPRequest = this.serializeRequest(request);
const socketId = serializedHTTPRequest.headers['sec-websocket-key']; const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
// Create wrapper socket that only receives events via emit()
// This prevents double-handling since Hocuspocus won't listen to raw WebSocket events
const wrappedSocket = new WsSocketWrapper(client); const wrappedSocket = new WsSocketWrapper(client);
// Route through RedisSync extension (this calls handleConnection internally) // Route through RedisSync extension (this calls handleConnection internally)
this.redisSync.onSocketOpen(wrappedSocket as any, serializedHTTPRequest); this.redisSync.onSocketOpen(wrappedSocket, serializedHTTPRequest);
// Forward raw WebSocket messages to the extension
client.on('message', (data: ArrayBuffer) => { client.on('message', (data: ArrayBuffer) => {
this.redisSync!.onSocketMessage( this.redisSync!.onSocketMessage(serializedHTTPRequest, data);
wrappedSocket as any,
serializedHTTPRequest,
data,
);
}); });
// Forward close events
client.on('close', (code: number, reason: Buffer) => { client.on('close', (code: number, reason: Buffer) => {
this.redisSync!.onSocketClose(socketId, code, reason.buffer as ArrayBuffer); this.redisSync!.onSocketClose(
}); socketId,
code,
// Forward pong events for keepalive new Uint8Array(reason).buffer,
client.on('pong', (data: Buffer) => { );
wrappedSocket.emit('pong', data);
}); });
} else { } else {
// Fallback to direct Hocuspocus connection // Fallback to direct Hocuspocus connection
this.hocuspocus.handleConnection(client, request); const clientConnection = this.hocuspocus.handleConnection(
client,
toWebRequest(this.serializeRequest(request)),
);
client.on('message', (data: Buffer) => {
clientConnection.handleMessage(new Uint8Array(data));
});
client.on('close', (code: number, reason: Buffer) => {
clientConnection.handleClose({ code, reason: reason.toString() });
});
} }
} }
@@ -178,6 +181,7 @@ export class CollaborationGateway {
if (this.hocuspocus.getDocumentsCount() === 0) resolve(''); if (this.hocuspocus.getDocumentsCount() === 0) resolve('');
this.hocuspocus.closeConnections(); this.hocuspocus.closeConnections();
this.hocuspocus.flushPendingStores();
} catch (error) { } catch (error) {
console.error(error); console.error(error);
} }
@@ -96,7 +96,7 @@ export class PersistenceExtension implements Extension {
} }
async onStoreDocument(data: onStoreDocumentPayload) { async onStoreDocument(data: onStoreDocumentPayload) {
const { documentName, document, context } = data; const { documentName, document, lastContext } = data;
const pageId = getPageId(documentName); const pageId = getPageId(documentName);
@@ -151,7 +151,7 @@ export class PersistenceExtension implements Extension {
content: tiptapJson, content: tiptapJson,
textContent: textContent, textContent: textContent,
ydoc: ydocState, ydoc: ydocState,
lastUpdatedById: context.user.id, lastUpdatedById: lastContext.user.id,
contributorIds: contributorIds, contributorIds: contributorIds,
}, },
pageId, pageId,
@@ -169,12 +169,12 @@ export class PersistenceExtension implements Extension {
JSON.stringify({ JSON.stringify({
type: 'page.updated', type: 'page.updated',
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
lastUpdatedById: context?.user?.id, lastUpdatedById: lastContext?.user?.id,
lastUpdatedBy: context?.user lastUpdatedBy: lastContext?.user
? { ? {
id: context.user?.id, id: lastContext.user?.id,
name: context.user?.name, name: lastContext.user?.name,
avatarUrl: context.user?.avatarUrl, avatarUrl: lastContext.user?.avatarUrl,
} }
: undefined, : undefined,
}), }),
@@ -1,61 +1,37 @@
import type RedisClient from 'ioredis'; import type RedisClient from 'ioredis';
import { EventEmitter } from 'tseep'; import type { WebSocketLike } from '@hocuspocus/server';
import type { import type { Pack, RSAMessageClose, RSAMessageSend } from './redis-sync.types';
Pack,
RSAMessageClose,
RSAMessagePing,
RSAMessageSend,
} from './redis-sync.types';
export class CollabProxySocket extends EventEmitter { // Stands in for the client WebSocket on the server that owns the document.
// Outgoing traffic is relayed over redis to the origin server, which holds the real socket.
export class CollabProxySocket implements WebSocketLike {
private readonly replyTo: string; private readonly replyTo: string;
private readonly serverChannel: string;
private readonly socketId: string; private readonly socketId: string;
private pub: RedisClient; private pub: RedisClient;
private readonly pack: Pack; private readonly pack: Pack;
readyState = 1; readyState = 1;
onClose?: (code?: number, reason?: string) => void;
constructor( constructor(pub: RedisClient, pack: Pack, replyTo: string, socketId: string) {
pub: RedisClient,
pack: Pack,
replyTo: string,
serverChannel: string,
socketId: string,
) {
super();
this.replyTo = replyTo; this.replyTo = replyTo;
this.socketId = socketId; this.socketId = socketId;
this.serverChannel = serverChannel;
this.pub = pub; this.pub = pub;
this.pack = pack; this.pack = pack;
this.once('close', () => {
this.readyState = 3;
});
} }
private publish(msg: RSAMessageClose | RSAMessagePing | RSAMessageSend) { private publish(msg: RSAMessageClose | RSAMessageSend) {
this.pub.publish(this.replyTo, this.pack(msg)); this.pub.publish(this.replyTo, this.pack(msg));
} }
// The origin server already closed the real socket; stop relaying without echoing a close back
markClosed() {
this.readyState = 3;
}
close(code?: number, reason?: string) { close(code?: number, reason?: string) {
if (this.readyState !== 1) return; if (this.readyState !== 1) return;
const msg: RSAMessageClose = { this.readyState = 3;
type: 'close', this.onClose?.(code, reason);
code,
reason,
socketId: this.socketId,
};
this.publish(msg);
}
ping() {
if (this.readyState !== 1) return;
const msg: RSAMessagePing = {
type: 'ping',
socketId: this.socketId,
replyTo: this.serverChannel,
};
this.publish(msg);
} }
send(message: Uint8Array) { send(message: Uint8Array) {
@@ -3,27 +3,30 @@ import {
Extension, Extension,
Hocuspocus, Hocuspocus,
IncomingMessage, IncomingMessage,
afterUnloadDocumentPayload,
onConfigurePayload, onConfigurePayload,
onLoadDocumentPayload, onLoadDocumentPayload,
afterUnloadDocumentPayload,
WebSocketLike,
} from '@hocuspocus/server'; } from '@hocuspocus/server';
import { ConnectionTimeout, Unauthorized } from '@hocuspocus/common';
import RedisClient from 'ioredis'; import RedisClient from 'ioredis';
import { readVarString } from 'lib0/decoding.js';
import { CollabProxySocket } from './collab-proxy-socket'; import { CollabProxySocket } from './collab-proxy-socket';
import { import {
BaseWebSocket,
Configuration, Configuration,
CustomEvents, CustomEvents,
Pack, Pack,
RSAMessage, RSAMessage,
RSAMessageClose,
RSAMessageCloseProxy, RSAMessageCloseProxy,
RSAMessageCustomEventComplete, RSAMessageCustomEventComplete,
RSAMessageCustomEventStart, RSAMessageCustomEventStart,
RSAMessagePong,
RSAMessageProxy, RSAMessageProxy,
RSAMessageUnload, RSAMessageUnload,
SerializedHTTPRequest, SerializedHTTPRequest,
Unpack, Unpack,
OriginConnection,
ProxyConnection,
toWebRequest,
} from './redis-sync.types'; } from './redis-sync.types';
export type { Pack, SerializedHTTPRequest } from './redis-sync.types'; export type { Pack, SerializedHTTPRequest } from './redis-sync.types';
@@ -38,10 +41,10 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
private sub: RedisClient; private sub: RedisClient;
private readonly pack: Pack; private readonly pack: Pack;
private readonly unpack: Unpack; private readonly unpack: Unpack;
private originSockets: Record<SocketId, BaseWebSocket> = {}; private originConnections: Record<SocketId, OriginConnection> = {};
private locks: Record<DocumentName, NodeJS.Timeout> = {}; private locks: Record<DocumentName, NodeJS.Timeout> = {};
private lockPromises: Record<DocumentName, Promise<ServerId | null>> = {}; private lockPromises: Record<DocumentName, Promise<ServerId | null>> = {};
private proxySockets: Record<SocketId, CollabProxySocket> = {}; private proxyConnections: Record<SocketId, ProxyConnection> = {};
private readonly prefix: string; private readonly prefix: string;
private readonly lockPrefix: string; private readonly lockPrefix: string;
private readonly msgChannel: string; private readonly msgChannel: string;
@@ -54,6 +57,9 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
// @ts-ignore // @ts-ignore
private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> = private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> =
{}; {};
private deriveContext: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
constructor(configuration: Configuration<TCE>) { constructor(configuration: Configuration<TCE>) {
const { const {
@@ -65,6 +71,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
prefix, prefix,
customEvents, customEvents,
customEventTTL, customEventTTL,
deriveContext,
} = configuration; } = configuration;
this.pub = redis.duplicate(); this.pub = redis.duplicate();
this.sub = redis.duplicate(); this.sub = redis.duplicate();
@@ -77,6 +84,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
this.lockPrefix = `${this.prefix}Lock`; this.lockPrefix = `${this.prefix}Lock`;
this.msgChannel = `${this.prefix}Msg`; this.msgChannel = `${this.prefix}Msg`;
this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents); this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents);
this.deriveContext = deriveContext ?? (() => ({}));
this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`); this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`);
this.sub.on('messageBuffer', this.handleRedisMessage); this.sub.on('messageBuffer', this.handleRedisMessage);
this.pub.on('error', () => {}); this.pub.on('error', () => {});
@@ -87,44 +95,63 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
} }
private closeProxy(socketId: string) { private closeProxy(socketId: string) {
const proxySocket = this.proxySockets[socketId]; const entry = this.proxyConnections[socketId];
if (proxySocket) { if (entry) {
proxySocket.emit( delete this.proxyConnections[socketId];
'close', const { socket, clientConnection } = entry;
1000, // The origin socket is already gone; don't echo a close message back
Buffer.from('provider_initiated', 'utf-8'), socket.markClosed();
); clientConnection.handleClose({
delete this.proxySockets[socketId]; code: 1000,
reason: 'provider_initiated',
});
} }
} }
private pongProxy(socketId: string) {
this.proxySockets[socketId]?.emit('pong');
}
private handleProxyMessage( private handleProxyMessage(
msg: Pick<RSAMessageProxy, 'replyTo' | 'message' | 'serializedHTTPRequest'>, msg: Pick<RSAMessageProxy, 'replyTo' | 'message' | 'serializedHTTPRequest'>,
) { ) {
const { replyTo, message, serializedHTTPRequest } = msg; const { replyTo, message, serializedHTTPRequest } = msg;
const { headers } = serializedHTTPRequest; const { headers } = serializedHTTPRequest;
const socketId = headers['sec-websocket-key']!; const socketId = headers['sec-websocket-key'];
let socket = this.proxySockets[socketId]; let entry = this.proxyConnections[socketId];
if (!socket) { if (!entry) {
socket = new CollabProxySocket( const socket = new CollabProxySocket(
this.pub, this.pub,
this.pack, this.pack,
replyTo, replyTo,
`${this.msgChannel}:${this.serverId}`,
socketId, socketId,
); );
this.proxySockets[socketId] = socket; // A proxy connection with no live documents (client left the page, auth
this.instance.handleConnection( // failed, or the origin server crashed) is reaped by hocuspocus' message
socket as any, // timeout. Dispose it silently in that case: relaying the timeout close
serializedHTTPRequest as any, // to the origin would kill the client's real socket, which may be busy
{}, // serving other documents. Genuine protocol closes are still relayed.
socket.onClose = (code, reason) => {
delete this.proxyConnections[socketId];
if (code !== ConnectionTimeout.code) {
const msg: RSAMessageClose = {
type: 'close',
code,
reason,
socketId,
};
this.pub.publish(replyTo, this.pack(msg));
}
};
const clientConnection = this.instance.handleConnection(
socket,
toWebRequest(serializedHTTPRequest),
this.deriveContext(serializedHTTPRequest),
); );
entry = { clientConnection, socket };
this.proxyConnections[socketId] = entry;
} }
socket.emit('message', message); entry.clientConnection.handleMessage(message);
}
private getLock(documentName: string) {
return this.pub.get(this.getKey(documentName));
} }
private getOrClaimLock(documentName: string) { private getOrClaimLock(documentName: string) {
@@ -166,10 +193,6 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
this.closeProxy(msg.socketId); this.closeProxy(msg.socketId);
return; return;
} }
if (type === 'pong') {
this.pongProxy(msg.socketId);
return;
}
if (type === 'unload') { if (type === 'unload') {
delete this.lockPromises[msg.documentName]; delete this.lockPromises[msg.documentName];
return; return;
@@ -198,22 +221,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return; return;
} }
const { socketId } = msg; const { socketId } = msg;
const socket = this.originSockets[socketId]; const entry = this.originConnections[socketId];
if (!socket) { if (!entry) {
// origin socket already cleaned up // origin socket already cleaned up
return; return;
} }
const { socket } = entry;
if (type === 'close') { if (type === 'close') {
socket.close(msg.code, msg.reason); socket.close(msg.code, msg.reason);
} else if (type === 'ping') {
// Reply instantly to the proxy socket, without forwarding to client
// The origin socket handles heartbeat for itself
const { replyTo, socketId } = msg;
const reply: RSAMessagePong = {
type: 'pong',
socketId,
};
this.pub.publish(`${replyTo}`, this.pack(reply));
} else if (type === 'send') { } else if (type === 'send') {
socket.send(msg.message); socket.send(msg.message);
} }
@@ -251,6 +266,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
eventName: TName, eventName: TName,
documentName: string, documentName: string,
payload: any, payload: any,
// if true, don't claim the lock. Useful for targeting pages that are currently open
onlyIfOpen = false,
) { ) {
const isDocLoadedOnInstance = this.instance.documents.has(documentName); const isDocLoadedOnInstance = this.instance.documents.has(documentName);
@@ -258,7 +275,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return this.handleEventLocally(eventName, documentName, payload); return this.handleEventLocally(eventName, documentName, payload);
} }
const proxyTo = await this.getOrClaimLockThrottled(documentName); const proxyTo = await (onlyIfOpen
? this.getLock(documentName)
: this.getOrClaimLockThrottled(documentName));
if (!proxyTo && onlyIfOpen) {
return;
}
if (proxyTo && proxyTo !== this.serverId) { if (proxyTo && proxyTo !== this.serverId) {
++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below ++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below
const replyId = this.replyIdCounter; const replyId = this.replyIdCounter;
@@ -277,7 +301,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
const { promise, resolve, reject } = Promise.withResolvers(); const { promise, resolve, reject } = Promise.withResolvers();
this.pendingReplies[replyId] = resolve; this.pendingReplies[replyId] = resolve;
setTimeout(() => { setTimeout(() => {
reject('TIMEOUT'); delete this.pendingReplies[replyId];
reject(new Error('TIMEOUT'));
}, this.customEventTTL); }, this.customEventTTL);
return promise as Promise<ReturnType<TCE[TName]>>; return promise as Promise<ReturnType<TCE[TName]>>;
} }
@@ -296,36 +321,59 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
/* WebSocket Server Hooks */ /* WebSocket Server Hooks */
onSocketOpen( onSocketOpen(
ws: BaseWebSocket, ws: WebSocketLike,
serializedHTTPRequest: SerializedHTTPRequest, serializedHTTPRequest: SerializedHTTPRequest,
context = {},
) { ) {
const socketId = serializedHTTPRequest.headers['sec-websocket-key']!; const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
this.originSockets[socketId] = ws; const clientConnection = this.instance.handleConnection(
this.instance.handleConnection( ws,
ws as any, toWebRequest(serializedHTTPRequest),
serializedHTTPRequest as any, this.deriveContext(serializedHTTPRequest),
context,
); );
this.originConnections[socketId] = { clientConnection, socket: ws };
} }
async onSocketMessage( async onSocketMessage(
ws: BaseWebSocket,
serializedHTTPRequest: SerializedHTTPRequest, serializedHTTPRequest: SerializedHTTPRequest,
detachableMsg: ArrayBuffer, detachableMsg: ArrayBuffer,
) { ) {
const message = new Uint8Array(detachableMsg.slice()); const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
const tmpMsg = new IncomingMessage(detachableMsg); const entry = this.originConnections[socketId];
const documentName = readVarString(tmpMsg.decoder); if (!entry) return;
const { clientConnection } = entry;
let message: Uint8Array;
let documentName: string;
try {
message = new Uint8Array(detachableMsg.slice());
const tmpMsg = new IncomingMessage(detachableMsg);
const documentNameAndSessionId = tmpMsg.readVarString();
// session-aware providers suffix the documentName with \0sessionId
const sepIdx = documentNameAndSessionId.indexOf('\0');
documentName =
sepIdx === -1
? documentNameAndSessionId
: documentNameAndSessionId.slice(0, sepIdx);
} catch (error) {
entry.socket.close(Unauthorized.code, Unauthorized.reason);
return;
}
const isDocLoadedOnInstance = this.instance.documents.has(documentName); const isDocLoadedOnInstance = this.instance.documents.has(documentName);
if (isDocLoadedOnInstance) { if (isDocLoadedOnInstance) {
ws.emit('message', message); clientConnection.handleMessage(message);
return; return;
} }
const proxyTo = await this.getOrClaimLockThrottled(documentName); const proxyTo = await this.getOrClaimLockThrottled(documentName);
if (proxyTo && proxyTo !== this.serverId) { if (proxyTo && proxyTo !== this.serverId) {
// Proxied messages bypass handleMessage, so refresh the connection's
// liveness fields manually or hocuspocus' message timeout would reap the
// real socket every `timeout` ms. connectionEstablishedAt is the
// reference while unauthenticated (auth for remote docs is proxied too)
// and is private upstream.
clientConnection.lastMessageReceivedAt = Date.now();
(clientConnection as any).connectionEstablishedAt = Date.now();
// another server owns the doc // another server owns the doc
const proxyMessage: RSAMessageProxy = { const proxyMessage: RSAMessageProxy = {
serializedHTTPRequest: serializedHTTPRequest, serializedHTTPRequest: serializedHTTPRequest,
@@ -338,16 +386,17 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return; return;
} }
// This server owns the document, but hocuspocus hasn't loaded it yet // This server owns the document, but hocuspocus hasn't loaded it yet
ws.emit('message', message); clientConnection.handleMessage(message);
} }
onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) { onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) {
const socket = this.originSockets[socketId]; const entry = this.originConnections[socketId];
if (!socket) return; if (!entry) return;
// at this point the socket is considered GC'd and we cannot call close delete this.originConnections[socketId];
// The origin socket did not set up any connections for the proxy, so none of the hooks will work if we just emit entry.clientConnection.handleClose({
socket?.emit('close', code, reason); code: code ?? 1000,
delete this.originSockets[socketId]; reason: reason ? Buffer.from(reason).toString() : '',
});
const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId }; const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId };
this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {}); this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {});
} }
@@ -372,6 +421,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
} }
async onDestroy() { async onDestroy() {
this.pendingReplies = {};
this.pub.disconnect(false); this.pub.disconnect(false);
this.sub.disconnect(false); this.sub.disconnect(false);
} }
@@ -1,12 +1,13 @@
import EventEmitter from 'node:events';
import { IncomingHttpHeaders } from 'node:http2'; import { IncomingHttpHeaders } from 'node:http2';
import RedisClient from 'ioredis'; import RedisClient from 'ioredis';
import { CollabProxySocket } from './collab-proxy-socket';
import { type Hocuspocus, type WebSocketLike } from '@hocuspocus/server';
export type SecondParam<T> = T extends ( export type SecondParam<T> = T extends (
arg1: unknown, arg1: any,
arg2: infer A, arg2: infer A,
...args: unknown[] ...args: any[]
) => unknown ) => any
? A ? A
: never; : never;
@@ -41,17 +42,6 @@ export type RSAMessageClose = {
socketId: string; socketId: string;
}; };
export type RSAMessagePing = {
type: 'ping';
socketId: string;
replyTo: string;
};
export type RSAMessagePong = {
type: 'pong';
socketId: string;
};
export type RSAMessageSend = { export type RSAMessageSend = {
type: 'send'; type: 'send';
// @ts-ignore // @ts-ignore
@@ -59,7 +49,7 @@ export type RSAMessageSend = {
socketId: string; socketId: string;
}; };
export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = { export type RSAMessageCustomEventStart<TName = string, TPayload = any> = {
type: 'customEventStart'; type: 'customEventStart';
documentName: string; documentName: string;
eventName: TName; eventName: TName;
@@ -71,7 +61,7 @@ export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = {
export type RSAMessageCustomEventComplete = { export type RSAMessageCustomEventComplete = {
type: 'customEventComplete'; type: 'customEventComplete';
replyId: number; replyId: number;
payload: unknown; payload: any;
}; };
export type RSAMessage = export type RSAMessage =
@@ -79,8 +69,6 @@ export type RSAMessage =
| RSAMessageCloseProxy | RSAMessageCloseProxy
| RSAMessageUnload | RSAMessageUnload
| RSAMessageClose | RSAMessageClose
| RSAMessagePing
| RSAMessagePong
| RSAMessageSend | RSAMessageSend
| RSAMessageCustomEventStart | RSAMessageCustomEventStart
| RSAMessageCustomEventComplete; | RSAMessageCustomEventComplete;
@@ -99,9 +87,20 @@ type CustomEventName = string;
export type CustomEvents = Record< export type CustomEvents = Record<
CustomEventName, CustomEventName,
(documentName: string, payload: unknown) => Promise<unknown> (documentName: string, payload: any) => Promise<any>
>; >;
// Not exported by @hocuspocus/server
export type ClientConnection = ReturnType<Hocuspocus['handleConnection']>;
export type OriginConnection = {
clientConnection: ClientConnection;
socket: WebSocketLike;
};
export type ProxyConnection = {
clientConnection: ClientConnection;
socket: CollabProxySocket;
};
export interface Configuration<TCE> { export interface Configuration<TCE> {
redis: RedisClient; redis: RedisClient;
pack: Pack; pack: Pack;
@@ -111,11 +110,29 @@ export interface Configuration<TCE> {
customEventTTL?: number; customEventTTL?: number;
prefix?: string; prefix?: string;
customEvents?: TCE; customEvents?: TCE;
// Derive the hocuspocus context once per socket instead of re-deriving it in a
// per-document hook like onConnect/onAuthenticate. Runs on the origin server when
// the socket opens and on the doc owner when the first proxied message arrives.
deriveContext?: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
} }
export type BaseWebSocket = EventEmitter & { // Hocuspocus expects a web-standard Request, so rehydrate one from what crossed the wire
readyState: number; export const toWebRequest = (serializedHTTPRequest: SerializedHTTPRequest) => {
close(code?: number, reason?: string): void; const { method, url, headers } = serializedHTTPRequest;
ping(): void; const webHeaders = new Headers();
send(message: Uint8Array): void; Object.entries(headers).forEach(([name, value]) => {
if (Array.isArray(value)) {
value.forEach((v) => {
webHeaders.append(name, v);
});
} else if (value !== undefined) {
webHeaders.set(name, value);
}
});
return new Request(new URL(url, 'http://localhost'), {
method,
headers: webHeaders,
});
}; };
@@ -1,20 +1,17 @@
import { EventEmitter } from 'events';
import type WebSocket from 'ws'; import type WebSocket from 'ws';
import type { WebSocketLike } from '@hocuspocus/server';
/** /**
* Wrapper around ws WebSocket that only receives events via emit(). * Wrapper around ws WebSocket that Hocuspocus only writes to.
* This prevents double-handling when used with RedisSyncExtension. * Incoming socket events are forwarded separately by the gateway,
* which prevents double-handling with RedisSyncExtension.
*/ */
export class WsSocketWrapper extends EventEmitter { export class WsSocketWrapper implements WebSocketLike {
private ws: WebSocket; private ws: WebSocket;
readyState = 1; readyState = 1;
constructor(ws: WebSocket) { constructor(ws: WebSocket) {
super();
this.ws = ws; this.ws = ws;
this.once('close', () => {
this.readyState = 3;
});
} }
close(code?: number, reason?: string) { close(code?: number, reason?: string) {
@@ -27,15 +24,6 @@ export class WsSocketWrapper extends EventEmitter {
} }
} }
ping() {
if (this.readyState !== 1) return;
try {
this.ws.ping();
} catch (e) {
// Socket already closed
}
}
send(message: Uint8Array) { send(message: Uint8Array) {
if (this.readyState !== 1) return; if (this.readyState !== 1) return;
try { try {
+34 -32
View File
@@ -23,41 +23,43 @@
"@casl/ability": "6.8.0", "@casl/ability": "6.8.0",
"@docmost/editor-ext": "workspace:*", "@docmost/editor-ext": "workspace:*",
"@floating-ui/dom": "1.7.3", "@floating-ui/dom": "1.7.3",
"@hocuspocus/provider": "3.4.4", "@hocuspocus/common": "4.4.0",
"@hocuspocus/server": "3.4.4", "@hocuspocus/provider": "4.4.0",
"@hocuspocus/transformer": "3.4.4", "@hocuspocus/provider-react": "4.4.0",
"@hocuspocus/server": "4.4.0",
"@hocuspocus/transformer": "4.4.0",
"@joplin/turndown": "4.0.82", "@joplin/turndown": "4.0.82",
"@joplin/turndown-plugin-gfm": "1.0.64", "@joplin/turndown-plugin-gfm": "1.0.64",
"@sindresorhus/slugify": "3.0.0", "@sindresorhus/slugify": "3.0.0",
"@tiptap/core": "3.27.1", "@tiptap/core": "3.28.0",
"@tiptap/extension-audio": "3.27.1", "@tiptap/extension-audio": "3.28.0",
"@tiptap/extension-code-block": "3.27.1", "@tiptap/extension-code-block": "3.28.0",
"@tiptap/extension-collaboration": "3.27.1", "@tiptap/extension-collaboration": "3.28.0",
"@tiptap/extension-collaboration-caret": "3.27.1", "@tiptap/extension-collaboration-caret": "3.28.0",
"@tiptap/extension-color": "3.27.1", "@tiptap/extension-color": "3.28.0",
"@tiptap/extension-document": "3.27.1", "@tiptap/extension-document": "3.28.0",
"@tiptap/extension-heading": "3.27.1", "@tiptap/extension-heading": "3.28.0",
"@tiptap/extension-highlight": "3.27.1", "@tiptap/extension-highlight": "3.28.0",
"@tiptap/extension-history": "3.27.1", "@tiptap/extension-history": "3.28.0",
"@tiptap/extension-image": "3.27.1", "@tiptap/extension-image": "3.28.0",
"@tiptap/extension-link": "3.27.1", "@tiptap/extension-link": "3.28.0",
"@tiptap/extension-list": "3.27.1", "@tiptap/extension-list": "3.28.0",
"@tiptap/extension-placeholder": "3.27.1", "@tiptap/extension-placeholder": "3.28.0",
"@tiptap/extension-subscript": "3.27.1", "@tiptap/extension-subscript": "3.28.0",
"@tiptap/extension-superscript": "3.27.1", "@tiptap/extension-superscript": "3.28.0",
"@tiptap/extension-table": "3.27.1", "@tiptap/extension-table": "3.28.0",
"@tiptap/extension-text": "3.27.1", "@tiptap/extension-text": "3.28.0",
"@tiptap/extension-text-align": "3.27.1", "@tiptap/extension-text-align": "3.28.0",
"@tiptap/extension-text-style": "3.27.1", "@tiptap/extension-text-style": "3.28.0",
"@tiptap/extension-typography": "3.27.1", "@tiptap/extension-typography": "3.28.0",
"@tiptap/extension-unique-id": "3.27.1", "@tiptap/extension-unique-id": "3.28.0",
"@tiptap/extension-youtube": "3.27.1", "@tiptap/extension-youtube": "3.28.0",
"@tiptap/html": "3.27.1", "@tiptap/html": "3.28.0",
"@tiptap/pm": "3.27.1", "@tiptap/pm": "3.28.0",
"@tiptap/react": "3.27.1", "@tiptap/react": "3.28.0",
"@tiptap/starter-kit": "3.27.1", "@tiptap/starter-kit": "3.28.0",
"@tiptap/suggestion": "3.27.1", "@tiptap/suggestion": "3.28.0",
"@tiptap/y-tiptap": "3.0.5", "@tiptap/y-tiptap": "3.0.7",
"bytes": "3.1.2", "bytes": "3.1.2",
"cross-env": "10.1.0", "cross-env": "10.1.0",
"date-fns": "4.1.0", "date-fns": "4.1.0",
@@ -422,6 +422,8 @@ export const SearchAndReplace = Extension.create<
state: { state: {
init: () => DecorationSet.empty, init: () => DecorationSet.empty,
apply({ doc, docChanged }, oldState) { apply({ doc, docChanged }, oldState) {
const storage = editor.storage.searchAndReplace;
if (!storage) return oldState;
const { const {
searchTerm, searchTerm,
lastSearchTerm, lastSearchTerm,
@@ -429,7 +431,7 @@ export const SearchAndReplace = Extension.create<
lastCaseSensitive, lastCaseSensitive,
resultIndex, resultIndex,
lastResultIndex, lastResultIndex,
} = editor.storage.searchAndReplace; } = storage;
if ( if (
!docChanged && !docChanged &&
+426 -416
View File
File diff suppressed because it is too large Load Diff