live-captions

Pexip live captions

Pexip's MCU produces real-time transcripts via the conferencing node's speech engine. The signaling channel emits onLiveCaptions events with {data, isFinal, speakers}. Webapp3's useLiveCaptions hook turns this into a flickerless caption overlay with auto-clear.

The trick is that captions arrive as a stream of refinements: the engine sends interim text ("hello there"), then a longer interim ("hello there how are"), then finally isFinal: true ("hello there, how are you"). Render naïvely and the caption flashes between revisions. Webapp3's hook handles this.

Quick start

import {useLiveCaptions, useLiveCaptionsAvailable} from '../hooks/useLiveCaptions';

export const CaptionsOverlay: React.FC = () => {
    const isAvailable = useLiveCaptionsAvailable();
    const {data, isEnabled, toggleCaptions} = useLiveCaptions();

    if (!isAvailable) return null;

    return (
        <>
            <button onClick={toggleCaptions}>
                {isEnabled ? 'Hide captions' : 'Show captions'}
            </button>
            {isEnabled && data && <div className="captions-overlay">{data}</div>}
        </>
    );
};

useLiveCaptionsAvailable() checks both the conference's capability and the brand's showLiveCaptionsFeature flag. Use it to gate the toggle button.

The hook implementation

const CLEAR_CAPTIONS_AFTER = 5000;  // ms — clear if no new caption within window

export const useLiveCaptions = () => {
    const meeting = useMeetingContext();
    const liveCaptionsEnabled = useLiveCaptionsEnabled(meeting.getLiveCaptionsEnabled);
    const [captions, setCaptions] = useState('');
    const previousCaptions = useRef('');
    const timerRef = useRef<number>(undefined);

    const toggleCaptions = useCallback(() => {
        const enable = !liveCaptionsEnabled;
        meeting.setLiveCaptionsEnabled(enable);  // optimistic UI
        meeting.enableLiveCaptions({enable}, undefined, () => {
            // SDK call failed — roll back UI
            meeting.setLiveCaptionsEnabled(false);
        });
        if (!enable) setCaptions('');
    }, [liveCaptionsEnabled, meeting]);

    useEffect(() => {
        if (!liveCaptionsEnabled) return;

        const renderCaptions = (next: {data: string; isFinal: boolean}) => {
            if (next.isFinal) {
                setCaptions(next.data);
                previousCaptions.current = '';
            } else if (next.data.length > previousCaptions.current.length) {
                // Interim refinement — only update if it grew (avoid flicker on revisions)
                setCaptions(next.data);
                previousCaptions.current = next.data;
            }

            // Reset auto-clear timer on every new caption
            clearTimeout(timerRef.current);
            timerRef.current = window.setTimeout(() => {
                setCaptions('');
            }, CLEAR_CAPTIONS_AFTER);
        };

        const detach = infinityClientSignals.onLiveCaptions.add(renderCaptions);
        return () => {
            clearTimeout(timerRef.current);
            detach();
        };
    }, [liveCaptionsEnabled]);

    // Breakout-room reset
    useEffect(() => {
        if (!meeting.getConferenceFeatureFlags()?.isDirectMedia &&
            meeting.getLiveCaptionsAvailability()) {
            return infinityClientSignals.onCallDisconnected.add(() => {
                meeting.setLiveCaptionsEnabled(false);
                setCaptions('');
                previousCaptions.current = '';
                clearTimeout(timerRef.current);
                meeting.enableLiveCaptions({enable: false}, undefined, e => {
                    logger.error(e, 'Failed to turn off live captions');
                });
            });
        }
    }, [meeting]);

    return {data: captions, isEnabled: liveCaptionsEnabled, toggleCaptions};
};