diff --git a/template/package.json b/template/package.json index 445b26a24..d5bd0c7e4 100644 --- a/template/package.json +++ b/template/package.json @@ -81,7 +81,7 @@ "nosleep.js": "0.12.0", "npm": "^9.7.2", "p-queue": "^7.3.4", - "protobufjs": "^7.2.3", + "protobufjs": "^7.2.5", "react": "18.2.0", "react-dom": "18.2.0", "react-is": "18.0.0", diff --git a/template/src/atoms/ActionMenu.tsx b/template/src/atoms/ActionMenu.tsx index 3fbabb59c..9df780db7 100644 --- a/template/src/atoms/ActionMenu.tsx +++ b/template/src/atoms/ActionMenu.tsx @@ -12,6 +12,7 @@ import { StyleProp, TextStyle, Pressable, + ScrollView, } from 'react-native'; import React, {SetStateAction, useState} from 'react'; @@ -50,6 +51,7 @@ export interface ActionMenuItem { iconSize?: number; hide?: ToolbarItemHide; type?: string; + iconPosition?: 'start' | 'end'; } export interface ActionMenuProps { from: string; @@ -83,6 +85,7 @@ export interface UserActionMenuItemProps { isBase64Icon?: boolean; externalIconString?: string; titleStyle?: StyleProp; + iconPosition?: 'start' | 'end'; } export const UserActionMenuItem = ({ @@ -101,12 +104,18 @@ export const UserActionMenuItem = ({ isExternalIcon = false, isBase64Icon = false, externalIconString = '', + iconPosition = 'start', }: UserActionMenuItemProps) => { const iconToShow = isHovered && onHoverIcon && !disabled ? onHoverIcon : icon; - const content = ( - <> - + const renderIcon = () => { + if (!iconToShow) return null; + + return ( + {isExternalIcon ? ( )} - {label} + ); + }; + + const content = ( + <> + {iconPosition === 'start' && renderIcon()} + + + {label} + + + {iconPosition === 'end' && renderIcon()} {typeof toggleStatus === 'boolean' && ( @@ -202,6 +227,7 @@ const ActionMenu = (props: ActionMenuProps) => { iconSize = 20, titleStyle = {}, type = '', + iconPosition = 'start', } = item; return ( @@ -272,6 +298,7 @@ const ActionMenu = (props: ActionMenuProps) => { isHovered={isHovered} isExternalIcon={isExternalIcon} externalIconString={externalIconString} + iconPosition={iconPosition} isBase64Icon={isBase64Icon} onHoverIcon={onHoverIcon} /> @@ -284,6 +311,22 @@ const ActionMenu = (props: ActionMenuProps) => { }); }; + const shouldUseScrollView = props?.containerStyle?.maxHeight !== undefined; + + const renderScrollableContent = () => { + return shouldUseScrollView ? ( + + {renderItems()} + + ) : ( + renderItems() + ); + }; + return ( {hoverMode ? ( @@ -293,7 +336,7 @@ const ActionMenu = (props: ActionMenuProps) => {
props?.onHover && props.onHover(true)} onMouseLeave={() => props?.onHover && props.onHover(false)}> - {renderItems()} + {renderScrollableContent()}
) @@ -311,7 +354,7 @@ const ActionMenu = (props: ActionMenuProps) => { - {renderItems()} + {renderScrollableContent()} )} @@ -388,6 +431,16 @@ const styles = StyleSheet.create({ marginVertical: 12, marginLeft: 12, }, + iconContainerEnd: { + marginLeft: 10, + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + }, + textWithEndIcon: { + marginRight: 0, + paddingLeft:12 + }, toggleContainer: { justifyContent: 'center', alignItems: 'center', diff --git a/template/src/atoms/DropDownMulti.tsx b/template/src/atoms/DropDownMulti.tsx index 578e60fdc..149a3acd0 100644 --- a/template/src/atoms/DropDownMulti.tsx +++ b/template/src/atoms/DropDownMulti.tsx @@ -14,6 +14,7 @@ import { Modal, View, Image, + ScrollView, } from 'react-native'; import {isWebInternal} from '../utils/common'; import ThemeConfig from '../theme'; @@ -34,6 +35,8 @@ interface Props { defaultSelectedValues?: LanguageType[]; isOpen: boolean; setIsOpen: React.Dispatch>; + maxAllowedSelection?: number; + protectedLanguages?: LanguageType[]; } const DropdownMulti: FC = ({ @@ -45,6 +48,8 @@ const DropdownMulti: FC = ({ icon, isOpen, setIsOpen, + maxAllowedSelection = 2, + protectedLanguages = [], }) => { const DropdownButton = useRef(); const maxHeight = 170; @@ -78,20 +83,20 @@ const DropdownMulti: FC = ({ //setIsOpen(false); const isSelected = selectedValues.includes(item.value); + const isProtected = protectedLanguages.includes(item.value); let updatedValues = [...selectedValues]; if (isSelected) { - // Item is already selected, remove it if there are more than one selected languages - if (selectedValues.length > 0) { - updatedValues = selectedValues.filter((value) => value !== item.value); + // Item is already selected, remove it if it's not protected and there are more than one selected languages + if (selectedValues.length > 0 && !isProtected) { + updatedValues = selectedValues.filter(value => value !== item.value); } } else { // Item is not selected, add it - if (selectedValues.length < 2) { + if (selectedValues.length < maxAllowedSelection) { updatedValues = [...selectedValues, item.value]; } else { - // Max selection limit reached, replace the second selected value - // updatedValues = [selectedValues[1], item.value]; + // Max selection limit reached, do nothing or implement replacement logic } } @@ -102,13 +107,15 @@ const DropdownMulti: FC = ({ // renders each lang checkbox row const renderItem = ({item}): ReactElement => { const isSelected = selectedValues.includes(item.value); + const isProtected = protectedLanguages.includes(item.value); const isUSEngLangSelected = selectedValues.includes('en-US'); const isINEngLangSelected = selectedValues.includes('en-IN'); const isDisabled = - (!isSelected && selectedValues.length === 2) || + (!isSelected && selectedValues.length === maxAllowedSelection) || (item.value === 'en-US' && isINEngLangSelected) || - (item.value === 'en-IN' && isUSEngLangSelected); + (item.value === 'en-IN' && isUSEngLangSelected) || + (isSelected && isProtected); setError(isDisabled || selectedValues.length === 0); @@ -119,8 +126,14 @@ const DropdownMulti: FC = ({ onItemPress(item)} /> @@ -144,26 +157,38 @@ const DropdownMulti: FC = ({ ); }; - const selectedLabels = selectedValues.map((value) => { - const selectedLanguage = data.find((item) => item.value === value); + const selectedLabels = selectedValues.map(value => { + const selectedLanguage = data.find(item => item.value === value); + const isProtected = protectedLanguages.includes(value); return selectedLanguage ? ( { - const updatedValues = selectedValues.filter( - (value) => value !== selectedLanguage.value, - ); - setSelectedValues(updatedValues); + if (!isProtected) { + const updatedValues = selectedValues.filter( + value => value !== selectedLanguage.value, + ); + setSelectedValues(updatedValues); + } }}> - + {selectedLanguage.label} @@ -210,13 +235,16 @@ const DropdownMulti: FC = ({ <> )} {/* Dropdown Text */} - {formattedSelectedLanguages} - + {/* Dropdown end Icon */} @@ -274,6 +302,10 @@ const styles = StyleSheet.create({ alignSelf: 'center', flexDirection: 'row', }, + scrollContainer: { + alignItems: 'center', + flexDirection: 'row', + }, dropdownOptionText: { textAlign: 'left', fontFamily: ThemeConfig.FontFamily.sansPro, @@ -281,7 +313,6 @@ const styles = StyleSheet.create({ fontSize: ThemeConfig.FontSize.normal, color: $config.FONT_COLOR, marginLeft: 4, - flex: 1, }, dropdownIconContainer: { alignSelf: 'center', @@ -311,7 +342,7 @@ const styles = StyleSheet.create({ paddingLeft: 8, backgroundColor: $config.CARD_LAYER_4_COLOR, borderRadius: 6, - flex: 1, + maxWidth: 140, justifyContent: 'flex-start', alignItems: 'center', flexDirection: 'row', @@ -344,6 +375,11 @@ const styles = StyleSheet.create({ justifyContent: 'center', alignItems: 'center', }, + protectedItemText: { + paddingVertical: 12, + opacity: 0.6, + fontStyle: 'italic', + }, }); export default DropdownMulti; diff --git a/template/src/components/Controls.tsx b/template/src/components/Controls.tsx index 0828af9e2..fe98d51d4 100644 --- a/template/src/components/Controls.tsx +++ b/template/src/components/Controls.tsx @@ -871,10 +871,10 @@ const MoreButton = (props: {fields: ToolbarMoreButtonDefaultFields}) => { } try { - const res = await start(language); + const res = await start(language, language); if (res?.message.includes('STARTED')) { // channel is already started now restart - await restart(language); + await restart(language, language); } } catch (error) { logger.error(LogSource.Internals, 'STT', 'error in starting stt', error); @@ -1234,15 +1234,15 @@ const Controls = (props: ControlsProps) => { }; } - Toast.show({ - leadingIconName: 'lang-select', - type: 'info', - text1: heading(prevLang.indexOf('') !== -1 ? 'Set' : 'Changed'), - visibilityTime: 3000, - primaryBtn: null, - secondaryBtn: null, - text2: subheading(subheadingObj), - }); + // Toast.show({ + // leadingIconName: 'lang-select', + // type: 'info', + // text1: heading(prevLang.indexOf('') !== -1 ? 'Set' : 'Changed'), + // visibilityTime: 3000, + // primaryBtn: null, + // secondaryBtn: null, + // text2: subheading(subheadingObj), + // }); setRoomInfo(prev => { return { ...prev, diff --git a/template/src/components/EventsConfigure.tsx b/template/src/components/EventsConfigure.tsx index 61d10b8e0..5a4e7931d 100644 --- a/template/src/components/EventsConfigure.tsx +++ b/template/src/components/EventsConfigure.tsx @@ -591,16 +591,70 @@ const EventsConfigure: React.FC = ({ prevLang, newLang, uid, + remoteLang, }: RoomInfoContextInterface['sttLanguage'] = JSON.parse(data?.payload); - // set this on roominfo then use it in Controls - const sttLangObj = { + + setRoomInfo(prev => { + // Merge remoteLang with existing remoteLang to accumulate protected languages + const existingRemoteLang = prev.sttLanguage?.remoteLang || []; + const newRemoteLang = remoteLang || []; + const mergedRemoteLang = [...new Set([...existingRemoteLang, ...newRemoteLang])]; + + const sttLangObj = { + username, + prevLang, + newLang, // All languages in the channel + uid, + langChanged: true, + remoteLang: mergedRemoteLang, // Accumulated protected languages + }; + + return { + ...prev, + sttLanguage: sttLangObj, + }; + }); + }); + + events.on(EventNames.STT_TRANSLATE_LANGUAGE, data => { + const { username, - prevLang, - newLang, uid, - langChanged: true, - }; + translateConfig, + } = JSON.parse(data?.payload); + setRoomInfo(prev => { + // Merge translate configs with existing configuration + const existingTranslateConfig = prev.sttLanguage?.translateConfig || []; + const newTranslateConfig = translateConfig || []; + + // Merge logic: for each new source language, merge with existing or add new + const mergedTranslateConfig = [...existingTranslateConfig]; + + newTranslateConfig.forEach(newConfig => { + const existingIndex = mergedTranslateConfig.findIndex( + existing => existing.source_lang === newConfig.source_lang + ); + + if (existingIndex !== -1) { + // Same source language - merge target languages + const existingTargets = mergedTranslateConfig[existingIndex].target_lang; + const mergedTargets = [...new Set([...existingTargets, ...newConfig.target_lang])]; + mergedTranslateConfig[existingIndex] = { + ...mergedTranslateConfig[existingIndex], + target_lang: mergedTargets, + }; + } else { + // Different source language - add new config + mergedTranslateConfig.push(newConfig); + } + }); + + const sttLangObj = { + ...prev.sttLanguage, + translateConfig: mergedTranslateConfig, + }; + return { ...prev, sttLanguage: sttLangObj, @@ -873,6 +927,7 @@ const EventsConfigure: React.FC = ({ events.off(EventNames.BOARD_COLOR_CHANGED); events.off(EventNames.STT_ACTIVE); events.off(EventNames.STT_LANGUAGE); + events.off(EventNames.STT_TRANSLATE_LANGUAGE); }; }, []); diff --git a/template/src/components/room-info/useRoomInfo.tsx b/template/src/components/room-info/useRoomInfo.tsx index 4e4883313..82d6c8c0f 100644 --- a/template/src/components/room-info/useRoomInfo.tsx +++ b/template/src/components/room-info/useRoomInfo.tsx @@ -103,6 +103,11 @@ export interface RoomInfoContextInterface { newLang?: LanguageType[]; uid?: UidType; langChanged?: boolean; + remoteLang?: LanguageType[]; + translateConfig?: { + target_lang: string[]; + source_lang: string; + }[]; }; isSTTActive?: boolean; roomPreference?: joinRoomPreference; diff --git a/template/src/language/default-labels/videoCallScreenLabels.ts b/template/src/language/default-labels/videoCallScreenLabels.ts index fe1e96211..b68d48081 100644 --- a/template/src/language/default-labels/videoCallScreenLabels.ts +++ b/template/src/language/default-labels/videoCallScreenLabels.ts @@ -199,6 +199,10 @@ export const sttChangeSpokenLanguageText = `${stt}ChangeSpokenLanguageText` as const; export const sttSettingSpokenLanguageText = `${stt}SettingSpokenLanguageText` as const; +export const sttSettingTranslationLanguageText = + `${stt}SettingTranslationLanguageText` as const; +export const sttChangeTranslationLanguageText = + `${stt}ChangeTranslationLanguageText` as const; export const sttTranscriptPanelHeaderText = `${stt}TranscriptPanelHeaderText` as const; export const sttDownloadBtnText = `${stt}DownloadBtnText` as const; @@ -638,6 +642,8 @@ export interface I18nVideoCallScreenLabelsInterface { [sttChangeSpokenLanguageText]?: I18nBaseType; [sttSettingSpokenLanguageText]?: I18nBaseType; + [sttSettingTranslationLanguageText]?: I18nBaseType; + [sttChangeTranslationLanguageText]?: I18nBaseType; [sttTranscriptPanelHeaderText]?: I18nBaseType; [sttDownloadBtnText]?: I18nBaseType; [sttDownloadTranscriptBtnText]?: I18nBaseType; @@ -1032,7 +1038,7 @@ export const VideoCallScreenLabels: I18nVideoCallScreenLabelsInterface = { 'What language(s) are being spoken by everyone in this meeting?', [sttChangeLanguagePopupPrimaryBtnText]: 'CONFIRM', [sttChangeLanguagePopupDropdownInfo]: - 'You can choose a maximum of two languages', + 'You can choose a maximum of four languages', [sttChangeLanguagePopupDropdownError]: 'Choose at least one language to proceed', [sttChangeSpokenLanguageText]: 'Change Spoken Language', @@ -1041,6 +1047,8 @@ export const VideoCallScreenLabels: I18nVideoCallScreenLabelsInterface = { [sttDownloadBtnText]: 'Download', [sttDownloadTranscriptBtnText]: 'Download Transcript', [sttSettingSpokenLanguageText]: 'Setting Spoken Language', + [sttSettingTranslationLanguageText]: 'Setting Translation Language', + [sttChangeTranslationLanguageText]: 'Change Translation Language', [sttLanguageChangeInProgress]: 'Language Change is in progress...', [peoplePanelHeaderText]: 'People', diff --git a/template/src/pages/video-call/SidePanelHeader.tsx b/template/src/pages/video-call/SidePanelHeader.tsx index a173cdeac..73c55fb9b 100644 --- a/template/src/pages/video-call/SidePanelHeader.tsx +++ b/template/src/pages/video-call/SidePanelHeader.tsx @@ -26,7 +26,7 @@ import {calculatePosition, isMobileUA} from '../../utils/common'; import LanguageSelectorPopup from '../../subComponents/caption/LanguageSelectorPopup'; import useSTTAPI from '../../subComponents/caption/useSTTAPI'; import useGetName from '../../utils/useGetName'; -import {LanguageType} from '../../subComponents/caption/utils'; +import {LanguageType, mergeTranslationConfigs, TranslateConfig} from '../../subComponents/caption/utils'; import {useRoomInfo, usePreCall} from 'customization-api'; import useTranscriptDownload from '../../subComponents/caption/useTranscriptDownload'; import {useVB} from '../../components/virtual-background/useVB'; @@ -42,10 +42,12 @@ import { chatPanelPrivateTabText, peoplePanelHeaderText, sttChangeSpokenLanguageText, + sttChangeTranslationLanguageText, sttDownloadTranscriptBtnText, sttTranscriptPanelHeaderText, } from '../../language/default-labels/videoCallScreenLabels'; import {logger, LogSource} from '../../logger/AppBuilderLogger'; +import {TranslateActionMenu} from '../../subComponents/caption/CaptionContainer'; export const SettingsHeader = props => { const {setSidePanel} = useSidePanel(); @@ -263,6 +265,7 @@ const TranscriptHeaderActionMenu = (props: TranscriptHeaderActionMenuProps) => { meetingTranscript, isLangChangeInProgress, setLanguage, + selectedTranslationLanguage } = useCaption(); const {downloadTranscript} = useTranscriptDownload(); const [modalPosition, setModalPosition] = React.useState({}); @@ -270,20 +273,25 @@ const TranscriptHeaderActionMenu = (props: TranscriptHeaderActionMenuProps) => { const {width: globalWidth, height: globalHeight} = useWindowDimensions(); const [isLanguagePopupOpen, setLanguagePopup] = React.useState(false); + const [isTranslateMenuOpen, setTranslateMenuOpen] = + React.useState(false); const {restart} = useSTTAPI(); const username = useGetName(); const actionMenuitems: ActionMenuItem[] = []; const { - data: {isHost}, + data: {isHost},sttLanguage } = useRoomInfo(); const downloadTranscriptLabel = useString(sttDownloadTranscriptBtnText)(); const changeSpokenLanguage = useString( sttChangeSpokenLanguageText, )(); + const changeTranslationLanguage = useString( + sttChangeTranslationLanguageText, + )(); isHost && actionMenuitems.push({ - icon: 'lang-select', + icon: 'globe', iconColor: $config.SECONDARY_ACTION_COLOR, textColor: $config.FONT_COLOR, title: changeSpokenLanguage + ' ', @@ -294,6 +302,18 @@ const TranscriptHeaderActionMenu = (props: TranscriptHeaderActionMenuProps) => { }, }); + actionMenuitems.push({ + icon: 'lang-select', + iconColor: $config.SECONDARY_ACTION_COLOR, + textColor: $config.FONT_COLOR, + title: changeTranslationLanguage, + disabled: false, + onPress: () => { + setActionMenuVisible(false); + setTranslateMenuOpen(true); + }, + }); + actionMenuitems.push({ icon: 'download', iconColor: $config.SECONDARY_ACTION_COLOR, @@ -306,10 +326,33 @@ const TranscriptHeaderActionMenu = (props: TranscriptHeaderActionMenuProps) => { }, }); - const onLanguageChange = (langChanged = false, language: LanguageType[]) => { + const onLanguageChange = (langChanged = false, allLanguages: LanguageType[], + userOwnLanguages?: LanguageType[]) => { + console.log('TranscriptHeaderActionMenu - onLanguageChange - selectedTranslationLanguage, sttLanguage:', selectedTranslationLanguage, sttLanguage); setLanguagePopup(false); if (langChanged) { - restart(language) + // If user has translation selected, we need to merge translation configs + let translateConfigToPass = null; + + if (selectedTranslationLanguage && selectedTranslationLanguage !== '') { + // Get existing translate config from room state + const existingTranslateConfig = sttLanguage?.translateConfig || []; + + // Use utility function to merge translation configs + const mergedTranslateConfig = mergeTranslationConfigs( + existingTranslateConfig, + userOwnLanguages || [], + selectedTranslationLanguage, + ); + + translateConfigToPass = { + translate_config: mergedTranslateConfig, + userSelectedTranslation: selectedTranslationLanguage, + }; + } + + // Pass translation config to restart if available + restart(allLanguages, userOwnLanguages, translateConfigToPass) .then(() => { logger.debug( LogSource.Internals, @@ -370,6 +413,12 @@ const TranscriptHeaderActionMenu = (props: TranscriptHeaderActionMenuProps) => { setModalVisible={setLanguagePopup} onConfirm={onLanguageChange} /> + + ); }; diff --git a/template/src/rtm-events/constants.ts b/template/src/rtm-events/constants.ts index f65b84b4e..1aa9bf6c0 100644 --- a/template/src/rtm-events/constants.ts +++ b/template/src/rtm-events/constants.ts @@ -30,6 +30,7 @@ const VIDEO_MEETING_ATTENDEE = 'VIDEO_MEETING_ATTENDEE'; // 6. STT const STT_ACTIVE = 'STT_IS_ACTIVE'; const STT_LANGUAGE = 'STT_LANGUAGE_CHANGED'; +const STT_TRANSLATE_LANGUAGE = 'STT_TRANSLATE_LANGUAGE_CHANGED'; // 7. WAITING ROOM const WAITING_ROOM_REQUEST = 'WAITING_ROOM_REQUEST'; const WAITING_ROOM_RESPONSE = 'WAITING_ROOM_RESPONSE'; @@ -53,6 +54,7 @@ const EventNames = { VIDEO_MEETING_ATTENDEE, STT_ACTIVE, STT_LANGUAGE, + STT_TRANSLATE_LANGUAGE, WAITING_ROOM_REQUEST, WAITING_ROOM_RESPONSE, WAITING_ROOM_STATUS_UPDATE, diff --git a/template/src/subComponents/caption/Caption.tsx b/template/src/subComponents/caption/Caption.tsx index 9784fdc67..1ca811d1a 100644 --- a/template/src/subComponents/caption/Caption.tsx +++ b/template/src/subComponents/caption/Caption.tsx @@ -8,7 +8,10 @@ import {isWebInternal} from '../../utils/common'; import useStreamMessageUtils from './useStreamMessageUtils'; import hexadecimalTransparency from '../../utils/hexadecimalTransparency'; import {useString} from '../../utils/useString'; -import {sttSettingSpokenLanguageText} from '../../language/default-labels/videoCallScreenLabels'; +import { + sttSettingSpokenLanguageText, + sttSettingTranslationLanguageText, +} from '../../language/default-labels/videoCallScreenLabels'; export type WebStreamMessageArgs = [number, Uint8Array]; export type NativeStreamMessageArgs = [ @@ -33,6 +36,7 @@ const Caption: React.FC = ({ const {RtcEngineUnsafe} = useRtc(); const { isLangChangeInProgress, + isTranslationChangeInProgress, captionObj, //state for current live caption for all users isSTTListenerAdded, setIsSTTListenerAdded, @@ -40,6 +44,7 @@ const Caption: React.FC = ({ prevSpeakerRef, } = useCaption(); const ssLabel = useString(sttSettingSpokenLanguageText)(); + const stLabel = useString(sttSettingTranslationLanguageText)(); const {streamMessageCallback} = useStreamMessageUtils(); const {defaultContent} = useContent(); @@ -76,6 +81,16 @@ const Caption: React.FC = ({ /> ); + if (isTranslationChangeInProgress) + return ( + + ); + console.log('current speaker uid', activeSpeakerRef.current); console.log('prev current uid ', prevSpeakerRef.current); @@ -106,6 +121,7 @@ const Caption: React.FC = ({ = ({ = ({ captionTextStyle = {}, }) => { const moreIconRef = React.useRef(null); + const langSelectIconRef = React.useRef(null); const [actionMenuVisible, setActionMenuVisible] = React.useState(false); + const [langActionMenuVisible, setLangActionMenuVisible] = + React.useState(false); const [isHovered, setIsHovered] = React.useState(false); const isDesktop = useIsDesktop(); const isSmall = useIsSmall(); @@ -115,11 +120,23 @@ const CaptionContainer: React.FC = ({ btnRef={moreIconRef} /> + + {(isHovered || isMobileUA()) && !isLangChangeInProgress && ( - + <> + + + )} ((props, ref) => { ); }); +const LanguageSelectMenu = React.forwardRef( + (props, ref) => { + const {setActionMenuVisible} = props; + const isMobile = isMobileUA(); + return ( + + { + setActionMenuVisible(true); + }} + /> + + ); + }, +); + interface CaptionsActionMenuProps { actionMenuVisible: boolean; setActionMenuVisible: (actionMenuVisible: boolean) => void; @@ -212,6 +278,7 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => { language: prevLang, isLangChangeInProgress, setLanguage, + selectedTranslationLanguage, } = useCaption(); const actionMenuitems: ActionMenuItem[] = []; const [modalPosition, setModalPosition] = React.useState({}); @@ -223,6 +290,7 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => { const username = useGetName(); const { data: {isHost}, + sttLanguage, } = useRoomInfo(); const changeSpokenLangLabel = useString( @@ -234,7 +302,7 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => { // only Host is authorized to start/stop stt isHost && actionMenuitems.push({ - icon: 'lang-select', + icon: 'globe', iconColor: $config.SECONDARY_ACTION_COLOR, textColor: $config.FONT_COLOR, title: changeSpokenLangLabel + ' ', @@ -256,15 +324,46 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => { }, }); - const onLanguageChange = (langChanged = false, language: LanguageType[]) => { + const onLanguageChange = ( + langChanged = false, + allLanguages: LanguageType[], + userOwnLanguages?: LanguageType[], + ) => { + console.log( + `CaptionContainer - onLanguageChange - selectedTranslationLanguage, sttLanguage:`, + selectedTranslationLanguage, + sttLanguage, + ); setLanguagePopup(false); if (langChanged) { logger.log( LogSource.Internals, 'STT', - `Language changed to ${language}. Restarting STT`, + `Language changed to ${allLanguages}. Restarting STT`, ); - restart(language) + + // If user has translation selected, we need to merge translation configs + let translateConfigToPass = null; + + if (selectedTranslationLanguage && selectedTranslationLanguage !== '') { + // Get existing translate config from room state + const existingTranslateConfig = sttLanguage?.translateConfig || []; + + // Use utility function to merge translation configs + const mergedTranslateConfig = mergeTranslationConfigs( + existingTranslateConfig, + userOwnLanguages || [], + selectedTranslationLanguage, + ); + + translateConfigToPass = { + translate_config: mergedTranslateConfig, + userSelectedTranslation: selectedTranslationLanguage, + }; + } + + // Pass translation config to restart if available + restart(allLanguages, userOwnLanguages, translateConfigToPass) .then(() => { logger.debug( LogSource.Internals, @@ -328,6 +427,188 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => { ); }; +export interface TranslateActionMenuProps { + actionMenuVisible: boolean; + setActionMenuVisible: (actionMenuVisible: boolean) => void; + btnRef: React.RefObject; +} + +export const TranslateActionMenu = (props: TranslateActionMenuProps) => { + const {actionMenuVisible, setActionMenuVisible, btnRef} = props; + const [modalPosition, setModalPosition] = React.useState({}); + const [isPosCalculated, setIsPosCalculated] = React.useState(false); + const {width: globalWidth, height: globalHeight} = useWindowDimensions(); + const { + language: currentSpokenLanguages, + selectedTranslationLanguage, + setMeetingTranscript, + } = useCaption(); + const {update} = useSTTAPI(); + const localUid = useLocalUid(); + const {sttLanguage} = useRoomInfo(); + + const actionMenuitems: ActionMenuItem[] = []; + + actionMenuitems.push({ + iconColor: $config.SECONDARY_ACTION_COLOR, + textColor: $config.FONT_COLOR, + title: 'Translate to', + iconPosition: 'end', + disabled: true, + onPress: () => {}, + }); + + const handleTranslationToggle = async (targetLanguage: string) => { + try { + const prevTranslationLanguage = selectedTranslationLanguage; + + if (targetLanguage === '') { + // turn off translation - todo test + await update({ + translate_config: [], + lang: currentSpokenLanguages, + userSelectedTranslation: '', // Empty string for "off" + isTranslationChange: true, + }); + } else { + // Get existing translate config from room state + const existingTranslateConfig = sttLanguage?.translateConfig || []; + + // Use utility function to merge translation configs + const mergedTranslateConfig = mergeTranslationConfigs( + existingTranslateConfig, + currentSpokenLanguages, + targetLanguage, + ); + + await update({ + translate_config: mergedTranslateConfig, + lang: currentSpokenLanguages, + userSelectedTranslation: targetLanguage, + isTranslationChange: true, + }); + } + + // Add translation language change notification to transcript + const getLanguageName = (langCode: string) => { + if (!langCode) return ''; + const lang = langData.find(data => data.value === langCode); + return lang ? lang.label : langCode; + }; + + const actionText = + targetLanguage === '' + ? 'turned off translation' + : prevTranslationLanguage === '' + ? `set the translation language to "${getLanguageName( + targetLanguage, + )}"` + : `changed the translation language from "${getLanguageName( + prevTranslationLanguage, + )}" to "${getLanguageName(targetLanguage)}"`; + + setMeetingTranscript(prev => [ + ...prev, + { + name: 'translationUpdate', + time: new Date().getTime(), + uid: `translationUpdate-${localUid}`, + text: actionText, + }, + ]); + + setActionMenuVisible(false); + } catch (error) { + logger.error( + LogSource.Internals, + 'STT', + 'Failed to update translation configuration', + error, + ); + } + }; + + actionMenuitems.push({ + icon: selectedTranslationLanguage === '' ? 'tick-fill' : undefined, + iconColor: $config.PRIMARY_ACTION_BRAND_COLOR, + textColor: $config.FONT_COLOR, + title: 'Off', + iconPosition: 'end', + onPress: () => handleTranslationToggle(''), + }); + + // Add selected translation language right after "Off" if one is selected + if (selectedTranslationLanguage && selectedTranslationLanguage !== '') { + const selectedLanguage = langData.find( + lang => lang.value === selectedTranslationLanguage, + ); + if (selectedLanguage) { + actionMenuitems.push({ + icon: 'tick-fill', + iconColor: $config.PRIMARY_ACTION_BRAND_COLOR, + textColor: $config.FONT_COLOR, + title: selectedLanguage.label, + iconPosition: 'end', + onPress: () => handleTranslationToggle(selectedLanguage.value), + }); + } + } + + // Add remaining Translation language options (excluding the selected one) + langData.forEach(language => { + if (language.value !== selectedTranslationLanguage) { + actionMenuitems.push({ + icon: undefined, + iconColor: $config.PRIMARY_ACTION_BRAND_COLOR, + textColor: $config.FONT_COLOR, + title: language.label, + iconPosition: 'end', + onPress: () => handleTranslationToggle(language.value), + }); + } + }); + + React.useEffect(() => { + if (actionMenuVisible) { + btnRef?.current?.measure( + ( + _fx: number, + _fy: number, + localWidth: number, + localHeight: number, + px: number, + py: number, + ) => { + const data = calculatePosition({ + px, + py, + localWidth, + localHeight, + globalHeight, + globalWidth, + }); + setModalPosition(data); + setIsPosCalculated(true); + }, + ); + } + }, [actionMenuVisible]); + + return ( + + ); +}; + export default CaptionContainer; const styles = StyleSheet.create({ diff --git a/template/src/subComponents/caption/CaptionIcon.tsx b/template/src/subComponents/caption/CaptionIcon.tsx index 1d1e1737b..abee029fa 100644 --- a/template/src/subComponents/caption/CaptionIcon.tsx +++ b/template/src/subComponents/caption/CaptionIcon.tsx @@ -80,7 +80,7 @@ const CaptionIcon = (props: CaptionIconProps) => { iconButtonProps.toolTipMessage = label; } - const onConfirm = async (langChanged, language) => { + const onConfirm = async (langChanged, language, userOwnLanguages) => { setLanguagePopup(false); closeActionSheet(); isFirstTimePopupOpen.current = false; @@ -89,10 +89,10 @@ const CaptionIcon = (props: CaptionIconProps) => { if (method === 'start' && isSTTActive === true) return; // not triggering the start service if STT Service already started by anyone else in the channel setIsCaptionON(prev => !prev); try { - const res = await start(language); + const res = await start(language, userOwnLanguages); if (res?.message.includes('STARTED')) { // channel is already started now restart - await restart(language); + await restart(language, userOwnLanguages); } } catch (error) { console.log('eror in starting stt', error); diff --git a/template/src/subComponents/caption/CaptionText.tsx b/template/src/subComponents/caption/CaptionText.tsx index 995970b88..564c8b240 100644 --- a/template/src/subComponents/caption/CaptionText.tsx +++ b/template/src/subComponents/caption/CaptionText.tsx @@ -8,12 +8,20 @@ import { import React from 'react'; import ThemeConfig from '../../../src/theme'; +import {useCaption} from './useCaption'; import hexadecimalTransparency from '../../../src/utils/hexadecimalTransparency'; import {isAndroid, isMobileUA} from '../../utils/common'; +type TranslationItem = { + lang: string; + text: string; + isFinal: boolean; +}; + interface CaptionTextProps { user: string; value: string; + translations?: TranslationItem[]; activeSpeakersCount: number; isActiveSpeaker?: boolean; activelinesAvailable?: number; @@ -31,6 +39,7 @@ const MAX_CAPTIONS_LINES_ALLOWED = 3; const CaptionText = ({ user, value, + translations = [], activeSpeakersCount, isActiveSpeaker = false, activelinesAvailable, @@ -41,6 +50,7 @@ const CaptionText = ({ captionTextStyle = {}, }: CaptionTextProps) => { const isMobile = isMobileUA(); + const {selectedTranslationLanguage} = useCaption(); const LINE_HEIGHT = isMobile ? MOBILE_LINE_HEIGHT : DESKTOP_LINE_HEIGHT; @@ -125,17 +135,22 @@ const CaptionText = ({ )) * LINE_HEIGHT, }, ]}> + {/* Transcription */} - {value} + {selectedTranslationLanguage + ? translations.find(t => t.lang === selectedTranslationLanguage) + ?.text || value + : value} @@ -155,8 +170,10 @@ const styles = StyleSheet.create({ }, captionTextContainerStyle: { - overflow: 'hidden', width: '100%', + // flexDirection: 'column', + // justifyContent: 'flex-end', + overflow: 'hidden', position: 'relative', }, @@ -168,6 +185,17 @@ const styles = StyleSheet.create({ bottom: 0, }, + transcriptionText: { + marginBottom: 2, + }, + + translationText: { + fontFamily: ThemeConfig.FontFamily.sansPro, + fontWeight: '300', + color: $config.FONT_COLOR, + marginTop: 1, + }, + captionUserName: { fontWeight: '600', fontFamily: ThemeConfig.FontFamily.sansPro, diff --git a/template/src/subComponents/caption/LanguageSelectorPopup.tsx b/template/src/subComponents/caption/LanguageSelectorPopup.tsx index 65520cee0..4d1f74641 100644 --- a/template/src/subComponents/caption/LanguageSelectorPopup.tsx +++ b/template/src/subComponents/caption/LanguageSelectorPopup.tsx @@ -1,4 +1,4 @@ -import React, {SetStateAction, useContext} from 'react'; +import React, {SetStateAction} from 'react'; import {StyleSheet, Text, View} from 'react-native'; import Spacer from '../../atoms/Spacer'; import Popup from '../../atoms/Popup'; @@ -12,6 +12,7 @@ import hexadecimalTransparency from '../../utils/hexadecimalTransparency'; import Loading from '../Loading'; import {LanguageType} from './utils'; import {useString} from '../../utils/useString'; +import {useRoomInfo} from '../../components/room-info/useRoomInfo'; import { sttChangeLanguagePopupDropdownError, sttChangeLanguagePopupDropdownInfo, @@ -25,7 +26,7 @@ import {cancelText} from '../../language/default-labels/commonLabels'; interface LanguageSelectorPopup { modalVisible: boolean; setModalVisible: React.Dispatch>; - onConfirm: (param: boolean, lang: LanguageType[]) => void; + onConfirm: (param: boolean, lang: LanguageType[], userOwnLang?: LanguageType[]) => void; isFirstTimePopupOpen?: boolean; } @@ -40,16 +41,32 @@ const LanguageSelectorPopup = (props: LanguageSelectorPopup) => { const ddError = useString(sttChangeLanguagePopupDropdownError)(); const ddInfo = useString(sttChangeLanguagePopupDropdownInfo)(); const languageChangeInProgress = useString(sttLanguageChangeInProgress)(); - const {language, setLanguage, isLangChangeInProgress, isSTTActive} = - useCaption(); + const {language, isLangChangeInProgress, isSTTActive} = useCaption(); + + const {sttLanguage} = useRoomInfo(); + + // Get protected languages from accumulated remoteLang + const protectedLanguages = React.useMemo(() => { + return sttLanguage?.remoteLang || []; + }, [sttLanguage?.remoteLang]); const [error, setError] = React.useState(false); const [selectedValues, setSelectedValues] = React.useState(language); const isNotValidated = - isOpen && (selectedValues.length === 0 || selectedValues.length === 2); + isOpen && (selectedValues.length === 0 || selectedValues.length === 4); + + // Initialize selectedValues with current languages plus protected languages + React.useEffect(() => { + console.log('LanguagePopup Debug - language:', language); + console.log('LanguagePopup Debug - protectedLanguages:', protectedLanguages); + const combinedLanguages = [...language, ...protectedLanguages]; + // Remove duplicates + const uniqueLanguages = Array.from(new Set(combinedLanguages)); + console.log('LanguagePopup Debug - uniqueLanguages:', uniqueLanguages); + setSelectedValues(uniqueLanguages); + }, [language, protectedLanguages]); - // React.useEffect(() => setSelectedValues(() => language), []); return ( { setError={setError} isOpen={isOpen} setIsOpen={setIsOpen} + maxAllowedSelection={4} + protectedLanguages={protectedLanguages} /> @@ -118,7 +137,7 @@ const LanguageSelectorPopup = (props: LanguageSelectorPopup) => { paddingVertical: 12, paddingHorizontal: 12, }} - disabled={selectedValues.length === 0} + disabled={selectedValues.length === 0 } text={ConfirmBtnLabel} textStyle={styles.btnText} onPress={() => { @@ -126,13 +145,19 @@ const LanguageSelectorPopup = (props: LanguageSelectorPopup) => { console.log(language); if (selectedValues.length === 0) { - // setError(true); return; } - const lang1 = language.slice().sort().toString(); - const lang2 = selectedValues.slice().sort().toString(); - const isLangChanged = lang1 !== lang2 || !isSTTActive; - props.onConfirm(isLangChanged, selectedValues); + + // Get user's own languages (not protected) + const userOwnLanguages = selectedValues.filter(lang => !protectedLanguages.includes(lang)); + + // Compare current languages with new selection (ignoring order) + const currentLangs = language.slice().sort().join(','); + const newLangs = selectedValues.slice().sort().join(','); + const isLangChanged = currentLangs !== newLangs || !isSTTActive; + + // Pass all selected languages for STT API, and user's own for RTM + props.onConfirm(isLangChanged, selectedValues, userOwnLanguages); }} /> @@ -162,9 +187,10 @@ const styles = StyleSheet.create({ }, contentContainer: { padding: 24, - maxWidth: 446, + maxWidth: 500, width: '100%', }, + heading: { fontFamily: ThemeConfig.FontFamily.sansPro, fontWeight: '600', diff --git a/template/src/subComponents/caption/Transcript.tsx b/template/src/subComponents/caption/Transcript.tsx index 8c9662cc3..19fff88ab 100644 --- a/template/src/subComponents/caption/Transcript.tsx +++ b/template/src/subComponents/caption/Transcript.tsx @@ -108,19 +108,34 @@ const Transcript = (props: TranscriptProps) => { iconType="plain" iconSize={20} tintColor={$config.PRIMARY_ACTION_BRAND_COLOR} - name={'lang-select'} + name={'globe'} /> {defaultContent[item?.uid?.split('-')[1]].name + ' ' + item.text} + ) : item.uid.toString().indexOf('translationUpdate') !== -1 ? ( + + + + + {defaultContent[item?.uid?.split('-')[1]].name + ' has ' + item.text} + + ) : ( ); }; @@ -164,9 +179,20 @@ const Transcript = (props: TranscriptProps) => { const handleSearch = (text: string) => { setSearchQuery(text); // Filter the data based on the search query - const filteredResults = meetingTranscript.filter(item => - item.text.toLowerCase().includes(text.toLowerCase()), - ); + const filteredResults = meetingTranscript.filter(item => { + const searchText = text.toLowerCase(); + // Search in original text + if (item.text.toLowerCase().includes(searchText)) { + return true; + } + // Search in translations if available + if (item.translations) { + return item.translations.some(translation => + translation.text.toLowerCase().includes(searchText) + ); + } + return false; + }); setShowButton(false); setSearchResults(filteredResults); // Scroll to the top of the FlatList when searching diff --git a/template/src/subComponents/caption/TranscriptIcon.tsx b/template/src/subComponents/caption/TranscriptIcon.tsx index 0fe41e1b0..c819551fe 100644 --- a/template/src/subComponents/caption/TranscriptIcon.tsx +++ b/template/src/subComponents/caption/TranscriptIcon.tsx @@ -85,7 +85,7 @@ const TranscriptIcon = (props: TranscriptIconProps) => { iconButtonProps.toolTipMessage = label(isTranscriptON); } - const onConfirm = async (langChanged, language) => { + const onConfirm = async (langChanged, language, userOwnLanguages) => { setLanguagePopup(false); isFirstTimePopupOpen.current = false; @@ -98,10 +98,10 @@ const TranscriptIcon = (props: TranscriptIconProps) => { setSidePanel(SidePanelType.None); } try { - const res = await start(language); + const res = await start(language, userOwnLanguages); if (res?.message.includes('STARTED')) { // channel is already started now restart - await restart(language); + await restart(language, userOwnLanguages); } } catch (error) { console.log('eror in starting stt', error); diff --git a/template/src/subComponents/caption/TranscriptText.tsx b/template/src/subComponents/caption/TranscriptText.tsx index e966df218..c2a70a979 100644 --- a/template/src/subComponents/caption/TranscriptText.tsx +++ b/template/src/subComponents/caption/TranscriptText.tsx @@ -5,22 +5,53 @@ import ThemeConfig from '../../../src/theme'; import hexadecimalTransparency from '../../../src/utils/hexadecimalTransparency'; import {formatTime} from './utils'; +type TranslationItem = { + lang: string; + text: string; + isFinal: boolean; +}; + interface TranscriptTextProps { user: string; time: number; value: string; + translations?: TranslationItem[]; searchQuery?: string; + selectedTranslationLanguage?: string; } export const TranscriptText = ({ user, time, value, + translations = [], searchQuery = '', + selectedTranslationLanguage: storedTranslationLanguage, }: TranscriptTextProps) => { const t = time ? formatTime(Number(time)) : ''; + + // text to display based on stored translation language + const getDisplayText = () => { + if (!storedTranslationLanguage) { + return value; // no translation selected, show original + } + + // find translation for the stored language + const currentTranslation = translations.find( + t => t.lang === storedTranslationLanguage, + ); + if (currentTranslation?.text) { + return currentTranslation.text; + } + + // if stored language not available, show original + return value; + }; + + const displayText = getDisplayText(); + const regex = searchQuery ? new RegExp(`(${searchQuery})`, 'gi') : ' '; - const parts = value.split(regex); + const parts = displayText.split(regex); return ( diff --git a/template/src/subComponents/caption/proto/ptoto.js b/template/src/subComponents/caption/proto/ptoto.js index dc1bd7bc0..e0786409f 100644 --- a/template/src/subComponents/caption/proto/ptoto.js +++ b/template/src/subComponents/caption/proto/ptoto.js @@ -1,9 +1,9 @@ /*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/ "use strict"; -var $protobuf = require("protobufjs/light"); +var $protobuf = require("protobufjs"); -var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $protobuf.Root())) +var $protobufRoot = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root())) .addJSON({ agora: { nested: { @@ -55,7 +55,24 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr rule: "repeated", type: "Word", id: 10 - } + }, + end_of_segment: { + type: "bool", + id: 11 + }, + duration_ms: { + type: "int32", + id: 12 + }, + data_type: { + type: "string", + id: 13 + }, + trans: { + rule: "repeated", + type: "Translation", + id: 14 + }, } }, Word: { @@ -81,6 +98,23 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr id: 5 } } + }, + Translation: { + fields: { + isFinal: { + type: "bool", + id: 1 + }, + lang: { + type: "string", + id: 2 + }, + texts: { + rule: "repeated", + type: "string", + id: 3 + } + } } } } @@ -88,4 +122,4 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr } }); -module.exports = $root; +module.exports = $protobufRoot; diff --git a/template/src/subComponents/caption/proto/test.proto b/template/src/subComponents/caption/proto/test.proto index b3eec2145..f8e35fdc2 100644 --- a/template/src/subComponents/caption/proto/test.proto +++ b/template/src/subComponents/caption/proto/test.proto @@ -1,23 +1,38 @@ syntax = "proto3"; -package agora.audio2text; -option java_package = "io.agora.rtc.audio2text"; -option java_outer_classname = "Audio2TextProtobuffer"; + +package Agora.SpeechToText; +option objc_class_prefix = "Stt"; +option csharp_namespace = "AgoraSTTSample.Protobuf"; +option java_package = "io.agora.rtc.speech2text"; +option java_outer_classname = "AgoraSpeech2TextProtobuffer"; + message Text { - int32 vendor = 1; - int32 version = 2; - int32 seqnum = 3; - uint32 uid = 4; - int32 flag = 5; - int64 time = 6; - int32 lang = 7; - int32 starttime = 8; - int32 offtime = 9; - repeated Word words = 10; + int32 vendor = 1; + int32 version = 2; + int32 seqnum = 3; + int64 uid = 4; + int32 flag = 5; + int64 time = 6; + int32 lang = 7; + int32 starttime = 8; + int32 offtime = 9; + repeated Word words = 10; + bool end_of_segment = 11; + int32 duration_ms = 12; + string data_type = 13; + repeated Translation trans = 14; + string culture = 15; + int64 text_ts = 16; } message Word { - string text = 1; - int32 start_ms = 2; - int32 duration_ms = 3; - bool is_final = 4; - double confidence = 5; -} \ No newline at end of file + string text = 1; + int32 start_ms = 2; + int32 duration_ms = 3; + bool is_final = 4; + double confidence = 5; +} +message Translation { + bool is_final = 1; + string lang = 2; + repeated string texts = 3; +} diff --git a/template/src/subComponents/caption/useCaption.tsx b/template/src/subComponents/caption/useCaption.tsx index a924d4d2c..24312e037 100644 --- a/template/src/subComponents/caption/useCaption.tsx +++ b/template/src/subComponents/caption/useCaption.tsx @@ -2,15 +2,26 @@ import {createHook} from 'customization-implementation'; import React from 'react'; import {LanguageType} from './utils'; +type TranslationItem = { + lang: string; + text: string; + isFinal: boolean; +}; + export type TranscriptItem = { uid: string; time: number; text: string; + translations?: TranslationItem[]; + // Stores which translation language was active when this transcript was created + // This preserves historical context when users switch translation languages mid-meeting + selectedTranslationLanguage?: string; }; type CaptionObj = { [key: string]: { text: string; + translations: TranslationItem[]; lastUpdated: number; }; }; @@ -40,6 +51,10 @@ export const CaptionContext = React.createContext<{ isLangChangeInProgress: boolean; setIsLangChangeInProgress: React.Dispatch>; + // holds status of translation language change process + isTranslationChangeInProgress: boolean; + setIsTranslationChangeInProgress: React.Dispatch>; + // holds live captions captionObj: CaptionObj; setCaptionObj: React.Dispatch>; @@ -50,6 +65,11 @@ export const CaptionContext = React.createContext<{ activeSpeakerRef: React.MutableRefObject; prevSpeakerRef: React.MutableRefObject; + + selectedTranslationLanguage: string; + setSelectedTranslationLanguage: React.Dispatch>; + // Ref for translation language - prevents stale closures in callbacks + selectedTranslationLanguageRef: React.MutableRefObject; }>({ isCaptionON: false, setIsCaptionON: () => {}, @@ -63,12 +83,17 @@ export const CaptionContext = React.createContext<{ setMeetingTranscript: () => {}, isLangChangeInProgress: false, setIsLangChangeInProgress: () => {}, + isTranslationChangeInProgress: false, + setIsTranslationChangeInProgress: () => {}, captionObj: {}, setCaptionObj: () => {}, isSTTListenerAdded: false, setIsSTTListenerAdded: () => {}, activeSpeakerRef: {current: ''}, prevSpeakerRef: {current: ''}, + selectedTranslationLanguage: '', + setSelectedTranslationLanguage: () => {}, + selectedTranslationLanguageRef: {current: ''}, }); interface CaptionProviderProps { @@ -86,6 +111,8 @@ const CaptionProvider: React.FC = ({ const [language, setLanguage] = React.useState<[LanguageType]>(['']); const [isLangChangeInProgress, setIsLangChangeInProgress] = React.useState(false); + const [isTranslationChangeInProgress, setIsTranslationChangeInProgress] = + React.useState(false); const [meetingTranscript, setMeetingTranscript] = React.useState< TranscriptItem[] >([]); @@ -95,9 +122,12 @@ const CaptionProvider: React.FC = ({ const [activeSpeakerUID, setActiveSpeakerUID] = React.useState(''); const [prevActiveSpeakerUID, setPrevActiveSpeakerUID] = React.useState(''); + const [selectedTranslationLanguage, setSelectedTranslationLanguage] = + React.useState(''); const activeSpeakerRef = React.useRef(''); const prevSpeakerRef = React.useRef(''); + const selectedTranslationLanguageRef = React.useRef(''); return ( = ({ setMeetingTranscript, isLangChangeInProgress, setIsLangChangeInProgress, + isTranslationChangeInProgress, + setIsTranslationChangeInProgress, captionObj, setCaptionObj, isSTTListenerAdded, setIsSTTListenerAdded, activeSpeakerRef, prevSpeakerRef, + selectedTranslationLanguage, + setSelectedTranslationLanguage, + selectedTranslationLanguageRef, }}> {children} diff --git a/template/src/subComponents/caption/useSTTAPI.tsx b/template/src/subComponents/caption/useSTTAPI.tsx index 052bd434b..c11fc2900 100644 --- a/template/src/subComponents/caption/useSTTAPI.tsx +++ b/template/src/subComponents/caption/useSTTAPI.tsx @@ -12,13 +12,32 @@ import {logger, LogSource} from '../../logger/AppBuilderLogger'; import getUniqueID from '../../utils/getUniqueID'; interface IuseSTTAPI { - start: (lang: LanguageType[]) => Promise<{message: string} | null>; + start: (lang: LanguageType[], userOwnLang?: LanguageType[]) => Promise<{message: string} | null>; stop: () => Promise; - restart: (lang: LanguageType[]) => Promise; + restart: ( + lang: LanguageType[], + userOwnLang?: LanguageType[], + translationConfig?: { translate_config: any[], userSelectedTranslation: string } + ) => Promise; + update: (params: UpdateParams) => Promise; isAuthorizedSTTUser: () => boolean; isAuthorizedTranscriptUser: () => boolean; } +interface UpdateParams { + // Speaking language (max 4 allowed) + lang?: LanguageType[]; + // Translation configuration parameters + translate_config?: { + target_lang: string[]; + source_lang: string; + }[]; + // User's own selected translation language (for RTM message) + userSelectedTranslation?: string; + // Flag to indicate if this is a translation-only change + isTranslationChange?: boolean; +} + const useSTTAPI = (): IuseSTTAPI => { const {store} = React.useContext(StorageContext); const { @@ -29,9 +48,13 @@ const useSTTAPI = (): IuseSTTAPI => { isSTTActive, setIsSTTActive, setIsLangChangeInProgress, + setIsTranslationChangeInProgress, setLanguage, setMeetingTranscript, setIsSTTError, + setSelectedTranslationLanguage, + // Ref to track translation language - updated synchronously to avoid race conditions + selectedTranslationLanguageRef, } = useCaption(); const currentLangRef = React.useRef([]); @@ -44,21 +67,40 @@ const useSTTAPI = (): IuseSTTAPI => { currentLangRef.current = language; }, [language]); - const apiCall = async (method: string, lang: LanguageType[] = []) => { + const apiCall = async (method: string, payload?: any) => { const requestId = getUniqueID(); const startReqTs = Date.now(); + logger.log( LogSource.NetworkRest, 'stt', - `Trying to ${method} stt for lang ${lang}`, + `Trying to ${method} stt`, { method, - lang, + payload, requestId, startReqTs, }, ); + try { + + let requestBody: any = { + passphrase: roomId?.host || roomId?.attendee || '', + dataStream_uid: 111111, // default bot ID + encryption_mode: $config.ENCRYPTION_ENABLED + ? rtcProps.encryption.mode + : null, + }; + + + if (payload && typeof payload === 'object') { + requestBody = { + ...requestBody, + ...payload, + }; + } + const response = await fetch(`${STT_API_URL}/${method}`, { method: 'POST', headers: { @@ -67,22 +109,17 @@ const useSTTAPI = (): IuseSTTAPI => { 'X-Request-Id': requestId, 'X-Session-Id': logger.getSessionId(), }, - body: JSON.stringify({ - passphrase: roomId?.host || '', - lang: lang, - dataStream_uid: 111111, // bot ID - encryption_mode: $config.ENCRYPTION_ENABLED - ? rtcProps.encryption.mode - : null, - }), + body: JSON.stringify(requestBody), }); + const res = await response.json(); const endReqTs = Date.now(); const latency = endReqTs - startReqTs; + logger.log( LogSource.NetworkRest, 'stt', - `STT API Success - Called ${method} on stt with lang ${lang}`, + `STT API Success - Called ${method}`, { responseData: res, requestId, @@ -98,7 +135,7 @@ const useSTTAPI = (): IuseSTTAPI => { logger.error( LogSource.NetworkRest, 'stt', - `STT API Failure - Called ${method} on stt with lang ${lang}`, + `STT API Failure - Called ${method}`, error, { requestId, @@ -107,21 +144,22 @@ const useSTTAPI = (): IuseSTTAPI => { latency, }, ); + throw error; } }; - const startWithDelay = (lang: LanguageType[]): Promise => + const startWithDelay = (lang: LanguageType[], userOwnLang?: LanguageType[]): Promise => new Promise(resolve => { setTimeout(async () => { - const res = await start(lang); + const res = await start(lang, userOwnLang); resolve(res); }, 1000); // Delay of 1 seconds (1000 milliseconds) to allow existing stt service to fully stop }); - const start = async (lang: LanguageType[]) => { + const start = async (lang: LanguageType[], userOwnLang?: LanguageType[]) => { try { setIsLangChangeInProgress(true); - const res = await apiCall('startv7', lang); + const res = await apiCall('startv7', {lang}); // null means stt startred successfully const isSTTAlreadyActive = res?.error?.message @@ -160,26 +198,33 @@ const useSTTAPI = (): IuseSTTAPI => { 'stt', `stt lang update from: ${language} to ${lang}`, ); - // inform about the language set for stt + // Send RTM message with all languages and user's own selection as remoteLang + // From other users' perspective, userOwnLang are the new remote languages + const userSelectedLang = userOwnLang && userOwnLang.length > 0 ? userOwnLang : []; + events.send( EventNames.STT_LANGUAGE, JSON.stringify({ username: capitalizeFirstLetter(username), uid: localUid, prevLang: language, - newLang: lang, + newLang: lang, // Send all languages + remoteLang: userSelectedLang, // Send user's own selection as remote for others }), PersistanceLevel.Sender, ); + + // Set the user's language state to ALL languages (own + protected) + // This allows the popup to properly identify protected languages setLanguage(lang); // updaing transcript for self const actionText = language.indexOf('') !== -1 - ? `has set the spoken language to "${getLanguageLabel(lang)}" ` + ? `has set the spoken language to "${getLanguageLabel(userSelectedLang)}" ` : `changed the spoken language from "${getLanguageLabel( language, - )}" to "${getLanguageLabel(lang)}" `; + )}" to "${getLanguageLabel(userSelectedLang)}" `; //const msg = `${capitalizeFirstLetter(username)} ${actionText} `; setMeetingTranscript(prev => { return [ @@ -234,11 +279,60 @@ const useSTTAPI = (): IuseSTTAPI => { throw error; } }; - const restart = async (lang: LanguageType[]) => { + const restart = async ( + lang: LanguageType[], + userOwnLang?: LanguageType[], + translationConfig?: { translate_config: any[], userSelectedTranslation: string } + ) => { try { setIsLangChangeInProgress(true); - await stop(); - await startWithDelay(lang); + // await stop(); + + // If translation config is provided, use update method after start + + // await startWithDelay(lang, userOwnLang); + await update({ + ...translationConfig, + lang: lang, + }); + + const userSelectedLang = userOwnLang && userOwnLang.length > 0 ? userOwnLang : []; + + events.send( + EventNames.STT_LANGUAGE, + JSON.stringify({ + username: capitalizeFirstLetter(username), + uid: localUid, + prevLang: language, + newLang: lang, // Send all languages + remoteLang: userSelectedLang, // Send user's own selection as remote for others + }), + PersistanceLevel.Sender, + ); + + setLanguage(lang); + + // updaing transcript for self + const actionText = + language.indexOf('') !== -1 + ? `has set the spoken language to "${getLanguageLabel(userSelectedLang)}" ` + : `changed the spoken language from "${getLanguageLabel( + language, + )}" to "${getLanguageLabel(userSelectedLang)}" `; + //const msg = `${capitalizeFirstLetter(username)} ${actionText} `; + setMeetingTranscript(prev => { + return [ + ...prev, + { + name: 'langUpdate', + time: new Date().getTime(), + uid: `langUpdate-${localUid}`, + text: actionText, + }, + ]; + }); + + return Promise.resolve(); } catch (error) { logger.error( @@ -253,6 +347,94 @@ const useSTTAPI = (): IuseSTTAPI => { } }; + + const update = async (params: UpdateParams) => { + try { + // Use the appropriate progress state based on the type of change + if (params?.isTranslationChange) { + setIsTranslationChangeInProgress(true); + } else { + setIsLangChangeInProgress(true); + } + + logger.log( + LogSource.NetworkRest, + 'stt', + 'Calling STT update method', + { params } + ); + + const res = await apiCall('update', params); + + if (res?.error?.message) { + setIsSTTError(true); + logger.error( + LogSource.NetworkRest, + 'stt', + 'STT update failed', + res?.error, + ); + } else { + logger.log( + LogSource.NetworkRest, + 'stt', + 'STT update successful', + res, + ); + setIsSTTError(false); + + // If language was updated, update local state + if (params.lang) { + setLanguage(params.lang); + } + + // If translation language was updated, update local state and ref + if (params.isTranslationChange && params.userSelectedTranslation !== undefined) { + setSelectedTranslationLanguage(params.userSelectedTranslation); + // Update ref immediately to prevent race conditions + // This ensures callbacks see the updated value right away, before state updates + selectedTranslationLanguageRef.current = params.userSelectedTranslation; + } + + // Send RTM message for translation config sync + if (params.translate_config && params.userSelectedTranslation) { + // Create user's own translation config from userSelectedTranslation + const userOwnTranslateConfig = (params.lang || currentLangRef.current).map(spokenLang => ({ + source_lang: spokenLang, + target_lang: [params.userSelectedTranslation] + })); + + events.send( + EventNames.STT_TRANSLATE_LANGUAGE, + JSON.stringify({ + username: capitalizeFirstLetter(username), + uid: localUid, + translateConfig: userOwnTranslateConfig, // Only user's own choice + }), + PersistanceLevel.Sender, + ); + } + } + + return res; + } catch (error) { + logger.error( + LogSource.NetworkRest, + 'stt', + 'Error in STT update method', + error, + ); + throw error; + } finally { + // Reset the appropriate progress state based on the type of change + if (params?.isTranslationChange) { + setIsTranslationChangeInProgress(false); + } else { + setIsLangChangeInProgress(false); + } + } + }; + // attendee can view option if any host has started STT const isAuthorizedSTTUser = () => $config.ENABLE_STT && @@ -269,6 +451,7 @@ const useSTTAPI = (): IuseSTTAPI => { start, stop, restart, + update, isAuthorizedSTTUser, isAuthorizedTranscriptUser, }; diff --git a/template/src/subComponents/caption/useStreamMessageUtils.native.ts b/template/src/subComponents/caption/useStreamMessageUtils.native.ts index b6371856a..a37222939 100644 --- a/template/src/subComponents/caption/useStreamMessageUtils.native.ts +++ b/template/src/subComponents/caption/useStreamMessageUtils.native.ts @@ -6,10 +6,15 @@ type StreamMessageCallback = (args: [number, Uint8Array]) => void; type FinalListType = { [key: string]: string[]; }; -type TranscriptItem = { - uid: string; - time: number; +type TranslationData = { + lang: string; text: string; + isFinal: boolean; +}; +type FinalTranslationListType = { + [key: string]: { + [lang: string]: string[]; + }; }; const useStreamMessageUtils = (): { @@ -20,11 +25,15 @@ const useStreamMessageUtils = (): { setMeetingTranscript, activeSpeakerRef, prevSpeakerRef, + // Use ref instead of state to avoid stale closure issues + // The ref always has the current value, even in callbacks created at mount time + selectedTranslationLanguageRef, } = useCaption(); let captionStartTime: number = 0; const finalList: FinalListType = {}; const finalTranscriptList: FinalListType = {}; + const finalTranslationList: FinalTranslationListType = {}; const streamMessageCallback: StreamMessageCallback = args => { /* uid - bot which sends stream message in channel @@ -35,6 +44,7 @@ const useStreamMessageUtils = (): { let finalText = ''; // holds final strings let currentFinalText = ''; // holds current caption let isInterjecting = false; + let translations: TranslationData[] = []; const textstream = protoRoot .lookupType('Text') @@ -129,16 +139,51 @@ const useStreamMessageUtils = (): { captionStartTime = null; // Reset start time } + /* Process translations if available */ + if (textstream.translations && textstream.translations.length > 0) { + for (const translation of textstream.translations) { + const translationWords = translation.texts || []; + let translationFinalText = ''; + + for (const word of translationWords) { + if (word.isFinal) { + translationFinalText += word.text; + } + } + + if (translationFinalText) { + // Initialize translation storage for this uid and language + if (!finalTranslationList[textstream.uid]) { + finalTranslationList[textstream.uid] = {}; + } + if (!finalTranslationList[textstream.uid][translation.lang]) { + finalTranslationList[textstream.uid][translation.lang] = []; + } + + finalTranslationList[textstream.uid][translation.lang].push(translationFinalText); + + translations.push({ + lang: translation.lang, + text: finalTranslationList[textstream.uid][translation.lang].join(' '), + isFinal: true, + }); + } + } + } + /* Updating Meeting Transcript */ if (currentFinalText.length) { + // Prepare final translations for transcript + const finalTranslationsForTranscript = translations.filter(t => t.isFinal); + setMeetingTranscript(prevTranscript => { const lastTranscriptIndex = prevTranscript.length - 1; const lastTranscript = lastTranscriptIndex >= 0 ? prevTranscript[lastTranscriptIndex] : null; /* - checking if the last item transcript matches with current uid - If yes then updating the last transcript msg with current text + checking if the last item transcript matches with current uid + If yes then updating the last transcript msg with current text and translations If no then adding a new entry in the transcript */ if (lastTranscript && lastTranscript.uid === textstream.uid) { @@ -146,6 +191,9 @@ const useStreamMessageUtils = (): { ...lastTranscript, //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values text: finalTranscriptList[textstream.uid].join(' '), + translations: finalTranslationsForTranscript, + // preserve the original translation language from when this transcript was created + selectedTranslationLanguage: lastTranscript.selectedTranslationLanguage, }; return [ @@ -164,14 +212,18 @@ const useStreamMessageUtils = (): { uid: textstream.uid, time: new Date().getTime(), text: currentFinalText, + translations: finalTranslationsForTranscript, + // Store the current translation language with this transcript item + // This preserves which translation was active when this text was spoken + selectedTranslationLanguage: selectedTranslationLanguageRef.current, }, ]; } }); } - /* - Previous final words of the uid are prepended and + /* + Previous final words of the uid are prepended and then current non final words so that context of speech is not lost */ const existingStringBuffer = isInterjecting @@ -183,13 +235,45 @@ const useStreamMessageUtils = (): { ? existingStringBuffer + ' ' + latestString : latestString; - // updating the captions + // Process non-final translations for captions + let nonFinalTranslations: TranslationData[] = []; + if (textstream.translations && textstream.translations.length > 0) { + for (const translation of textstream.translations) { + const translationWords = translation.texts || []; + let nonFinalTranslationText = ''; + + for (const word of translationWords) { + if (!word.isFinal) { + nonFinalTranslationText += word.text !== '.' ? word.text : ''; + } + } + + if (nonFinalTranslationText) { + const existingTranslationBuffer = isInterjecting + ? '' + : finalTranslationList[textstream.uid]?.[translation.lang]?.join(' ') || ''; + + const fullTranslationText = existingTranslationBuffer.length > 0 + ? existingTranslationBuffer + ' ' + nonFinalTranslationText + : nonFinalTranslationText; + + nonFinalTranslations.push({ + lang: translation.lang, + text: fullTranslationText, + isFinal: false, + }); + } + } + } + + // updating the captions with translations captionText && setCaptionObj(prevState => { return { ...prevState, [textstream.uid]: { text: captionText, + translations: nonFinalTranslations, lastUpdated: new Date().getTime(), }, }; diff --git a/template/src/subComponents/caption/useStreamMessageUtils.ts b/template/src/subComponents/caption/useStreamMessageUtils.ts index d720260b7..f915eb807 100644 --- a/template/src/subComponents/caption/useStreamMessageUtils.ts +++ b/template/src/subComponents/caption/useStreamMessageUtils.ts @@ -1,4 +1,3 @@ -import React from 'react'; import {useCaption} from './useCaption'; import protoRoot from './proto/ptoto'; import PQueue from 'p-queue'; @@ -7,6 +6,16 @@ type StreamMessageCallback = (args: [number, Uint8Array]) => void; type FinalListType = { [key: string]: string[]; }; +type TranslationData = { + lang: string; + text: string; + isFinal: boolean; +}; +type FinalTranslationListType = { + [key: string]: { + [lang: string]: string[]; + }; +}; const useStreamMessageUtils = (): { streamMessageCallback: StreamMessageCallback; @@ -16,14 +25,16 @@ const useStreamMessageUtils = (): { setMeetingTranscript, activeSpeakerRef, prevSpeakerRef, + // Use ref instead of state to avoid stale closure issues + // The ref always has the current value, even in callbacks created at mount time + selectedTranslationLanguageRef, } = useCaption(); let captionStartTime: number = 0; const finalList: FinalListType = {}; const finalTranscriptList: FinalListType = {}; + const finalTranslationList: FinalTranslationListType = {}; const queue = new PQueue({concurrency: 1}); - let counter = 0; - let lastOfftime = 0; const streamMessageCallback: StreamMessageCallback = args => { const queueCallback = (args1: [number, Uint8Array]) => { @@ -35,10 +46,12 @@ const useStreamMessageUtils = (): { let finalText = ''; // holds final strings let currentFinalText = ''; // holds current caption let isInterjecting = false; + let translations: TranslationData[] = []; // holds translation data const textstream = protoRoot - .lookupType('Text') + .lookupType('agora.audio2text.Text') .decode(payload as Uint8Array) as any; + console.log('stt v7 textstream', textstream); //console.log('STT - Parsed Textstream : ', textstream); // console.log( @@ -84,7 +97,7 @@ const useStreamMessageUtils = (): { */ - const finalWord = textstream.words.filter(word => word.isFinal === true); + const finalWord = textstream.words.filter((word: any) => word.isFinal === true); // when we only get final word for the previous speaker then don't flip previous speaker as active but update in place. if ( @@ -94,6 +107,12 @@ const useStreamMessageUtils = (): { // we have a speaker change so clear the context for prev speaker if (prevSpeakerRef.current !== '') { finalList[prevSpeakerRef.current] = []; + // Clear translations for previous speaker + if (finalTranslationList[prevSpeakerRef.current]) { + Object.keys(finalTranslationList[prevSpeakerRef.current]).forEach(lang => { + finalTranslationList[prevSpeakerRef.current][lang] = []; + }); + } isInterjecting = true; // console.log( // '%cSTT-callback%c Interjection! ', @@ -114,6 +133,46 @@ const useStreamMessageUtils = (): { finalTranscriptList[textstream.uid] = []; } + // Process translations if available + if (textstream.trans && textstream.trans.length > 0) { + for (const trans of textstream.trans) { + const lang = trans.lang; + const texts = trans.texts || []; + const isFinal = trans.isFinal || false; + + if (!finalTranslationList[textstream.uid]) { + finalTranslationList[textstream.uid] = {}; + } + if (!finalTranslationList[textstream.uid][lang]) { + finalTranslationList[textstream.uid][lang] = []; + } + + const currentTranslationText = texts.join(' '); + if (currentTranslationText) { + if (isFinal) { + finalTranslationList[textstream.uid][lang].push(currentTranslationText); + } + + // Build complete translation text (final + current non-final) + const existingTranslationBuffer = isInterjecting + ? '' + : finalTranslationList[textstream.uid][lang]?.join(' '); + const latestTranslationString = isFinal ? '' : currentTranslationText; + const completeTranslationText = existingTranslationBuffer.length > 0 + ? (latestTranslationString ? existingTranslationBuffer + ' ' + latestTranslationString : existingTranslationBuffer) + : latestTranslationString; + + if (completeTranslationText || isFinal) { + translations.push({ + lang, + text: completeTranslationText, + isFinal, + }); + } + } + } + } + const words = textstream.words; //[Word,Word] /* categorize words into final & nonFinal objects per uid @@ -151,6 +210,21 @@ const useStreamMessageUtils = (): { /* Updating Meeting Transcript */ if (currentFinalText.length) { + // final translations for transcript - include ALL available final translations for this user + const finalTranslationsForTranscript: TranslationData[] = []; + if (finalTranslationList[textstream.uid]) { + Object.keys(finalTranslationList[textstream.uid]).forEach(lang => { + const translationText = finalTranslationList[textstream.uid][lang]?.join(' ') || ''; + if (translationText) { + finalTranslationsForTranscript.push({ + lang: lang, + text: translationText, + isFinal: true, + }); + } + }); + } + setMeetingTranscript(prevTranscript => { const lastTranscriptIndex = prevTranscript.length - 1; const lastTranscript = @@ -160,7 +234,7 @@ const useStreamMessageUtils = (): { /* checking if the last item transcript matches with current uid - If yes then updating the last transcript msg with current text + If yes then updating the last transcript msg with current text and translations If no then adding a new entry in the transcript */ if (lastTranscript && lastTranscript.uid === textstream.uid) { @@ -168,6 +242,9 @@ const useStreamMessageUtils = (): { ...lastTranscript, //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values text: finalTranscriptList[textstream.uid].join(' '), + translations: finalTranslationsForTranscript, + // preserve the original translation language from when this transcript was created + selectedTranslationLanguage: lastTranscript.selectedTranslationLanguage, }; return [ @@ -183,6 +260,10 @@ const useStreamMessageUtils = (): { uid: textstream.uid, time: new Date().getTime(), text: currentFinalText, + translations: finalTranslationsForTranscript, + // Store the current translation language with this transcript item + // This preserves which translation was active when this text was spoken + selectedTranslationLanguage: selectedTranslationLanguageRef.current, }, ]; } @@ -202,17 +283,34 @@ const useStreamMessageUtils = (): { ? existingStringBuffer + ' ' + latestString : latestString; - // updating the captions - captionText && - setCaptionObj(prevState => { - return { - ...prevState, - [textstream.uid]: { - text: captionText, - lastUpdated: new Date().getTime(), - }, - }; - }); + // updating the captions with both transcription and translations + setCaptionObj(prevState => { + const existingTranslations = prevState[textstream.uid]?.translations || []; + + // Update existing translations or add new ones + const updatedTranslations = [...existingTranslations]; + + for (const newTrans of translations) { + const existingIndex = updatedTranslations.findIndex( + t => t.lang === newTrans.lang + ); + + if (existingIndex >= 0) { + updatedTranslations[existingIndex] = newTrans; + } else { + updatedTranslations.push(newTrans); + } + } + + return { + ...prevState, + [textstream.uid]: { + text: captionText || prevState[textstream.uid]?.text || '', + translations: updatedTranslations, + lastUpdated: new Date().getTime(), + }, + }; + }); // console.group('STT-logs'); // console.log('Recived uid =>', textstream.uid); diff --git a/template/src/subComponents/caption/utils.ts b/template/src/subComponents/caption/utils.ts index a9a891321..a71643fe8 100644 --- a/template/src/subComponents/caption/utils.ts +++ b/template/src/subComponents/caption/utils.ts @@ -118,14 +118,35 @@ export const formatTranscriptContent = ( meetingTitle: string, defaultContent: ContentObjects, ) => { + // Helper function to get display text based on stored translation language + // Uses the translation language that was active when this transcript item was created + const getDisplayText = (item: TranscriptItem): string => { + // Use the stored translation language from when this item was created + const storedTranslationLanguage = item.selectedTranslationLanguage; + + if (!storedTranslationLanguage || !item.translations) { + return item.text; // no translation selected or no translations available, show original + } + + // find translation for the stored language + const currentTranslation = item.translations.find(t => t.lang === storedTranslationLanguage); + if (currentTranslation?.text) { + return currentTranslation.text; + } + + // if stored language not available, show original + return item.text; + }; + const formattedContent = meetingTranscript .map(item => { - if (item.uid.toString().indexOf('langUpdate') !== -1) { + if (item.uid.toString().indexOf('langUpdate') !== -1|| item.uid.toString().indexOf('translationUpdate')!== -1) { return `${defaultContent[item?.uid?.split('-')[1]]?.name} ${item.text}`; } - return `${defaultContent[item.uid].name} ${formatTime( + const displayText = getDisplayText(item); + return `${defaultContent[item.uid]?.name} ${formatTime( Number(item?.time), - )}:\n${item.text}`; + )}:\n${displayText}`; }) .join('\n\n'); @@ -160,3 +181,46 @@ export const formatTranscriptContent = ( return [finalContent, fileName]; }; + +export interface TranslateConfig { + source_lang: string; + target_lang: string[]; +} + +export const mergeTranslationConfigs = ( + existingTranslateConfig: TranslateConfig[], + userOwnLanguages: LanguageType[], + selectedTranslationLanguage: string, +): TranslateConfig[] => { + // Create new translate_config for user's own languages + const newTranslateConfigs = userOwnLanguages.map(spokenLang => ({ + source_lang: spokenLang, + target_lang: [selectedTranslationLanguage], + })); + + // Merge with existing configuration + const mergedTranslateConfig = [...existingTranslateConfig]; + + newTranslateConfigs.forEach(newConfig => { + const existingIndex = mergedTranslateConfig.findIndex( + existing => existing.source_lang === newConfig.source_lang, + ); + + if (existingIndex !== -1) { + // Same source language - merge target languages + const existingTargets = mergedTranslateConfig[existingIndex].target_lang; + const mergedTargets = [ + ...new Set([...existingTargets, ...newConfig.target_lang]), + ]; + mergedTranslateConfig[existingIndex] = { + ...mergedTranslateConfig[existingIndex], + target_lang: mergedTargets, + }; + } else { + // Different source language - add new config + mergedTranslateConfig.push(newConfig); + } + }); + + return mergedTranslateConfig; +};