### What problem does this PR solve? feat: Supports pronunciation while outputting text #2088 ### Type of change - [x] New Feature (non-breaking change which adds functionality)tags/v0.11.0
| content: string; | content: string; | ||||
| prompt?: string; | prompt?: string; | ||||
| showLikeButton: boolean; | showLikeButton: boolean; | ||||
| audioBinary?: string; | |||||
| } | } | ||||
| export const AssistantGroupButton = ({ | export const AssistantGroupButton = ({ | ||||
| messageId, | messageId, | ||||
| content, | content, | ||||
| prompt, | prompt, | ||||
| audioBinary, | |||||
| showLikeButton, | showLikeButton, | ||||
| }: IProps) => { | }: IProps) => { | ||||
| const { visible, hideModal, showModal, onFeedbackOk, loading } = | const { visible, hideModal, showModal, onFeedbackOk, loading } = | ||||
| showModal: showPromptModal, | showModal: showPromptModal, | ||||
| } = useSetModalState(); | } = useSetModalState(); | ||||
| const { t } = useTranslation(); | const { t } = useTranslation(); | ||||
| const { handleRead, ref, isPlaying } = useSpeech(content); | |||||
| const { handleRead, ref, isPlaying } = useSpeech(content, audioBinary); | |||||
| const handleLike = useCallback(() => { | const handleLike = useCallback(() => { | ||||
| onFeedbackOk({ thumbup: true }); | onFeedbackOk({ thumbup: true }); |
| return { onRemoveMessage, loading }; | return { onRemoveMessage, loading }; | ||||
| }; | }; | ||||
| export const useSpeech = (content: string) => { | |||||
| export const useSpeech = (content: string, audioBinary?: string) => { | |||||
| const ref = useRef<HTMLAudioElement>(null); | const ref = useRef<HTMLAudioElement>(null); | ||||
| const { read } = useSpeechWithSse(); | const { read } = useSpeechWithSse(); | ||||
| const player = useRef<SpeechPlayer>(); | const player = useRef<SpeechPlayer>(); | ||||
| } | } | ||||
| }, [setIsPlaying, speech, isPlaying, pause]); | }, [setIsPlaying, speech, isPlaying, pause]); | ||||
| // useEffect(() => { | |||||
| // if (audioBinary) { | |||||
| // const units = hexStringToUint8Array(audioBinary); | |||||
| // if (units) { | |||||
| // player.current?.feed(units); | |||||
| // } | |||||
| // } | |||||
| // }, [audioBinary]); | |||||
| useEffect(() => { | useEffect(() => { | ||||
| initialize(); | initialize(); | ||||
| }, [initialize]); | }, [initialize]); |
| content={item.content} | content={item.content} | ||||
| prompt={item.prompt} | prompt={item.prompt} | ||||
| showLikeButton={showLikeButton} | showLikeButton={showLikeButton} | ||||
| audioBinary={item.audio_binary} | |||||
| ></AssistantGroupButton> | ></AssistantGroupButton> | ||||
| ) | ) | ||||
| ) : ( | ) : ( |
| formData.append('file', file); | formData.append('file', file); | ||||
| formData.append('path', pathList[index]); | formData.append('path', pathList[index]); | ||||
| }); | }); | ||||
| const { data } = await fileManagerService.uploadFile(formData); | |||||
| if (data.retcode === 0) { | |||||
| message.success(t('message.uploaded')); | |||||
| setPaginationParams(1); | |||||
| queryClient.invalidateQueries({ queryKey: ['fetchFileList'] }); | |||||
| try { | |||||
| const { data } = await fileManagerService.uploadFile(formData); | |||||
| if (data.retcode === 0) { | |||||
| message.success(t('message.uploaded')); | |||||
| setPaginationParams(1); | |||||
| queryClient.invalidateQueries({ queryKey: ['fetchFileList'] }); | |||||
| } | |||||
| return data.retcode; | |||||
| } catch (error) { | |||||
| console.log('🚀 ~ useUploadFile ~ error:', error); | |||||
| } | } | ||||
| return data.retcode; | |||||
| }, | }, | ||||
| }); | }); | ||||
| role: MessageType.Assistant, | role: MessageType.Assistant, | ||||
| }), | }), | ||||
| prompt: answer.prompt, | prompt: answer.prompt, | ||||
| audio_binary: answer.audio_binary, | |||||
| }, | }, | ||||
| ]; | ]; | ||||
| }); | }); |
| doc_ids?: string[]; | doc_ids?: string[]; | ||||
| prompt?: string; | prompt?: string; | ||||
| id?: string; | id?: string; | ||||
| audio_binary?: string; | |||||
| } | } | ||||
| export interface IReference { | export interface IReference { | ||||
| conversationId?: string; | conversationId?: string; | ||||
| prompt?: string; | prompt?: string; | ||||
| id?: string; | id?: string; | ||||
| audio_binary?: string; | |||||
| } | } | ||||
| export interface Docagg { | export interface Docagg { |
| } | } | ||||
| return value; | return value; | ||||
| }; | }; | ||||
| export const stringToUint8Array = (str: string) => { | |||||
| // const byteString = str.replace(/b'|'/g, ''); | |||||
| const byteString = str.slice(2, -1); | |||||
| const uint8Array = new Uint8Array(byteString.length); | |||||
| for (let i = 0; i < byteString.length; i++) { | |||||
| uint8Array[i] = byteString.charCodeAt(i); | |||||
| } | |||||
| return uint8Array; | |||||
| }; | |||||
| export const hexStringToUint8Array = (hex: string) => { | |||||
| const arr = hex.match(/[\da-f]{2}/gi); | |||||
| if (Array.isArray(arr)) { | |||||
| return new Uint8Array( | |||||
| arr.map(function (h) { | |||||
| return parseInt(h, 16); | |||||
| }), | |||||
| ); | |||||
| } | |||||
| }; | |||||
| export function hexToArrayBuffer(input: string) { | |||||
| if (typeof input !== 'string') { | |||||
| throw new TypeError('Expected input to be a string'); | |||||
| } | |||||
| if (input.length % 2 !== 0) { | |||||
| throw new RangeError('Expected string to be an even number of characters'); | |||||
| } | |||||
| const view = new Uint8Array(input.length / 2); | |||||
| for (let i = 0; i < input.length; i += 2) { | |||||
| view[i / 2] = parseInt(input.substring(i, i + 2), 16); | |||||
| } | |||||
| return view.buffer; | |||||
| } |