Sfoglia il codice sorgente

feat: Supports pronunciation while outputting text #2088 (#2227)

### What problem does this PR solve?

feat: Supports pronunciation while outputting text #2088

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
tags/v0.11.0
balibabu 1 anno fa
parent
commit
07de36ec86
Nessun account collegato all'indirizzo email del committer

+ 3
- 1
web/src/components/message-item/group-button.tsx Vedi File

content: string; content: string;
prompt?: string; prompt?: string;
showLikeButton: boolean; showLikeButton: boolean;
audioBinary?: string;
} }


export const AssistantGroupButton = ({ export const AssistantGroupButton = ({
messageId, messageId,
content, content,
prompt, prompt,
audioBinary,
showLikeButton, showLikeButton,
}: IProps) => { }: IProps) => {
const { visible, hideModal, showModal, onFeedbackOk, loading } = const { visible, hideModal, showModal, onFeedbackOk, loading } =
showModal: showPromptModal, showModal: showPromptModal,
} = useSetModalState(); } = useSetModalState();
const { t } = useTranslation(); const { t } = useTranslation();
const { handleRead, ref, isPlaying } = useSpeech(content);
const { handleRead, ref, isPlaying } = useSpeech(content, audioBinary);


const handleLike = useCallback(() => { const handleLike = useCallback(() => {
onFeedbackOk({ thumbup: true }); onFeedbackOk({ thumbup: true });

+ 10
- 1
web/src/components/message-item/hooks.ts Vedi File

return { onRemoveMessage, loading }; return { onRemoveMessage, loading };
}; };


export const useSpeech = (content: string) => {
export const useSpeech = (content: string, audioBinary?: string) => {
const ref = useRef<HTMLAudioElement>(null); const ref = useRef<HTMLAudioElement>(null);
const { read } = useSpeechWithSse(); const { read } = useSpeechWithSse();
const player = useRef<SpeechPlayer>(); const player = useRef<SpeechPlayer>();
} }
}, [setIsPlaying, speech, isPlaying, pause]); }, [setIsPlaying, speech, isPlaying, pause]);


// useEffect(() => {
// if (audioBinary) {
// const units = hexStringToUint8Array(audioBinary);
// if (units) {
// player.current?.feed(units);
// }
// }
// }, [audioBinary]);

useEffect(() => { useEffect(() => {
initialize(); initialize();
}, [initialize]); }, [initialize]);

+ 1
- 0
web/src/components/message-item/index.tsx Vedi File

content={item.content} content={item.content}
prompt={item.prompt} prompt={item.prompt}
showLikeButton={showLikeButton} showLikeButton={showLikeButton}
audioBinary={item.audio_binary}
></AssistantGroupButton> ></AssistantGroupButton>
) )
) : ( ) : (

+ 10
- 6
web/src/hooks/file-manager-hooks.ts Vedi File

formData.append('file', file); formData.append('file', file);
formData.append('path', pathList[index]); formData.append('path', pathList[index]);
}); });
const { data } = await fileManagerService.uploadFile(formData);
if (data.retcode === 0) {
message.success(t('message.uploaded'));
setPaginationParams(1);
queryClient.invalidateQueries({ queryKey: ['fetchFileList'] });
try {
const { data } = await fileManagerService.uploadFile(formData);
if (data.retcode === 0) {
message.success(t('message.uploaded'));
setPaginationParams(1);
queryClient.invalidateQueries({ queryKey: ['fetchFileList'] });
}
return data.retcode;
} catch (error) {
console.log('🚀 ~ useUploadFile ~ error:', error);
} }
return data.retcode;
}, },
}); });



+ 1
- 0
web/src/hooks/logic-hooks.ts Vedi File

role: MessageType.Assistant, role: MessageType.Assistant,
}), }),
prompt: answer.prompt, prompt: answer.prompt,
audio_binary: answer.audio_binary,
}, },
]; ];
}); });

+ 2
- 0
web/src/interfaces/database/chat.ts Vedi File

doc_ids?: string[]; doc_ids?: string[];
prompt?: string; prompt?: string;
id?: string; id?: string;
audio_binary?: string;
} }


export interface IReference { export interface IReference {
conversationId?: string; conversationId?: string;
prompt?: string; prompt?: string;
id?: string; id?: string;
audio_binary?: string;
} }


export interface Docagg { export interface Docagg {

+ 41
- 0
web/src/utils/common-util.ts Vedi File

} }
return value; return value;
}; };

export const stringToUint8Array = (str: string) => {
// const byteString = str.replace(/b'|'/g, '');
const byteString = str.slice(2, -1);

const uint8Array = new Uint8Array(byteString.length);
for (let i = 0; i < byteString.length; i++) {
uint8Array[i] = byteString.charCodeAt(i);
}

return uint8Array;
};

export const hexStringToUint8Array = (hex: string) => {
const arr = hex.match(/[\da-f]{2}/gi);
if (Array.isArray(arr)) {
return new Uint8Array(
arr.map(function (h) {
return parseInt(h, 16);
}),
);
}
};

export function hexToArrayBuffer(input: string) {
if (typeof input !== 'string') {
throw new TypeError('Expected input to be a string');
}

if (input.length % 2 !== 0) {
throw new RangeError('Expected string to be an even number of characters');
}

const view = new Uint8Array(input.length / 2);

for (let i = 0; i < input.length; i += 2) {
view[i / 2] = parseInt(input.substring(i, i + 2), 16);
}

return view.buffer;
}

Loading…
Annulla
Salva