Browse Source

Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

tags/2.0.0-beta.1
jyong 2 months ago
parent
commit
451948d49c

+ 3
- 1
api/core/app/apps/advanced_chat/generate_task_pipeline.py View File

err = self._base_task_pipeline._handle_error(event=event, session=session, message_id=self._message_id) err = self._base_task_pipeline._handle_error(event=event, session=session, message_id=self._message_id)
yield self._base_task_pipeline._error_to_stream_response(err) yield self._base_task_pipeline._error_to_stream_response(err)


def _handle_workflow_started_event(self, **kwargs) -> Generator[StreamResponse, None, None]:
def _handle_workflow_started_event(
self, event: QueueWorkflowStartedEvent, **kwargs
) -> Generator[StreamResponse, None, None]:
"""Handle workflow started events.""" """Handle workflow started events."""
with self._database_session() as session: with self._database_session() as session:
workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start() workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start()

+ 3
- 3
web/app/components/datasets/create-from-pipeline/create-options/create-from-dsl-modal/uploader.tsx View File

import React, { useEffect, useRef, useState } from 'react' import React, { useEffect, useRef, useState } from 'react'
import { import {
RiDeleteBinLine, RiDeleteBinLine,
RiNodeTree,
} from '@remixicon/react' } from '@remixicon/react'
import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector' import { useContext } from 'use-context-selector'
import { formatFileSize } from '@/utils/format' import { formatFileSize } from '@/utils/format'
import cn from '@/utils/classnames' import cn from '@/utils/classnames'
import { Yaml as YamlIcon } from '@/app/components/base/icons/src/public/files'
import { ToastContext } from '@/app/components/base/toast' import { ToastContext } from '@/app/components/base/toast'
import { UploadCloud01 } from '@/app/components/base/icons/src/vender/line/general' import { UploadCloud01 } from '@/app/components/base/icons/src/vender/line/general'
import Button from '@/app/components/base/button' import Button from '@/app/components/base/button'
{file && ( {file && (
<div className={cn('group flex items-center rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs', 'hover:border-[#B2CCFF] hover:bg-[#F5F8FF]')}> <div className={cn('group flex items-center rounded-lg border-[0.5px] border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs', 'hover:border-[#B2CCFF] hover:bg-[#F5F8FF]')}>
<div className='flex items-center justify-center p-3'> <div className='flex items-center justify-center p-3'>
<YamlIcon className='h-6 w-6 shrink-0' />
<RiNodeTree className='h-6 w-6 shrink-0' />
</div> </div>
<div className='flex grow flex-col items-start gap-0.5 py-1 pr-2'> <div className='flex grow flex-col items-start gap-0.5 py-1 pr-2'>
<span className='font-inter max-w-[calc(100%_-_30px)] overflow-hidden text-ellipsis whitespace-nowrap text-[12px] font-medium leading-4 text-text-secondary'>{file.name}</span> <span className='font-inter max-w-[calc(100%_-_30px)] overflow-hidden text-ellipsis whitespace-nowrap text-[12px] font-medium leading-4 text-text-secondary'>{file.name}</span>
<div className='font-inter flex h-3 items-center gap-1 self-stretch text-[10px] font-medium uppercase leading-3 text-text-tertiary'> <div className='font-inter flex h-3 items-center gap-1 self-stretch text-[10px] font-medium uppercase leading-3 text-text-tertiary'>
<span>YAML</span>
<span>PIPELINE</span>
<span className='text-text-quaternary'>·</span> <span className='text-text-quaternary'>·</span>
<span>{formatFileSize(file.size)}</span> <span>{formatFileSize(file.size)}</span>
</div> </div>

+ 4
- 1
web/app/components/plugins/marketplace/search-box/trigger/tool-selector.tsx View File

!!selectedTagsLength && ( !!selectedTagsLength && (
<RiCloseCircleFill <RiCloseCircleFill
className='size-4 text-text-quaternary' className='size-4 text-text-quaternary'
onClick={() => onTagsChange([])}
onClick={(e) => {
e.stopPropagation()
onTagsChange([])
}}
/> />
) )
} }

+ 1
- 1
web/app/components/rag-pipeline/components/panel/input-field/editor/form/hooks.ts View File

break break
case PipelineInputVarType.checkbox: case PipelineInputVarType.checkbox:
fieldNames = [ fieldNames = [
t('appDebug.variableConfig.startedChecked'),
t('appDebug.variableConfig.startChecked'),
t('appDebug.variableConfig.tooltips'), t('appDebug.variableConfig.tooltips'),
] ]
break break

+ 15
- 15
web/app/components/rag-pipeline/components/rag-pipeline-main.tsx View File

...configsMap, ...configsMap,
}) })
const { const {
hasNodeInspectVars,
hasSetInspectVar,
fetchInspectVarValue,
editInspectVarValue,
renameInspectVarName,
appendNodeInspectVars,
deleteInspectVar,
deleteNodeInspectorVars,
deleteAllInspectorVars,
isInspectVarEdited,
resetToLastRunVar,
invalidateSysVarValues,
resetConversationVar,
invalidateConversationVarValues,
} = useInspectVarsCrud()
hasNodeInspectVars,
hasSetInspectVar,
fetchInspectVarValue,
editInspectVarValue,
renameInspectVarName,
appendNodeInspectVars,
deleteInspectVar,
deleteNodeInspectorVars,
deleteAllInspectorVars,
isInspectVarEdited,
resetToLastRunVar,
invalidateSysVarValues,
resetConversationVar,
invalidateConversationVarValues,
} = useInspectVarsCrud()


const hooksStore = useMemo(() => { const hooksStore = useMemo(() => {
return { return {

+ 2
- 1
web/app/components/workflow-app/hooks/use-DSL.ts View File



const appDetail = useAppStore(s => s.appDetail) const appDetail = useAppStore(s => s.appDetail)


const handleExportDSL = useCallback(async (include = false) => {
const handleExportDSL = useCallback(async (include = false, workflowId?: string) => {
if (!appDetail) if (!appDetail)
return return


const { data } = await exportAppConfig({ const { data } = await exportAppConfig({
appID: appDetail.id, appID: appDetail.id,
include, include,
workflowID: workflowId,
}) })
const a = document.createElement('a') const a = document.createElement('a')
const file = new Blob([data], { type: 'application/yaml' }) const file = new Blob([data], { type: 'application/yaml' })

+ 2
- 10
web/app/components/workflow/block-selector/all-tools.tsx View File

import { useMarketplacePlugins } from '../../plugins/marketplace/hooks' import { useMarketplacePlugins } from '../../plugins/marketplace/hooks'
import { useGlobalPublicStore } from '@/context/global-public-context' import { useGlobalPublicStore } from '@/context/global-public-context'
import RAGToolSuggestions from './rag-tool-suggestions' import RAGToolSuggestions from './rag-tool-suggestions'
import { useRAGRecommendedPlugins } from '@/service/use-tools'


type AllToolsProps = { type AllToolsProps = {
className?: string className?: string
const wrapElemRef = useRef<HTMLDivElement>(null) const wrapElemRef = useRef<HTMLDivElement>(null)
const isSupportGroupView = [ToolTypeEnum.All, ToolTypeEnum.BuiltIn].includes(activeTab) const isSupportGroupView = [ToolTypeEnum.All, ToolTypeEnum.BuiltIn].includes(activeTab)


const isShowRAGRecommendations = isInRAGPipeline && activeTab === ToolTypeEnum.All && !searchText && tags.length === 0
const { data: ragRecommendedPlugins } = useRAGRecommendedPlugins(isShowRAGRecommendations)
const recommendedPlugins = useMemo(() => {
if (ragRecommendedPlugins)
return [...ragRecommendedPlugins.installed_recommended_plugins]
return []
}, [ragRecommendedPlugins])
const isShowRAGRecommendations = isInRAGPipeline && activeTab === ToolTypeEnum.All && !hasFilter


return ( return (
<div className={cn('min-w-[400px] max-w-[500px]', className)}> <div className={cn('min-w-[400px] max-w-[500px]', className)}>
className='max-h-[464px] overflow-y-auto' className='max-h-[464px] overflow-y-auto'
onScroll={pluginRef.current?.handleScroll} onScroll={pluginRef.current?.handleScroll}
> >
{recommendedPlugins.length > 0 && (
{isShowRAGRecommendations && (
<RAGToolSuggestions <RAGToolSuggestions
tools={recommendedPlugins}
viewType={isSupportGroupView ? activeView : ViewType.flat} viewType={isSupportGroupView ? activeView : ViewType.flat}
onSelect={onSelect} onSelect={onSelect}
onTagsChange={onTagsChange} onTagsChange={onTagsChange}

+ 3
- 3
web/app/components/workflow/block-selector/market-place-plugin/list.tsx View File

disableMaxWidth = false, disableMaxWidth = false,
}, ref) => { }, ref) => {
const { t } = useTranslation() const { t } = useTranslation()
const hasFilter = !searchText
const noFilter = !searchText && tags.length === 0
const hasRes = list.length > 0 const hasRes = list.length > 0
const urlWithSearchText = getMarketplaceUrl('', { q: searchText, tags: tags.join(',') }) const urlWithSearchText = getMarketplaceUrl('', { q: searchText, tags: tags.join(',') })
const nextToStickyELemRef = useRef<HTMLDivElement>(null) const nextToStickyELemRef = useRef<HTMLDivElement>(null)
window.open(urlWithSearchText, '_blank') window.open(urlWithSearchText, '_blank')
} }


if (hasFilter) {
if (noFilter) {
return ( return (
<Link <Link
className='system-sm-medium sticky bottom-0 z-10 flex h-8 cursor-pointer items-center rounded-b-lg border-[0.5px] border-t border-components-panel-border bg-components-panel-bg-blur px-4 py-1 text-text-accent-light-mode-only shadow-lg' className='system-sm-medium sticky bottom-0 z-10 flex h-8 cursor-pointer items-center rounded-b-lg border-[0.5px] border-t border-components-panel-border bg-components-panel-bg-blur px-4 py-1 text-text-accent-light-mode-only shadow-lg'
onAction={noop} onAction={noop}
/> />
))} ))}
{list.length > 0 && (
{hasRes && (
<div className='mb-3 mt-2 flex items-center justify-center space-x-2'> <div className='mb-3 mt-2 flex items-center justify-center space-x-2'>
<div className="h-[2px] w-[90px] bg-gradient-to-l from-[rgba(16,24,40,0.08)] to-[rgba(255,255,255,0.01)]"></div> <div className="h-[2px] w-[90px] bg-gradient-to-l from-[rgba(16,24,40,0.08)] to-[rgba(255,255,255,0.01)]"></div>
<Link <Link

+ 63
- 24
web/app/components/workflow/block-selector/rag-tool-suggestions.tsx View File

import type { Dispatch, SetStateAction } from 'react' import type { Dispatch, SetStateAction } from 'react'
import React, { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import type { OnSelectBlock, ToolWithProvider } from '../types'
import React, { useCallback, useMemo } from 'react'
import { Trans, useTranslation } from 'react-i18next'
import type { OnSelectBlock } from '../types'
import Tools from './tools' import Tools from './tools'
import { ToolTypeEnum } from './types' import { ToolTypeEnum } from './types'
import type { ViewType } from './view-type-select' import type { ViewType } from './view-type-select'
import { RiMoreLine } from '@remixicon/react' import { RiMoreLine } from '@remixicon/react'
import Loading from '@/app/components/base/loading'
import Link from 'next/link'
import { getMarketplaceUrl } from '@/utils/var'
import { useRAGRecommendedPlugins } from '@/service/use-tools'


type RAGToolSuggestionsProps = { type RAGToolSuggestionsProps = {
tools: ToolWithProvider[]
viewType: ViewType viewType: ViewType
onSelect: OnSelectBlock onSelect: OnSelectBlock
onTagsChange: Dispatch<SetStateAction<string[]>> onTagsChange: Dispatch<SetStateAction<string[]>>
} }


const RAGToolSuggestions: React.FC<RAGToolSuggestionsProps> = ({ const RAGToolSuggestions: React.FC<RAGToolSuggestionsProps> = ({
tools,
viewType, viewType,
onSelect, onSelect,
onTagsChange, onTagsChange,
}) => { }) => {
const { t } = useTranslation() const { t } = useTranslation()


const {
data: ragRecommendedPlugins,
isFetching: isFetchingRAGRecommendedPlugins,
} = useRAGRecommendedPlugins()

const recommendedPlugins = useMemo(() => {
if (ragRecommendedPlugins)
return [...ragRecommendedPlugins.installed_recommended_plugins]
return []
}, [ragRecommendedPlugins])

const loadMore = useCallback(() => { const loadMore = useCallback(() => {
onTagsChange((prev) => { onTagsChange((prev) => {
if (prev.includes('rag')) if (prev.includes('rag'))
<div className='system-xs-medium px-3 pb-0.5 pt-1 text-text-tertiary'> <div className='system-xs-medium px-3 pb-0.5 pt-1 text-text-tertiary'>
{t('pipeline.ragToolSuggestions.title')} {t('pipeline.ragToolSuggestions.title')}
</div> </div>
<Tools
className='p-0'
tools={tools}
onSelect={onSelect}
canNotSelectMultiple
toolType={ToolTypeEnum.All}
viewType={viewType}
hasSearchText={false}
/>
<div
className='flex cursor-pointer items-center gap-x-2 py-1 pl-3 pr-2'
onClick={loadMore}
>
<div className='px-1'>
<RiMoreLine className='size-4 text-text-tertiary' />
{isFetchingRAGRecommendedPlugins && (
<div className='py-2'>
<Loading type='app' />
</div> </div>
<div className='system-xs-regular text-text-tertiary'>
{t('common.operation.more')}
</div>
</div>
)}
{!isFetchingRAGRecommendedPlugins && recommendedPlugins.length === 0 && (
<p className='system-xs-regular px-3 py-1 text-text-tertiary'>
<Trans
i18nKey='pipeline.ragToolSuggestions.noRecommendationPluginsInstalled'
components={{
CustomLink: (
<Link
className='text-text-accent'
target='_blank'
rel='noopener noreferrer'
href={getMarketplaceUrl('', { tags: 'rag' })}
/>
),
}}
/>
</p>
)}
{!isFetchingRAGRecommendedPlugins && recommendedPlugins.length > 0 && (
<>
<Tools
className='p-0'
tools={recommendedPlugins}
onSelect={onSelect}
canNotSelectMultiple
toolType={ToolTypeEnum.All}
viewType={viewType}
hasSearchText={false}
/>
<div
className='flex cursor-pointer items-center gap-x-2 py-1 pl-3 pr-2'
onClick={loadMore}
>
<div className='px-1'>
<RiMoreLine className='size-4 text-text-tertiary' />
</div>
<div className='system-xs-regular text-text-tertiary'>
{t('common.operation.more')}
</div>
</div>
</>
)}
</div> </div>
) )
} }

+ 1
- 1
web/app/components/workflow/hooks-store/store.ts View File

availableNodesMetaData?: AvailableNodesMetaData availableNodesMetaData?: AvailableNodesMetaData
getWorkflowRunAndTraceUrl: (runId?: string) => { runUrl: string; traceUrl: string } getWorkflowRunAndTraceUrl: (runId?: string) => { runUrl: string; traceUrl: string }
exportCheck?: () => Promise<void> exportCheck?: () => Promise<void>
handleExportDSL?: (include?: boolean) => Promise<void>
handleExportDSL?: (include?: boolean, flowId?: string) => Promise<void>
fetchInspectVars: (params: { passInVars?: boolean, vars?: VarInInspect[], passedInAllPluginInfoList?: Record<string, ToolWithProvider[]>, passedInSchemaTypeDefinitions?: SchemaTypeDefinition[] }) => Promise<void> fetchInspectVars: (params: { passInVars?: boolean, vars?: VarInInspect[], passedInAllPluginInfoList?: Record<string, ToolWithProvider[]>, passedInSchemaTypeDefinitions?: SchemaTypeDefinition[] }) => Promise<void>
hasNodeInspectVars: (nodeId: string) => boolean hasNodeInspectVars: (nodeId: string) => boolean
hasSetInspectVar: (nodeId: string, name: string, sysVars: VarInInspect[], conversationVars: VarInInspect[]) => boolean hasSetInspectVar: (nodeId: string, name: string, sysVars: VarInInspect[], conversationVars: VarInInspect[]) => boolean

+ 1
- 1
web/app/components/workflow/panel/version-history-panel/index.tsx View File

}) })
break break
case VersionHistoryContextMenuOptions.exportDSL: case VersionHistoryContextMenuOptions.exportDSL:
handleExportDSL(false, item.id)
handleExportDSL?.(false, item.id)
break break
} }
}, [t, handleExportDSL]) }, [t, handleExportDSL])

+ 3
- 2
web/app/components/workflow/variable-inspect/display-content.tsx View File

readonly: boolean readonly: boolean
handleTextChange?: (value: string) => void handleTextChange?: (value: string) => void
handleEditorChange?: (value: string) => void handleEditorChange?: (value: string) => void
className?: string
} }


const DisplayContent = (props: DisplayContentProps) => { const DisplayContent = (props: DisplayContentProps) => {
const { previewType, varType, schemaType, mdString, jsonString, readonly, handleTextChange, handleEditorChange } = props
const { previewType, varType, schemaType, mdString, jsonString, readonly, handleTextChange, handleEditorChange, className } = props
const [viewMode, setViewMode] = useState<ViewMode>(ViewMode.Code) const [viewMode, setViewMode] = useState<ViewMode>(ViewMode.Code)
const [isFocused, setIsFocused] = useState(false) const [isFocused, setIsFocused] = useState(false)
const { t } = useTranslation() const { t } = useTranslation()
}, [previewType, schemaType, jsonString]) }, [previewType, schemaType, jsonString])


return ( return (
<div className={cn('flex h-full flex-col rounded-[10px] bg-components-input-bg-normal', isFocused && 'bg-components-input-bg-active outline outline-1 outline-components-input-border-active')}>
<div className={cn('flex h-full flex-col rounded-[10px] bg-components-input-bg-normal', isFocused && 'bg-components-input-bg-active outline outline-1 outline-components-input-border-active', className)}>
<div className='flex shrink-0 items-center justify-end p-1'> <div className='flex shrink-0 items-center justify-end p-1'>
{previewType === PreviewType.Markdown && ( {previewType === PreviewType.Markdown && (
<div className='system-xs-semibold-uppercase flex grow items-center px-2 py-0.5 text-text-secondary'> <div className='system-xs-semibold-uppercase flex grow items-center px-2 py-0.5 text-text-secondary'>

+ 1
- 0
web/app/components/workflow/variable-inspect/value-content.tsx View File

mdString={value as any} mdString={value as any}
readonly={textEditorDisabled} readonly={textEditorDisabled}
handleTextChange={handleTextChange} handleTextChange={handleTextChange}
className={cn(isTruncated && 'pt-[36px]')}
/> />
) : ( ) : (
<Textarea <Textarea

+ 0
- 1
web/i18n/en-US/app-debug.ts View File

'tooltips': 'Tooltips', 'tooltips': 'Tooltips',
'tooltipsPlaceholder': 'Enter helpful text shown when hovering over the label', 'tooltipsPlaceholder': 'Enter helpful text shown when hovering over the label',
'showAllSettings': 'Show All Settings', 'showAllSettings': 'Show All Settings',
'checkbox': 'Checkbox',
'startSelectedOption': 'Start selected option', 'startSelectedOption': 'Start selected option',
'noDefaultSelected': 'Don\'t select', 'noDefaultSelected': 'Don\'t select',
'hide': 'Hide', 'hide': 'Hide',

+ 1
- 0
web/i18n/en-US/pipeline.ts View File

}, },
ragToolSuggestions: { ragToolSuggestions: {
title: 'Suggestions for RAG', title: 'Suggestions for RAG',
noRecommendationPluginsInstalled: 'No recommended plugins installed, find more in <CustomLink>Marketplace</CustomLink>',
}, },
} }



+ 0
- 1
web/i18n/zh-Hans/app-debug.ts View File

'tooltips': '提示', 'tooltips': '提示',
'tooltipsPlaceholder': '输入悬停在标签上时显示的提示文本', 'tooltipsPlaceholder': '输入悬停在标签上时显示的提示文本',
'showAllSettings': '显示所有设置', 'showAllSettings': '显示所有设置',
'checkbox': '复选框',
'startSelectedOption': '默认选中项', 'startSelectedOption': '默认选中项',
'noDefaultSelected': '不默认选中', 'noDefaultSelected': '不默认选中',
'file': { 'file': {

+ 1
- 0
web/i18n/zh-Hans/pipeline.ts View File

}, },
ragToolSuggestions: { ragToolSuggestions: {
title: 'RAG 工具推荐', title: 'RAG 工具推荐',
noRecommendationPluginsInstalled: '暂无已安装的推荐插件,更多插件请在 <CustomLink>Marketplace</CustomLink> 中查找',
}, },
} }



+ 3
- 1
web/service/use-plugins.ts View File

useQuery, useQuery,
useQueryClient, useQueryClient,
} from '@tanstack/react-query' } from '@tanstack/react-query'
import { useInvalidateAllBuiltInTools } from './use-tools'
import { useInvalidateAllBuiltInTools, useInvalidateRAGRecommendedPlugins } from './use-tools'
import useReferenceSetting from '@/app/components/plugins/plugin-page/use-reference-setting' import useReferenceSetting from '@/app/components/plugins/plugin-page/use-reference-setting'
import { uninstallPlugin } from '@/service/plugins' import { uninstallPlugin } from '@/service/plugins'
import useRefreshPluginList from '@/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list' import useRefreshPluginList from '@/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list'
export const useInvalidateInstalledPluginList = () => { export const useInvalidateInstalledPluginList = () => {
const queryClient = useQueryClient() const queryClient = useQueryClient()
const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools() const invalidateAllBuiltInTools = useInvalidateAllBuiltInTools()
const invalidateRAGRecommendedPlugins = useInvalidateRAGRecommendedPlugins()
return () => { return () => {
queryClient.invalidateQueries( queryClient.invalidateQueries(
{ {
queryKey: useInstalledPluginListKey, queryKey: useInstalledPluginListKey,
}) })
invalidateAllBuiltInTools() invalidateAllBuiltInTools()
invalidateRAGRecommendedPlugins()
} }
} }



+ 8
- 3
web/service/use-tools.ts View File

}) })
} }


export const useRAGRecommendedPlugins = (enabled: boolean) => {
const useRAGRecommendedPluginListKey = [NAME_SPACE, 'rag-recommended-plugins']

export const useRAGRecommendedPlugins = () => {
return useQuery<RAGRecommendedPlugins>({ return useQuery<RAGRecommendedPlugins>({
queryKey: [NAME_SPACE, 'rag-recommended-plugins'],
queryKey: useRAGRecommendedPluginListKey,
queryFn: () => get<RAGRecommendedPlugins>('/rag/pipelines/recommended-plugins'), queryFn: () => get<RAGRecommendedPlugins>('/rag/pipelines/recommended-plugins'),
enabled,
}) })
} }

export const useInvalidateRAGRecommendedPlugins = () => {
return useInvalid(useRAGRecommendedPluginListKey)
}

Loading…
Cancel
Save