Merge branch 'feat/rag-pipeline' into deploy/rag-dev

web/app/components/datasets/create/website/base/header.tsx

@@ -30,21 +30,23 @@ const Header = ({
         )}>
           {title}
         </div>
-        <Divider type='vertical' className='mx-1 h-3.5' />
         {!isInPipeline && (
-          <Button
-            variant='secondary'
-            size='small'
-            className='px-1.5'
-          >
-            <RiEqualizer2Line
-              className='h-4 w-4'
-              onClick={onClickConfiguration}
-            />
-            <span className='system-xs-medium'>
-              {buttonText}
-            </span>
-          </Button>
+          <>
+            <Divider type='vertical' className='mx-1 h-3.5' />
+            <Button
+              variant='secondary'
+              size='small'
+              className='px-1.5'
+            >
+              <RiEqualizer2Line
+                className='h-4 w-4'
+                onClick={onClickConfiguration}
+              />
+              <span className='system-xs-medium'>
+                {buttonText}
+              </span>
+            </Button>
+          </>
         )}
       </div>
       <a

web/app/components/rag-pipeline/components/input-field/field-list/field-item.tsx

@@ -35,12 +35,14 @@ const FieldItem = ({
   const ref = useRef(null)
   const isHovering = useHover(ref)
 
-  const handleOnClickEdit = useCallback(() => {
+  const handleOnClickEdit = useCallback((e: React.MouseEvent) => {
+    e.stopPropagation()
     if (readonly) return
     onClickEdit(payload.variable)
   }, [onClickEdit, payload.variable, readonly])
 
-  const handleRemove = useCallback(() => {
+  const handleRemove = useCallback((e: React.MouseEvent) => {
+    e.stopPropagation()
     if (readonly) return
     onRemove(index)
   }, [index, onRemove, readonly])

web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useEffect, useMemo, useState } from 'react'
 import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
-import SearchInput from '@/app/education-apply/search-input'
+import SearchInput from '@/app/components/base/notion-page-selector/search-input'
 import PageSelector from '@/app/components/base/notion-page-selector/page-selector'
 import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
 import Header from '@/app/components/datasets/create/website/base/header'
@@ -36,12 +36,15 @@ const NotionPageSelector = ({
 
   const getNotionData = useCallback(async () => {
     if (pipeline_id) {
-      const notionData = await getNotionPages({
+      await getNotionPages({
         pipeline_id,
         node_id: nodeId,
         inputs: {},
-      }) as DataSourceNotionWorkspace[]
-      setNotionData(notionData)
+      }, {
+        onSuccess(notionData) {
+          setNotionData(notionData as DataSourceNotionWorkspace[])
+        },
+      })
     }
   }, [getNotionPages, nodeId, pipeline_id])
 

web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts

@@ -23,6 +23,7 @@ export const useNodesSyncDraft = () => {
       edges,
       transform,
     } = store.getState()
+    const nodes = getNodes()
     const [x, y, zoom] = transform
     const {
       pipelineId,
@@ -32,8 +33,6 @@ export const useNodesSyncDraft = () => {
     } = workflowStore.getState()
 
     if (pipelineId) {
-      const nodes = getNodes()
-
       const producedNodes = produce(nodes, (draft) => {
         draft.forEach((node) => {
           Object.keys(node.data).forEach((key) => {

web/app/components/workflow/block-selector/data-sources.tsx

@@ -32,7 +32,7 @@ const DataSources = ({
   const wrapElemRef = useRef<HTMLDivElement>(null)
   const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => {
     onSelect(BlockEnum.DataSource, toolDefaultValue && {
-      provider_id: toolDefaultValue?.provider_id,
+      plugin_id: toolDefaultValue?.provider_id,
       provider_type: toolDefaultValue?.provider_type,
       provider_name: toolDefaultValue?.provider_name,
       datasource_name: toolDefaultValue?.tool_name,

web/app/components/workflow/block-selector/types.ts

@@ -36,7 +36,7 @@ export type ToolDefaultValue = {
 }
 
 export type DataSourceDefaultValue = {
-  provider_id: string
+  plugin_id: string
   provider_type: string
   provider_name: string
   datasource_name: string

web/app/components/workflow/block-selector/utils.ts

@@ -5,7 +5,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
   return {
     id: dataSourceItem.plugin_id,
     provider: dataSourceItem.provider,
-    name: dataSourceItem.declaration.identity.name,
+    name: dataSourceItem.provider,
     author: dataSourceItem.declaration.identity.author,
     description: dataSourceItem.declaration.identity.description,
     icon: dataSourceItem.declaration.identity.icon,

web/app/components/workflow/index.tsx

@@ -375,4 +375,4 @@ const WorkflowWithDefaultContext = ({
   )
 }
 
-export default memo(WorkflowWithDefaultContext)
+export default WorkflowWithDefaultContext

web/app/components/workflow/nodes/_base/components/variable/utils.ts

@@ -53,6 +53,8 @@ export const isConversationVar = (valueSelector: ValueSelector) => {
 }
 
 export const isRagVariableVar = (valueSelector: ValueSelector) => {
+  if(!valueSelector)
+    return false
   return valueSelector[0] === 'rag'
 }
 
@@ -498,7 +500,7 @@ const formatItem = (
     case 'rag': {
       res.vars = data.ragVariables.map((ragVar: RAGPipelineVariable) => {
         return {
-          variable: `rag.${ragVar.variable}`,
+          variable: `rag.shared.${ragVar.variable}`,
           type: inputVarTypeToVarType(ragVar.type as any),
           des: ragVar.label,
           isRagVariable: true,
@@ -636,10 +638,11 @@ export const toNodeOutputVars = (
     let ragVariablesInDataSource: RAGPipelineVariable[] = []
     if (node.data.type === BlockEnum.DataSource)
       ragVariablesInDataSource = ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === node.id)
+    console.log(ragVariables, ragVariablesInDataSource, node.id)
     return {
       ...formatItem(node, isChatMode, filterVar, ragVariablesInDataSource.map(
         (ragVariable: RAGPipelineVariable) => ({
-          variable: ragVariable.variable,
+          variable: `rag.${node.id}.${ragVariable.variable}`,
           type: inputVarTypeToVarType(ragVariable.type as any),
           description: ragVariable.label,
           isRagVariable: true,

web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx

@@ -183,7 +183,7 @@ const VarReferencePicker: FC<Props> = ({
     }
   }, [value, hasValue, isConstant, isIterationVar, iterationNode, availableNodes, outputVarNodeId, startNode, isLoopVar, loopNode])
 
-  const isShowAPart = (value as ValueSelector).length > 2
+  const isShowAPart = (value as ValueSelector).length > 2 && !isRagVariableVar((value as ValueSelector))
 
   const varName = useMemo(() => {
     if (!hasValue)

web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx

@@ -165,7 +165,7 @@ const Item: FC<ItemProps> = ({
               <div title={itemData.des} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable.replace('conversation.', '')}</div>
             )}
             {isRagVariable && (
-              <div title={itemData.des} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable.replace('rag.', '')}</div>
+              <div title={itemData.des} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable.split('.').slice(-1)[0]}</div>
             )}
           </div>
           <div className='ml-1 shrink-0 text-xs font-normal capitalize text-text-tertiary'>{itemData.type}</div>

web/app/components/workflow/nodes/data-source/constants.ts

@@ -23,13 +23,16 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [
   'html',
 ]
 
-export const OUTPUT_VARIABLES_MAP = {
-  datasource_type: {
+export const COMMON_OUTPUT = [
+  {
     name: 'datasource_type',
     type: VarType.string,
     description: 'local_file, online_document, website_crawl',
   },
-  file: {
+]
+
+export const FILE_OUTPUT = [
+  {
     name: 'file',
     type: VarType.file,
     description: 'file',
@@ -76,4 +79,27 @@ export const OUTPUT_VARIABLES_MAP = {
       },
     ],
   },
-}
+]
+
+export const WEBSITE_OUTPUT = [
+  {
+    name: 'source_url',
+    type: VarType.string,
+    description: 'The URL of the crawled website',
+  },
+  {
+    name: 'content',
+    type: VarType.string,
+    description: 'The content of the crawled website',
+  },
+  {
+    name: 'title',
+    type: VarType.string,
+    description: 'The title of the crawled website',
+  },
+  {
+    name: 'description',
+    type: VarType.string,
+    description: 'The description of the crawled website',
+  },
+]

web/app/components/workflow/nodes/data-source/default.ts

@@ -1,8 +1,13 @@
 import type { NodeDefault } from '../../types'
 import type { DataSourceNodeType } from './types'
+import { DataSourceClassification } from './types'
 import { genNodeMetaData } from '@/app/components/workflow/utils'
 import { BlockEnum } from '@/app/components/workflow/types'
-import { OUTPUT_VARIABLES_MAP } from './constants'
+import {
+  COMMON_OUTPUT,
+  FILE_OUTPUT,
+  WEBSITE_OUTPUT,
+} from './constants'
 
 const metaData = genNodeMetaData({
   sort: -1,
@@ -24,20 +29,18 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
     const {
       provider_type,
     } = payload
-    const isLocalFile = provider_type === 'local_file'
+    const isLocalFile = provider_type === DataSourceClassification.file
+    const isWebsiteCrawl = provider_type === DataSourceClassification.website
     return [
-      {
-        variable: OUTPUT_VARIABLES_MAP.datasource_type.name,
-        type: OUTPUT_VARIABLES_MAP.datasource_type.type,
-      },
+      ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })),
       ...(
         isLocalFile
-          ? [
-              {
-                variable: OUTPUT_VARIABLES_MAP.file.name,
-                type: OUTPUT_VARIABLES_MAP.file.type,
-              },
-          ]
+          ? FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
+          : []
+      ),
+      ...(
+        isWebsiteCrawl
+          ? WEBSITE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
           : []
       ),
       ...ragVars,

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next'
 import { memo } from 'react'
 import { useBoolean } from 'ahooks'
 import type { DataSourceNodeType } from './types'
+import { DataSourceClassification } from './types'
 import type { NodePanelProps } from '@/app/components/workflow/types'
 import {
   BoxGroupField,
@@ -17,7 +18,11 @@ import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/compo
 import TagInput from '@/app/components/base/tag-input'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import { useConfig } from './hooks/use-config'
-import { OUTPUT_VARIABLES_MAP } from './constants'
+import {
+  COMMON_OUTPUT,
+  FILE_OUTPUT,
+  WEBSITE_OUTPUT,
+} from './constants'
 import { useStore } from '@/app/components/workflow/store'
 import Button from '@/app/components/base/button'
 import ConfigCredential from './components/config-credential'
@@ -35,7 +40,7 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const dataSourceList = useStore(s => s.dataSourceList)
   const {
     provider_type,
-    provider_id,
+    plugin_id,
     fileExtensions = [],
     datasource_parameters,
   } = data
@@ -43,8 +48,9 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
     handleFileExtensionsChange,
     handleParametersChange,
   } = useConfig(id)
-  const isLocalFile = provider_type === 'local_file'
-  const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === provider_id)
+  const isLocalFile = provider_type === DataSourceClassification.file
+  const isWebsiteCrawl = provider_type === DataSourceClassification.website
+  const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id)
   const isAuthorized = !!currentDataSource?.is_authorized
   const [showAuthModal, {
     setTrue: openAuthModal,
@@ -150,24 +156,37 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
         )
       }
       <OutputVars>
-        <VarItem
-          name={OUTPUT_VARIABLES_MAP.datasource_type.name}
-          type={OUTPUT_VARIABLES_MAP.datasource_type.type}
-          description={OUTPUT_VARIABLES_MAP.datasource_type.description}
-        />
         {
-          isLocalFile && (
+          COMMON_OUTPUT.map(item => (
             <VarItem
-              name={OUTPUT_VARIABLES_MAP.file.name}
-              type={OUTPUT_VARIABLES_MAP.file.type}
-              description={OUTPUT_VARIABLES_MAP.file.description}
-              subItems={OUTPUT_VARIABLES_MAP.file.subItems.map(item => ({
+              name={item.name}
+              type={item.type}
+              description={item.description}
+            />
+          ))
+        }
+        {
+          isLocalFile && FILE_OUTPUT.map(item => (
+            <VarItem
+              name={item.name}
+              type={item.type}
+              description={item.description}
+              subItems={item.subItems.map(item => ({
                 name: item.name,
                 type: item.type,
                 description: item.description,
               }))}
             />
-          )
+          ))
+        }
+        {
+          isWebsiteCrawl && WEBSITE_OUTPUT.map(item => (
+            <VarItem
+              name={item.name}
+              type={item.type}
+              description={item.description}
+            />
+          ))
         }
       </OutputVars>
       {

web/app/components/workflow/nodes/data-source/types.ts

@@ -6,6 +6,11 @@ export enum VarType {
   mixed = 'mixed',
 }
 
+export enum DataSourceClassification {
+  file = 'local_file',
+  website = 'website_crawl',
+}
+
 export type ToolVarInputs = Record<string, {
   type: VarType
   value?: string | ValueSelector | any
@@ -13,7 +18,7 @@ export type ToolVarInputs = Record<string, {
 
 export type DataSourceNodeType = CommonNodeType & {
   fileExtensions?: string[]
-  provider_id: string
+  plugin_id: string
   provider_type: string
   provider_name: string
   datasource_name: string

web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx

@@ -38,7 +38,13 @@ const NodeVariableItem = ({
   const isEnv = isENV(variable)
   const isChatVar = isConversationVar(variable)
   const isRagVar = isRagVariableVar(variable)
-  const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
+  const varName = useMemo(() => {
+    if(isSystem)
+      return `sys.${variable[variable.length - 1]}`
+    if(isRagVar)
+      return variable[variable.length - 1]
+    return variable.slice(1).join('.')
+  }, [isRagVar, isSystem, variable])
 
   const VariableIcon = useMemo(() => {
     if (isEnv) {

web/app/components/workflow/types.ts

@@ -100,7 +100,7 @@ export type CommonNodeType<T = {}> = {
   retry_config?: WorkflowRetryConfig
   default_value?: DefaultValueForm[]
 } & T & Partial<Pick<ToolDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'tool_name'>>
-  & Partial<Pick<DataSourceDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'datasource_name'>>
+  & Partial<Pick<DataSourceDefaultValue, 'plugin_id' | 'provider_type' | 'provider_name' | 'datasource_name'>>
 
 export type CommonEdgeType = {
   _hovering?: boolean