feat: knowledge pipeline (#25360)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: twwu <twwu@dify.ai>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: jyong <718720800@qq.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: lyzno1 <yuanyouhuilyz@gmail.com>
Co-authored-by: quicksand <quicksandzn@gmail.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: lyzno1 <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: Yongtao Huang <yongtaoh2022@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: nite-knite <nkCoding@gmail.com>
Co-authored-by: Hanqing Zhao <sherry9277@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry <xh001x@hotmail.com>

web/types/common.ts

@@ -0,0 +1,4 @@
+export enum FlowType {
+  appFlow = 'appFlow',
+  ragPipeline = 'ragPipeline',
+}

web/types/i18n.d.ts

@@ -17,6 +17,7 @@ type CustomMessages = typeof import('../i18n/en-US/custom').default
 type DatasetCreationMessages = typeof import('../i18n/en-US/dataset-creation').default
 type DatasetDocumentsMessages = typeof import('../i18n/en-US/dataset-documents').default
 type DatasetHitTestingMessages = typeof import('../i18n/en-US/dataset-hit-testing').default
+type DatasetPipelineMessages = typeof import('../i18n/en-US/dataset-pipeline').default
 type DatasetSettingsMessages = typeof import('../i18n/en-US/dataset-settings').default
 type DatasetMessages = typeof import('../i18n/en-US/dataset').default
 type EducationMessages = typeof import('../i18n/en-US/education').default
@@ -24,6 +25,7 @@ type ExploreMessages = typeof import('../i18n/en-US/explore').default
 type LayoutMessages = typeof import('../i18n/en-US/layout').default
 type LoginMessages = typeof import('../i18n/en-US/login').default
 type OauthMessages = typeof import('../i18n/en-US/oauth').default
+type PipelineMessages = typeof import('../i18n/en-US/pipeline').default
 type PluginTagsMessages = typeof import('../i18n/en-US/plugin-tags').default
 type PluginMessages = typeof import('../i18n/en-US/plugin').default
 type RegisterMessages = typeof import('../i18n/en-US/register').default
@@ -47,6 +49,7 @@ export type Messages = {
   datasetCreation: DatasetCreationMessages;
   datasetDocuments: DatasetDocumentsMessages;
   datasetHitTesting: DatasetHitTestingMessages;
+  datasetPipeline: DatasetPipelineMessages;
   datasetSettings: DatasetSettingsMessages;
   dataset: DatasetMessages;
   education: EducationMessages;
@@ -54,6 +57,7 @@ export type Messages = {
   layout: LayoutMessages;
   login: LoginMessages;
   oauth: OauthMessages;
+  pipeline: PipelineMessages;
   pluginTags: PluginTagsMessages;
   plugin: PluginMessages;
   register: RegisterMessages;

web/types/pipeline.tsx

@@ -0,0 +1,47 @@
+import type { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth'
+
+export type DataSourceNodeProcessingResponse = {
+  event: 'datasource_processing'
+  total: number
+  completed: number
+}
+
+export type DataSourceNodeError = {
+  event: 'datasource_error'
+  message: string
+  code?: string
+}
+
+export type OnlineDriveFile = {
+  id: string
+  name: string
+  size: number
+  type: 'file' | 'folder'
+}
+
+export type OnlineDriveData = {
+  bucket: string
+  files: OnlineDriveFile[]
+  is_truncated: boolean
+  next_page_parameters: Record<string, any>
+}
+
+export type DataSourceNodeCompletedResponse = {
+  event: 'datasource_completed'
+  data: any
+  time_consuming: number
+}
+
+export type DataSourceNodeErrorResponse = {
+  event: 'datasource_error'
+  error: string
+}
+
+export type DataSourceCredential = {
+  avatar_url?: string
+  credential: Record<string, any>
+  id: string
+  is_default: boolean
+  name: string
+  type: CredentialTypeEnum
+}

web/types/workflow.ts

@@ -2,13 +2,14 @@ import type { Viewport } from 'reactflow'
 import type { BlockEnum, CommonNodeType, ConversationVariable, Edge, EnvironmentVariable, InputVar, Node, ValueSelector, VarType, Variable } from '@/app/components/workflow/types'
 import type { TransferMethod } from '@/types/app'
 import type { ErrorHandleTypeEnum } from '@/app/components/workflow/nodes/_base/components/error-handle/types'
+import type { RAGPipelineVariables } from '@/models/pipeline'
 import type { BeforeRunFormProps } from '@/app/components/workflow/nodes/_base/components/before-run-form'
 import type { SpecialResultPanelProps } from '@/app/components/workflow/run/special-result-panel'
 import type { RefObject } from 'react'
 
 export type AgentLogItem = {
   node_execution_id: string,
-  id: string,
+  message_id: string,
   node_id: string,
   parent_id?: string,
   label: string,
@@ -37,8 +38,14 @@ export type NodeTracing = {
   node_type: BlockEnum
   title: string
   inputs: any
+  inputs_truncated: boolean
   process_data: any
+  process_data_truncated: boolean
   outputs?: Record<string, any>
+  outputs_truncated: boolean
+  outputs_full_content?: {
+    download_url: string
+  }
   status: string
   parallel_run_id?: string
   error?: string
@@ -123,6 +130,7 @@ export type FetchWorkflowDraftResponse = {
   tool_published: boolean
   environment_variables?: EnvironmentVariable[]
   conversation_variables?: ConversationVariable[]
+  rag_pipeline_variables?: RAGPipelineVariables
   version: string
   marked_name: string
   marked_comment: string
@@ -131,7 +139,7 @@ export type FetchWorkflowDraftResponse = {
 export type VersionHistory = FetchWorkflowDraftResponse
 
 export type FetchWorkflowDraftPageParams = {
-  appId: string
+  url: string
   initialPage: number
   limit: number
   userId?: string
@@ -345,12 +353,13 @@ export type WorkflowConfigResponse = {
 }
 
 export type PublishWorkflowParams = {
+  url: string
   title: string
   releaseNotes: string
 }
 
 export type UpdateWorkflowParams = {
-  workflowId: string
+  url: string
   title: string
   releaseNotes: string
 }
@@ -378,6 +387,11 @@ export enum VarInInspectType {
   system = 'sys',
 }
 
+export type FullContent = {
+  size_bytes: number
+  download_url: string
+}
+
 export type VarInInspect = {
   id: string
   type: VarInInspectType
@@ -388,6 +402,9 @@ export type VarInInspect = {
   value: any
   edited: boolean
   visible: boolean
+  is_truncated: boolean
+  full_content: FullContent
+  schemaType?: string
 }
 
 export type NodeWithVar = {