From 5d25199f428f8b46f7779599329f0a628a52b4f2 Mon Sep 17 00:00:00 2001 From: twwu Date: Thu, 19 Jun 2025 09:36:04 +0800 Subject: [PATCH 1/4] refactor: update layout for creation title and content in StepThree component --- web/app/components/datasets/create/step-three/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/step-three/index.tsx b/web/app/components/datasets/create/step-three/index.tsx index 3400313e99..bd1d7701fd 100644 --- a/web/app/components/datasets/create/step-three/index.tsx +++ b/web/app/components/datasets/create/step-three/index.tsx @@ -39,7 +39,7 @@ const StepThree = ({ datasetId, datasetName, indexingType, creationCache, retrie
{t('datasetCreation.stepThree.creationTitle')}
{t('datasetCreation.stepThree.creationContent')}
-
+
Date: Thu, 19 Jun 2025 10:56:03 +0800 Subject: [PATCH 2/4] fix: add type checks for workspace roles in DatasetsLayout component --- web/app/(commonLayout)/datasets/layout.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/app/(commonLayout)/datasets/layout.tsx b/web/app/(commonLayout)/datasets/layout.tsx index b3fc5147b5..e44a232146 100644 --- a/web/app/(commonLayout)/datasets/layout.tsx +++ b/web/app/(commonLayout)/datasets/layout.tsx @@ -12,6 +12,8 @@ export default function DatasetsLayout({ children }: { children: React.ReactNode const router = useRouter() useEffect(() => { + if (typeof isCurrentWorkspaceEditor !== 'boolean' || typeof isCurrentWorkspaceDatasetOperator !== 'boolean') + return if (!isCurrentWorkspaceEditor && !isCurrentWorkspaceDatasetOperator) router.replace('/apps') }, [isCurrentWorkspaceEditor, isCurrentWorkspaceDatasetOperator, router]) From 95ba55af4dd642f6d6e42374c1f9d3e4d55c18c2 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Thu, 19 Jun 2025 15:04:26 +0800 Subject: [PATCH 3/4] fix: import dsl sync rag variables --- .../components/rag-pipeline-main.tsx | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/web/app/components/rag-pipeline/components/rag-pipeline-main.tsx b/web/app/components/rag-pipeline/components/rag-pipeline-main.tsx index cf9b9744e9..82c80684b8 100644 --- a/web/app/components/rag-pipeline/components/rag-pipeline-main.tsx +++ b/web/app/components/rag-pipeline/components/rag-pipeline-main.tsx @@ -1,4 +1,5 @@ import { + useCallback, useMemo, } from 'react' import { WorkflowWithInnerContext } from '@/app/components/workflow' @@ -13,6 +14,7 @@ import { usePipelineRun, usePipelineStartRun, } from '../hooks' +import { useWorkflowStore } from '@/app/components/workflow/store' type RagPipelineMainProps = Pick const RagPipelineMain = ({ @@ -20,6 +22,23 @@ const RagPipelineMain = ({ edges, viewport, }: RagPipelineMainProps) => { + const workflowStore = useWorkflowStore() + + const handleWorkflowDataUpdate = useCallback((payload: any) => { + const { + rag_pipeline_variables, + environment_variables, + } = payload + if (rag_pipeline_variables) { + const { setRagPipelineVariables } = workflowStore.getState() + setRagPipelineVariables?.(rag_pipeline_variables) + } + if (environment_variables) { + const { setEnvironmentVariables } = workflowStore.getState() + setEnvironmentVariables(environment_variables) + } + }, [workflowStore]) + const { doSyncWorkflowDraft, syncWorkflowDraftWhenPageClose, @@ -83,6 +102,7 @@ const RagPipelineMain = ({ edges={edges} viewport={viewport} hooksStore={hooksStore as any} + onWorkflowDataUpdate={handleWorkflowDataUpdate} > From 335e1e360209b5bbcda6b4fb387145a850c00eed Mon Sep 17 00:00:00 2001 From: twwu Date: Thu, 19 Jun 2025 15:57:49 +0800 Subject: [PATCH 4/4] feat: enhance pipeline settings with execution log and processing capabilities --- .../preview/chunk-preview.tsx | 3 - .../settings/pipeline-settings/index.tsx | 151 +++++++++++++----- .../process-documents/actions.tsx | 3 + .../process-documents/hooks.ts | 56 ++++++- .../process-documents/index.tsx | 10 +- web/models/pipeline.ts | 12 ++ web/service/use-pipeline.ts | 13 ++ 7 files changed, 193 insertions(+), 55 deletions(-) diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx index a17024efa7..e926422b1d 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx @@ -147,9 +147,6 @@ const ChunkPreview = ({ {!isPending && currentDocForm === ChunkingMode.parentChild && estimateData?.preview && ( estimateData?.preview?.map((item, index) => { const indexForLabel = index + 1 - // const childChunks = parentChildConfig.chunkForContext === 'full-doc' - // ? item.child_chunks.slice(0, FULL_DOC_PREVIEW_LENGTH) - // : item.child_chunks return ( { const { t } = useTranslation() - const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const { push } = useRouter() const [estimateData, setEstimateData] = useState(undefined) + const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id) const isPreview = useRef(false) const formRef = useRef(null) - const { data: documentDetail, error, isFetching: isFetchingDocumentDetail } = useDocumentDetail({ - datasetId, - documentId, - params: { metadata: 'without' }, + const { data: lastRunData, isFetching: isFetchingLastRunData, isError } = usePipelineExecutionLog({ + dataset_id: datasetId, + document_id: documentId, }) - const handlePreviewChunks = useCallback(async (data: Record) => { - // todo: Preview - }, []) + const files = useMemo(() => { + const files: CustomFile[] = [] + if (lastRunData?.datasource_type === DatasourceType.localFile) { + const { related_id, name, extension } = lastRunData.datasource_info + files.push({ + id: related_id, + name, + extension, + } as CustomFile) + } + return files + }, [lastRunData]) + const websitePages = useMemo(() => { + const websitePages: CrawlResultItem[] = [] + if (lastRunData?.datasource_type === DatasourceType.websiteCrawl) { + const { content, description, source_url, title } = lastRunData.datasource_info + websitePages.push({ + markdown: content, + description, + source_url, + title, + }) + } + return websitePages + }, [lastRunData]) + + const onlineDocuments = useMemo(() => { + const onlineDocuments: NotionPage[] = [] + if (lastRunData?.datasource_type === DatasourceType.onlineDocument) { + const { workspace_id, page } = lastRunData.datasource_info + onlineDocuments.push({ + workspace_id, + ...page, + }) + } + return onlineDocuments + }, [lastRunData]) + + const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() + + const handlePreviewChunks = useCallback(async (data: Record) => { + if (!lastRunData) + return + const datasourceInfoList: Record[] = [] + const documentInfo = lastRunData.datasource_info + datasourceInfoList.push(documentInfo) + await runPublishedPipeline({ + pipeline_id: pipelineId!, + inputs: data, + start_node_id: lastRunData.datasource_node_id, + datasource_type: lastRunData.datasource_type, + datasource_info_list: datasourceInfoList, + is_preview: true, + }, { + onSuccess: (res) => { + setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) + }, + }) + }, [lastRunData, pipelineId, runPublishedPipeline]) + + const invalidDocumentList = useInvalidDocumentList(datasetId) const handleProcess = useCallback(async (data: Record) => { - // todo: Process - }, []) + if (!lastRunData) + return + const datasourceInfoList: Record[] = [] + const documentInfo = lastRunData.datasource_info + datasourceInfoList.push(documentInfo) + await runPublishedPipeline({ + pipeline_id: pipelineId!, + inputs: data, + start_node_id: lastRunData.datasource_node_id, + datasource_type: lastRunData.datasource_type, + datasource_info_list: datasourceInfoList, + is_preview: false, + }, { + onSuccess: () => { + invalidDocumentList() + push(`/datasets/${datasetId}/documents/${documentId}`) + }, + }) + }, [datasetId, documentId, invalidDocumentList, lastRunData, pipelineId, push, runPublishedPipeline]) const onClickProcess = useCallback(() => { isPreview.current = false @@ -55,25 +134,13 @@ const PipelineSettings = ({ isPreview.current ? handlePreviewChunks(data) : handleProcess(data) }, [handlePreviewChunks, handleProcess]) - const handlePreviewFileChange = useCallback((file: DocumentItem) => { - onClickPreview() - }, [onClickPreview]) - - const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { - onClickPreview() - }, [onClickPreview]) - - const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { - onClickPreview() - }, [onClickPreview]) - - if (isFetchingDocumentDetail) { + if (isFetchingLastRunData) { return ( ) } - if (error) + if (isError) return return ( @@ -85,7 +152,8 @@ const PipelineSettings = ({
diff --git a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx index 22057882d3..6505566b89 100644 --- a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx +++ b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/actions.tsx @@ -3,11 +3,13 @@ import Button from '@/app/components/base/button' import { useTranslation } from 'react-i18next' type ActionsProps = { + runDisabled?: boolean onProcess: () => void } const Actions = ({ onProcess, + runDisabled, }: ActionsProps) => { const { t } = useTranslation() @@ -16,6 +18,7 @@ const Actions = ({ diff --git a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts index 139e225656..271d7abdde 100644 --- a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts +++ b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/hooks.ts @@ -1,10 +1,58 @@ -import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types' +import { useMemo } from 'react' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline' +import { VAR_TYPE_MAP } from '@/models/pipeline' +import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types' -export const useConfigurations = (documentdId: string) => { - const initialData: Record = {} - const configurations: BaseConfiguration[] = [] +export const useConfigurations = (lastRunInputData: Record, datasourceNodeId: string) => { + const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id) + const { data: paramsConfig, isFetching: isFetchingParams } = usePublishedPipelineProcessingParams({ + pipeline_id: pipelineId!, + node_id: datasourceNodeId, + }) + + const initialData = useMemo(() => { + const variables = paramsConfig?.variables || [] + return variables.reduce((acc, item) => { + const type = VAR_TYPE_MAP[item.type] + const variableName = item.variable + if ([BaseFieldType.textInput, BaseFieldType.paragraph, BaseFieldType.select].includes(type)) + acc[item.variable] = lastRunInputData[variableName] ?? '' + if (type === BaseFieldType.numberInput) + acc[item.variable] = lastRunInputData[variableName] ?? 0 + if (type === BaseFieldType.checkbox) + acc[item.variable] = lastRunInputData[variableName] + if ([BaseFieldType.file, BaseFieldType.fileList].includes(type)) + acc[item.variable] = lastRunInputData[variableName] + return acc + }, {} as Record) + }, [lastRunInputData, paramsConfig?.variables]) + + const configurations = useMemo(() => { + const variables = paramsConfig?.variables || [] + const configs = variables.map(item => ({ + type: VAR_TYPE_MAP[item.type], + variable: item.variable, + label: item.label, + required: item.required, + maxLength: item.max_length, + options: item.options?.map(option => ({ + label: option, + value: option, + })), + showConditions: [], + placeholder: item.placeholder, + tooltip: item.tooltips, + unit: item.unit, + allowedFileTypes: item.allowed_file_types, + allowedFileExtensions: item.allowed_file_extensions, + allowedFileUploadMethods: item.allowed_file_upload_methods, + })) + return configs + }, [paramsConfig]) return { + isFetchingParams, initialData, configurations, } diff --git a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx index 54108bdfd0..d68695ed09 100644 --- a/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx +++ b/web/app/components/datasets/documents/detail/settings/pipeline-settings/process-documents/index.tsx @@ -4,7 +4,8 @@ import Actions from './actions' import Form from '../../../../create-from-pipeline/process-documents/form' type ProcessDocumentsProps = { - documentId: string + datasourceNodeId: string + lastRunInputData: Record ref: React.RefObject onProcess: () => void onPreview: () => void @@ -12,13 +13,14 @@ type ProcessDocumentsProps = { } const ProcessDocuments = ({ - documentId, + datasourceNodeId, + lastRunInputData, onProcess, onPreview, onSubmit, ref, }: ProcessDocumentsProps) => { - const { initialData, configurations } = useConfigurations(documentId) + const { isFetchingParams, initialData, configurations } = useConfigurations(lastRunInputData, datasourceNodeId) const schema = generateZodSchema(configurations) return ( @@ -31,7 +33,7 @@ const ProcessDocuments = ({ onSubmit={onSubmit} onPreview={onPreview} /> - +
) } diff --git a/web/models/pipeline.ts b/web/models/pipeline.ts index f557ec3f81..29496960c4 100644 --- a/web/models/pipeline.ts +++ b/web/models/pipeline.ts @@ -242,3 +242,15 @@ export type InitialDocumentDetail = { name: string position: number } + +export type PipelineExecutionLogRequest = { + dataset_id: string + document_id: string +} + +export type PipelineExecutionLogResponse = { + datasource_info: Record + datasource_type: DatasourceType + input_data: Record + datasource_node_id: string +} diff --git a/web/service/use-pipeline.ts b/web/service/use-pipeline.ts index f8c047000c..e55b72a808 100644 --- a/web/service/use-pipeline.ts +++ b/web/service/use-pipeline.ts @@ -8,6 +8,8 @@ import type { ImportPipelineDSLRequest, ImportPipelineDSLResponse, PipelineCheckDependenciesResponse, + PipelineExecutionLogRequest, + PipelineExecutionLogResponse, PipelinePreProcessingParamsRequest, PipelinePreProcessingParamsResponse, PipelineProcessingParamsRequest, @@ -311,3 +313,14 @@ export const usePublishAsCustomizedPipeline = () => { }, }) } + +export const usePipelineExecutionLog = (params: PipelineExecutionLogRequest) => { + const { dataset_id, document_id } = params + return useQuery({ + queryKey: [NAME_SPACE, 'pipeline-execution-log', dataset_id, document_id], + queryFn: () => { + return get(`/datasets/${dataset_id}/documents/${document_id}/pipeline-execution-log`) + }, + staleTime: 0, + }) +}