jandres/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-01-06 06:26:00 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

refactor: Refactor online document and online drive components to handle credential changes

Browse Source
This commit is contained in:
twwu
2025-08-08 13:43:01 +08:00
parent 097a6fc1e0
commit 184c3c88b7
17 changed files with 220 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx
View File

@@ -20,12 +20,14 @@ type OnlineDocumentsProps = {
isInPipeline?: boolean
nodeId: string
nodeData: DataSourceNodeType
onCredentialChange: (credentialId: string) => void
}
const OnlineDocuments = ({
nodeId,
nodeData,
isInPipeline = false,
onCredentialChange,
}: OnlineDocumentsProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
@@ -33,13 +35,11 @@ const OnlineDocuments = ({
documentsData,
searchValue,
selectedPagesId,
currentWorkspaceId,
currentCredentialId,
} = useDataSourceStoreWithSelector(useShallow(state => ({
documentsData: state.documentsData,
searchValue: state.searchValue,
selectedPagesId: state.selectedPagesId,
currentWorkspaceId: state.currentWorkspaceId,
currentCredentialId: state.currentCredentialId,
})))
@@ -69,19 +69,21 @@ const OnlineDocuments = ({
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const getOnlineDocuments = useCallback(async () => {
const { currentCredentialId } = dataSourceStore.getState()
if (!currentCredentialId) return
ssePost(
datasourceNodeRunURL,
{
body: {
inputs: {},
credential_id: currentCredentialId,
datasource_type: DatasourceType.onlineDocument,
},
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
const { setDocumentsData, setCurrentWorkspaceId } = dataSourceStore.getState()
const { setDocumentsData } = dataSourceStore.getState()
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
setCurrentWorkspaceId(documentsData.data[0].workspace_id)
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
Toast.notify({
@@ -94,33 +96,8 @@ const OnlineDocuments = ({
}, [dataSourceStore, datasourceNodeRunURL])
useEffect(() => {
const {
setDocumentsData,
setCurrentWorkspaceId,
setSearchValue,
setSelectedPagesId,
setOnlineDocuments,
setCurrentDocument,
currentNodeIdRef,
} = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) {
setDocumentsData([])
setCurrentWorkspaceId('')
setSearchValue('')
setSelectedPagesId(new Set())
setOnlineDocuments([])
setCurrentDocument(undefined)
currentNodeIdRef.current = nodeId
getOnlineDocuments()
}
else {
// Avoid fetching documents when come back from next step
if (!documentsData.length)
getOnlineDocuments()
}
}, [nodeId])
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
getOnlineDocuments()
}, [currentCredentialId])
const handleSearchValueChange = useCallback((value: string) => {
const { setSearchValue } = dataSourceStore.getState()
@@ -153,7 +130,7 @@ const OnlineDocuments = ({
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={currentCredentialId}
onCredentialChange={dataSourceStore.getState().setCurrentCredentialId}
onCredentialChange={onCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
@@ -172,13 +149,13 @@ const OnlineDocuments = ({
checkedIds={selectedPagesId}
disabledValue={new Set()}
searchValue={searchValue}
list={currentWorkspace?.pages || []}
list={documentsData[0].pages || []}
pagesMap={PagesMapAndSelectedPagesId}
onSelect={handleSelectPages}
canPreview={!isInPipeline}
onPreview={handlePreviewPage}
isMultipleChoice={!isInPipeline}
currentWorkspaceId={currentWorkspaceId}
currentCredentialId={currentCredentialId}
/>
) : (
<div className='flex h-[296px] items-center justify-center'>

7
web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx
View File

@@ -15,7 +15,7 @@ type PageSelectorProps = {
canPreview?: boolean
onPreview?: (selectedPageId: string) => void
isMultipleChoice?: boolean
currentWorkspaceId: string
currentCredentialId: string
}
export type NotionPageTreeItem = {
@@ -42,7 +42,7 @@ const PageSelector = ({
canPreview = true,
onPreview,
isMultipleChoice = true,
currentWorkspaceId,
currentCredentialId,
}: PageSelectorProps) => {
const { t } = useTranslation()
const [dataList, setDataList] = useState<NotionPageItem[]>([])
@@ -56,8 +56,7 @@ const PageSelector = ({
depth: 0,
}
}))
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [currentWorkspaceId])
}, [currentCredentialId])
const searchDataList = list.filter((item) => {
return item.page_name.includes(searchValue)

27
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
View File

@@ -4,7 +4,6 @@ import { useDataSourceStore } from '../../../../store'
import Bucket from './bucket'
import BreadcrumbItem from './item'
import Dropdown from './dropdown'
import type { OnlineDriveFile } from '@/models/pipeline'
type BreadcrumbsProps = {
prefix: string[]
@@ -12,12 +11,6 @@ type BreadcrumbsProps = {
bucket: string
searchResultsLength: number
isInPipeline: boolean
getOnlineDriveFiles: (params: {
prefix?: string[]
bucket?: string
startAfter?: string
fileList?: OnlineDriveFile[]
}) => void
}
const Breadcrumbs = ({
@@ -26,7 +19,6 @@ const Breadcrumbs = ({
bucket,
searchResultsLength,
isInPipeline,
getOnlineDriveFiles,
}: BreadcrumbsProps) => {
const { t } = useTranslation()
const dataSourceStore = useDataSourceStore()
@@ -56,23 +48,14 @@ const Breadcrumbs = ({
setSelectedFileKeys([])
setBucket('')
setPrefix([])
getOnlineDriveFiles({
prefix: [],
bucket: '',
fileList: [],
})
}, [dataSourceStore, getOnlineDriveFiles])
}, [dataSourceStore])
const handleClickBucketName = useCallback(() => {
const { setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState()
setFileList([])
setSelectedFileKeys([])
setPrefix([])
getOnlineDriveFiles({
prefix: [],
fileList: [],
})
}, [dataSourceStore, getOnlineDriveFiles])
}, [dataSourceStore])
const handleClickBreadcrumb = useCallback((index: number) => {
const { prefix, setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState()
@@ -80,11 +63,7 @@ const Breadcrumbs = ({
setFileList([])
setSelectedFileKeys([])
setPrefix(newPrefix)
getOnlineDriveFiles({
prefix: newPrefix,
fileList: [],
})
}, [dataSourceStore, getOnlineDriveFiles])
}, [dataSourceStore])
return (
<div className='flex grow items-center overflow-hidden'>

9
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/index.tsx
View File

@@ -2,7 +2,6 @@ import React from 'react'
import Breadcrumbs from './breadcrumbs'
import Input from '@/app/components/base/input'
import { useTranslation } from 'react-i18next'
import type { OnlineDriveFile } from '@/models/pipeline'
type HeaderProps = {
prefix: string[]
@@ -13,12 +12,6 @@ type HeaderProps = {
handleInputChange: React.ChangeEventHandler<HTMLInputElement>
handleResetKeywords: () => void
isInPipeline: boolean
getOnlineDriveFiles: (params: {
prefix?: string[]
bucket?: string
startAfter?: string
fileList?: OnlineDriveFile[]
}) => void
}
const Header = ({
@@ -30,7 +23,6 @@ const Header = ({
searchResultsLength,
handleInputChange,
handleResetKeywords,
getOnlineDriveFiles,
}: HeaderProps) => {
const { t } = useTranslation()
@@ -42,7 +34,6 @@ const Header = ({
bucket={bucket}
searchResultsLength={searchResultsLength}
isInPipeline={isInPipeline}
getOnlineDriveFiles={getOnlineDriveFiles}
/>
<Input
value={inputValue}

9
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
View File

@@ -17,12 +17,6 @@ type FileListProps = {
handleSelectFile: (file: OnlineDriveFile) => void
handleOpenFolder: (file: OnlineDriveFile) => void
isLoading: boolean
getOnlineDriveFiles: (params: {
prefix?: string[]
bucket?: string
startAfter?: string
fileList?: OnlineDriveFile[]
}) => void
}
const FileList = ({
@@ -38,7 +32,6 @@ const FileList = ({
handleOpenFolder,
isInPipeline,
isLoading,
getOnlineDriveFiles,
}: FileListProps) => {
const [inputValue, setInputValue] = useState(keywords)
@@ -71,7 +64,6 @@ const FileList = ({
handleInputChange={handleInputChange}
searchResultsLength={searchResultsLength}
handleResetKeywords={handleResetKeywords}
getOnlineDriveFiles={getOnlineDriveFiles}
/>
<List
fileList={fileList}
@@ -82,7 +74,6 @@ const FileList = ({
handleSelectFile={handleSelectFile}
isInPipeline={isInPipeline}
isLoading={isLoading}
getOnlineDriveFiles={getOnlineDriveFiles}
/>
</div>
)

18
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
View File

@@ -16,12 +16,6 @@ type FileListProps = {
handleResetKeywords: () => void
handleSelectFile: (file: OnlineDriveFile) => void
handleOpenFolder: (file: OnlineDriveFile) => void
getOnlineDriveFiles: (params: {
prefix?: string[]
bucket?: string
startAfter?: string
fileList?: OnlineDriveFile[]
}) => void
}
const List = ({
@@ -33,27 +27,23 @@ const List = ({
handleOpenFolder,
isInPipeline,
isLoading,
getOnlineDriveFiles,
}: FileListProps) => {
const anchorRef = useRef<HTMLDivElement>(null)
const observerRef = useRef<IntersectionObserver>()
const observerRef = useRef<IntersectionObserver>(null)
const dataSourceStore = useDataSourceStore()
useEffect(() => {
if (anchorRef.current) {
observerRef.current = new IntersectionObserver((entries) => {
const { startAfter, isTruncated } = dataSourceStore.getState()
if (entries[0].isIntersecting && isTruncated.current && !isLoading) {
startAfter.current = fileList[fileList.length - 1].key
getOnlineDriveFiles({ startAfter: fileList[fileList.length - 1].key })
}
const { setStartAfter, isTruncated } = dataSourceStore.getState()
if (entries[0].isIntersecting && isTruncated.current && !isLoading)
setStartAfter(fileList[fileList.length - 1].key)
}, {
rootMargin: '100px',
})
observerRef.current.observe(anchorRef.current)
}
return () => observerRef.current?.disconnect()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [anchorRef])
const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0

87
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
View File

@@ -13,54 +13,55 @@ import { convertOnlineDriveData } from './utils'
import produce from 'immer'
import { useShallow } from 'zustand/react/shallow'
import { useModalContextSelector } from '@/context/modal-context'
import { noop } from 'lodash-es'
import { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth'
import { useGetDataSourceAuth } from '@/service/use-datasource'
type OnlineDriveProps = {
nodeId: string
nodeData: DataSourceNodeType
isInPipeline?: boolean
onCredentialChange: (credentialId: string) => void
}
const OnlineDrive = ({
nodeId,
nodeData,
isInPipeline = false,
onCredentialChange,
}: OnlineDriveProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
const {
startAfter,
prefix,
keywords,
bucket,
selectedFileKeys,
fileList,
currentCredentialId,
} = useDataSourceStoreWithSelector(useShallow(state => ({
startAfter: state.startAfter,
prefix: state.prefix,
keywords: state.keywords,
bucket: state.bucket,
selectedFileKeys: state.selectedFileKeys,
fileList: state.fileList,
currentCredentialId: state.currentCredentialId,
})))
const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
const { data: dataSourceAuth } = useGetDataSourceAuth({
pluginId: nodeData.plugin_id,
provider: nodeData.provider_name,
})
const datasourceNodeRunURL = !isInPipeline
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const getOnlineDriveFiles = useCallback(async (params: {
prefix?: string[]
bucket?: string
startAfter?: string
fileList?: OnlineDriveFile[]
}) => {
const { startAfter, prefix, bucket, fileList } = dataSourceStore.getState()
const _prefix = params.prefix ?? prefix
const _bucket = params.bucket ?? bucket
const _startAfter = params.startAfter ?? startAfter.current
const _fileList = params.fileList ?? fileList
const prefixString = _prefix.length > 0 ? `${_prefix.join('/')}/` : ''
const getOnlineDriveFiles = useCallback(async () => {
const { startAfter, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState()
const prefixString = prefix.length > 0 ? `${prefix.join('/')}/` : ''
setIsLoading(true)
ssePost(
datasourceNodeRunURL,
@@ -68,18 +69,19 @@ const OnlineDrive = ({
body: {
inputs: {
prefix: prefixString,
bucket: _bucket,
start_after: _startAfter,
bucket,
start_after: startAfter,
max_keys: 30, // Adjust as needed
},
datasource_type: DatasourceType.onlineDrive,
credential_id: currentCredentialId,
},
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
const { setFileList, isTruncated } = dataSourceStore.getState()
const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, _prefix, _bucket)
setFileList([..._fileList, ...newFileList])
const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, prefix, bucket)
setFileList([...fileList, ...newFileList])
isTruncated.current = newIsTruncated
setIsLoading(false)
},
@@ -95,34 +97,8 @@ const OnlineDrive = ({
}, [datasourceNodeRunURL, dataSourceStore])
useEffect(() => {
const {
setFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileKeys,
currentNodeIdRef,
} = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) {
setFileList([])
setBucket('')
setPrefix([])
setKeywords('')
setSelectedFileKeys([])
currentNodeIdRef.current = nodeId
getOnlineDriveFiles({
prefix: [],
bucket: '',
fileList: [],
startAfter: '',
})
}
else {
// Avoid fetching files when come back from next step
if (fileList.length > 0) return
getOnlineDriveFiles({})
}
}, [nodeId])
getOnlineDriveFiles()
}, [startAfter, prefix, bucket, currentCredentialId])
const onlineDriveFileList = useMemo(() => {
if (keywords)
@@ -163,7 +139,6 @@ const OnlineDrive = ({
setFileList([])
if (file.type === OnlineDriveFileType.bucket) {
setBucket(file.displayName)
getOnlineDriveFiles({ bucket: file.displayName, fileList: [] })
}
else {
setSelectedFileKeys([])
@@ -172,7 +147,6 @@ const OnlineDrive = ({
draft.push(displayName)
})
setPrefix(newPrefix)
getOnlineDriveFiles({ prefix: newPrefix, fileList: [] })
}
}, [dataSourceStore, getOnlineDriveFiles])
@@ -185,23 +159,13 @@ const OnlineDrive = ({
return (
<div className='flex flex-col gap-y-2'>
<Header
// todo: delete mock data
docTitle='Online Drive Docs'
docLink='https://docs.dify.ai/'
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={'12345678'}
onCredentialChange={noop}
credentials={[{
avatar_url: 'https://cloud.dify.ai/logo/logo.svg',
credential: {
credentials: '......',
},
id: '12345678',
is_default: true,
name: 'test123',
type: CredentialTypeEnum.API_KEY,
}]}
currentCredentialId={currentCredentialId}
onCredentialChange={onCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<FileList
fileList={onlineDriveFileList}
@@ -216,7 +180,6 @@ const OnlineDrive = ({
handleOpenFolder={handleOpenFolder}
isInPipeline={isInPipeline}
isLoading={isLoading}
getOnlineDriveFiles={getOnlineDriveFiles}
/>
</div>
)

6
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts
View File

@@ -6,8 +6,6 @@ export type OnlineDocumentSliceShape = {
setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
searchValue: string
setSearchValue: (searchValue: string) => void
currentWorkspaceId: string
setCurrentWorkspaceId: (workspaceId: string) => void
onlineDocuments: NotionPage[]
setOnlineDocuments: (documents: NotionPage[]) => void
currentDocument: NotionPage | undefined
@@ -27,10 +25,6 @@ export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> =
setSearchValue: (searchValue: string) => set(() => ({
searchValue,
})),
currentWorkspaceId: '',
setCurrentWorkspaceId: (workspaceId: string) => set(() => ({
currentWorkspaceId: workspaceId,
})),
onlineDocuments: [],
setOnlineDocuments: (documents: NotionPage[]) => {
set(() => ({

8
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
View File

@@ -12,7 +12,8 @@ export type OnlineDriveSliceShape = {
setFileList: (fileList: OnlineDriveFile[]) => void
bucket: string
setBucket: (bucket: string) => void
startAfter: React.RefObject<string>
startAfter: string
setStartAfter: (startAfter: string) => void
isTruncated: React.RefObject<boolean>
previewOnlineDriveFileRef: React.RefObject<OnlineDriveFile | undefined>
}
@@ -44,7 +45,10 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set,
setBucket: (bucket: string) => set(() => ({
bucket,
})),
startAfter: { current: '' },
startAfter: '',
setStartAfter: (startAfter: string) => set(() => ({
startAfter,
})),
isTruncated: { current: false },
previewOnlineDriveFileRef: { current: undefined },
})

17
web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/options/index.tsx
View File

@@ -11,22 +11,21 @@ import Toast from '@/app/components/base/toast'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { useConfigurations, useInitialData } from '@/app/components/rag-pipeline/hooks/use-input-fields'
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
import { CrawlStep } from '@/models/datasets'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type OptionsProps = {
variables: RAGPipelineVariables
isRunning: boolean
step: CrawlStep
runDisabled?: boolean
controlFoldOptions?: number
onSubmit: (data: Record<string, any>) => void
}
const Options = ({
variables,
isRunning,
step,
runDisabled,
controlFoldOptions,
onSubmit,
}: OptionsProps) => {
const { t } = useTranslation()
@@ -62,12 +61,18 @@ const Options = ({
const [fold, {
toggle: foldToggle,
setTrue: foldHide,
setFalse: foldShow,
}] = useBoolean(false)
useEffect(() => {
if (controlFoldOptions !== 0)
// When the step change
if (step !== CrawlStep.init)
foldHide()
}, [controlFoldOptions])
else
foldShow()
}, [step])
const isRunning = useMemo(() => step === CrawlStep.running, [step])
return (
<form

74
web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx
View File

@@ -1,5 +1,5 @@
'use client'
import React, { useCallback, useEffect, useRef, useState } from 'react'
import React, { useCallback, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResultItem } from '@/models/datasets'
import { CrawlStep } from '@/models/datasets'
@@ -24,8 +24,7 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import { useShallow } from 'zustand/react/shallow'
import { useModalContextSelector } from '@/context/modal-context'
import { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth'
import { noop } from 'lodash-es'
import { useGetDataSourceAuth } from '@/service/use-datasource'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
@@ -33,15 +32,16 @@ export type WebsiteCrawlProps = {
nodeId: string
nodeData: DataSourceNodeType
isInPipeline?: boolean
onCredentialChange: (credentialId: string) => void
}
const WebsiteCrawl = ({
nodeId,
nodeData,
isInPipeline = false,
onCredentialChange,
}: WebsiteCrawlProps) => {
const { t } = useTranslation()
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
const [totalNum, setTotalNum] = useState(0)
const [crawledNum, setCrawledNum] = useState(0)
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
@@ -52,12 +52,20 @@ const WebsiteCrawl = ({
step,
checkedCrawlResult,
previewIndex,
currentCredentialId,
} = useDataSourceStoreWithSelector(useShallow(state => ({
crawlResult: state.crawlResult,
step: state.step,
checkedCrawlResult: state.websitePages,
previewIndex: state.previewIndex,
currentCredentialId: state.currentCredentialId,
})))
const { data: dataSourceAuth } = useGetDataSourceAuth({
pluginId: nodeData.plugin_id,
provider: nodeData.provider_name,
})
const dataSourceStore = useDataSourceStore()
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
@@ -66,33 +74,6 @@ const WebsiteCrawl = ({
node_id: nodeId,
}, !!pipelineId && !!nodeId)
useEffect(() => {
if (step !== CrawlStep.init)
setControlFoldOptions(Date.now())
}, [step])
useEffect(() => {
const {
setStep,
setCrawlResult,
setWebsitePages,
setPreviewIndex,
setCurrentWebsite,
currentNodeIdRef,
} = dataSourceStore.getState()
if (nodeId !== currentNodeIdRef.current) {
setStep(CrawlStep.init)
setCrawlResult(undefined)
setCurrentWebsite(undefined)
setWebsitePages([])
setPreviewIndex(-1)
setCrawledNum(0)
setTotalNum(0)
setCrawlErrorMessage('')
currentNodeIdRef.current = nodeId
}
}, [nodeId])
const isInit = step === CrawlStep.init
const isCrawlFinished = step === CrawlStep.finished
const isRunning = step === CrawlStep.running
@@ -113,7 +94,7 @@ const WebsiteCrawl = ({
}, [dataSourceStore])
const handleRun = useCallback(async (value: Record<string, any>) => {
const { setStep, setCrawlResult } = dataSourceStore.getState()
const { setStep, setCrawlResult, currentCredentialId } = dataSourceStore.getState()
setStep(CrawlStep.running)
ssePost(
@@ -122,6 +103,7 @@ const WebsiteCrawl = ({
body: {
inputs: value,
datasource_type: DatasourceType.websiteCrawl,
credential_id: currentCredentialId,
response_mode: 'streaming',
},
},
@@ -165,33 +147,29 @@ const WebsiteCrawl = ({
})
}, [setShowAccountSettingModal])
const handleCredentialChange = useCallback((credentialId: string) => {
setCrawledNum(0)
setTotalNum(0)
setCrawlErrorMessage('')
onCredentialChange(credentialId)
}, [dataSourceStore, onCredentialChange])
return (
<div className='flex flex-col'>
<Header
// todo: delete mock data
docTitle='How to use?'
docLink='https://docs.dify.ai'
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={'12345678'}
onCredentialChange={noop}
credentials={[{
avatar_url: 'https://cloud.dify.ai/logo/logo.svg',
credential: {
credentials: '......',
},
id: '12345678',
is_default: true,
name: 'test123',
type: CredentialTypeEnum.API_KEY,
}]}
currentCredentialId={currentCredentialId}
onCredentialChange={handleCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
<Options
variables={paramsConfig?.variables || []}
isRunning={isRunning}
runDisabled={isFetchingParams}
controlFoldOptions={controlFoldOptions}
step={step}
runDisabled={!currentCredentialId || isFetchingParams}
onSubmit={handleSubmit}
/>
</div>

6
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@@ -87,21 +87,19 @@ export const useLocalFile = () => {
}
}
export const useOnlineDocuments = () => {
export const useOnlineDocument = () => {
const {
documentsData,
currentWorkspaceId,
onlineDocuments,
currentDocument,
} = useDataSourceStoreWithSelector(useShallow(state => ({
documentsData: state.documentsData,
currentWorkspaceId: state.currentWorkspaceId,
onlineDocuments: state.onlineDocuments,
currentDocument: state.currentDocument,
})))
const dataSourceStore = useDataSourceStore()
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
const currentWorkspace = documentsData[0]
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {

11
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@@ -24,10 +24,15 @@ import WebsitePreview from './preview/web-preview'
import ProcessDocuments from './process-documents'
import ChunkPreview from './preview/chunk-preview'
import Processing from './processing'
import type { InitialDocumentDetail, OnlineDriveFile, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
import type {
InitialDocumentDetail,
OnlineDriveFile,
PublishedPipelineRunPreviewResponse,
PublishedPipelineRunResponse,
} from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { TransferMethod } from '@/types/app'
import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
import { useAddDocumentsSteps, useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './hooks'
import DataSourceProvider from './data-source/store/provider'
import { useDataSourceStore } from './data-source/store'
import { useFileUploadConfig } from '@/service/use-common'
@@ -67,7 +72,7 @@ const CreateFormPipeline = () => {
currentDocument,
PagesMapAndSelectedPagesId,
hidePreviewOnlineDocument,
} = useOnlineDocuments()
} = useOnlineDocument()
const {
websitePages,
currentWebsite,

3
web/app/components/datasets/list/datasets.tsx
View File

@@ -34,7 +34,7 @@ const Datasets = ({
})
const resetDatasetList = useResetDatasetList()
const anchorRef = useRef<HTMLDivElement>(null)
const observerRef = useRef<IntersectionObserver>()
const observerRef = useRef<IntersectionObserver>(null)
useEffect(() => {
document.title = `${t('dataset.knowledge')} - Dify`
@@ -51,7 +51,6 @@ const Datasets = ({
observerRef.current.observe(anchorRef.current)
}
return () => observerRef.current?.disconnect()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [anchorRef])
return (

71
web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
View File

@@ -5,6 +5,8 @@ import { useNodes } from 'reactflow'
import { BlockEnum } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { useCallback, useMemo, useState } from 'react'
import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { CrawlStep } from '@/models/datasets'
export const useTestRunSteps = () => {
const { t } = useTranslation()
@@ -56,3 +58,72 @@ export const useDatasourceOptions = () => {
return options
}
export const useOnlineDocument = () => {
const dataSourceStore = useDataSourceStore()
const clearOnlineDocumentData = useCallback(() => {
const {
setDocumentsData,
setSearchValue,
setSelectedPagesId,
setOnlineDocuments,
setCurrentDocument,
} = dataSourceStore.getState()
setDocumentsData([])
setSearchValue('')
setSelectedPagesId(new Set())
setOnlineDocuments([])
setCurrentDocument(undefined)
}, [dataSourceStore])
return {
clearOnlineDocumentData,
}
}
export const useWebsiteCrawl = () => {
const dataSourceStore = useDataSourceStore()
const clearWebsiteCrawlData = useCallback(() => {
const {
setStep,
setCrawlResult,
setWebsitePages,
setPreviewIndex,
setCurrentWebsite,
} = dataSourceStore.getState()
setStep(CrawlStep.init)
setCrawlResult(undefined)
setCurrentWebsite(undefined)
setWebsitePages([])
setPreviewIndex(-1)
}, [dataSourceStore])
return {
clearWebsiteCrawlData,
}
}
export const useOnlineDrive = () => {
const dataSourceStore = useDataSourceStore()
const clearOnlineDriveData = useCallback(() => {
const {
setFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileKeys,
} = dataSourceStore.getState()
setFileList([])
setBucket('')
setPrefix([])
setKeywords('')
setSelectedFileKeys([])
}, [dataSourceStore])
return {
clearOnlineDriveData,
}
}

53
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@@ -1,6 +1,11 @@
import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
import { useCallback, useMemo, useState } from 'react'
import { useTestRunSteps } from './hooks'
import {
useOnlineDocument,
useOnlineDrive,
useTestRunSteps,
useWebsiteCrawl,
} from './hooks'
import DataSourceOptions from './data-source-options'
import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
@@ -42,6 +47,10 @@ const TestRunPanel = () => {
handleBackStep,
} = useTestRunSteps()
const { clearOnlineDocumentData } = useOnlineDocument()
const { clearWebsiteCrawlData } = useWebsiteCrawl()
const { clearOnlineDriveData } = useOnlineDrive()
const datasourceType = datasource?.nodeData.provider_type
const nextBtnDisabled = useMemo(() => {
@@ -67,6 +76,7 @@ const TestRunPanel = () => {
if (!datasource)
return
const datasourceInfoList: Record<string, any>[] = []
const credentialId = dataSourceStore.getState().currentCredentialId
if (datasourceType === DatasourceType.localFile) {
const { id, name, type, size, extension, mime_type } = fileList[0].file
const documentInfo = {
@@ -86,16 +96,22 @@ const TestRunPanel = () => {
const documentInfo = {
workspace_id,
page: rest,
credential_id: credentialId,
}
datasourceInfoList.push(documentInfo)
}
if (datasourceType === DatasourceType.websiteCrawl)
datasourceInfoList.push(websitePages[0])
if (datasourceType === DatasourceType.websiteCrawl) {
datasourceInfoList.push({
...websitePages[0],
credential_id: credentialId,
})
}
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket } = dataSourceStore.getState()
datasourceInfoList.push({
bucket,
key: selectedFileKeys[0],
credential_id: credentialId,
})
}
handleRun({
@@ -106,6 +122,32 @@ const TestRunPanel = () => {
})
}, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileKeys, websitePages])
const clearDataSourceData = useCallback((dataSource: Datasource) => {
if (dataSource.nodeData.provider_type === DatasourceType.onlineDocument)
clearOnlineDocumentData()
else if (dataSource.nodeData.provider_type === DatasourceType.websiteCrawl)
clearWebsiteCrawlData()
else if (dataSource.nodeData.provider_type === DatasourceType.onlineDrive)
clearOnlineDriveData()
}, [])
const handleSwitchDataSource = useCallback((dataSource: Datasource) => {
const {
setCurrentCredentialId,
currentNodeIdRef,
} = dataSourceStore.getState()
clearDataSourceData(dataSource)
setCurrentCredentialId('')
currentNodeIdRef.current = dataSource.nodeId
setDatasource(dataSource)
}, [dataSourceStore])
const handleCredentialChange = useCallback((credentialId: string) => {
const { setCurrentCredentialId } = dataSourceStore.getState()
clearDataSourceData(datasource!)
setCurrentCredentialId(credentialId)
}, [dataSourceStore, datasource])
return (
<div
className='relative flex h-full w-[480px] flex-col rounded-l-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl shadow-shadow-shadow-1'
@@ -119,7 +161,7 @@ const TestRunPanel = () => {
<div className='flex flex-col gap-y-4 px-4 py-2'>
<DataSourceOptions
dataSourceNodeId={datasource?.nodeId || ''}
onSelect={setDatasource}
onSelect={handleSwitchDataSource}
/>
{datasourceType === DatasourceType.localFile && (
<LocalFile
@@ -132,6 +174,7 @@ const TestRunPanel = () => {
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
/>
)}
{datasourceType === DatasourceType.websiteCrawl && (
@@ -139,6 +182,7 @@ const TestRunPanel = () => {
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
/>
)}
{datasourceType === DatasourceType.onlineDrive && (
@@ -146,6 +190,7 @@ const TestRunPanel = () => {
nodeId={datasource!.nodeId}
nodeData={datasource!.nodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
/>
)}
</div>

1
web/app/components/workflow/hooks/use-shortcuts.ts
View File

@@ -218,7 +218,6 @@ export const useShortcuts = (): void => {
useKeyPress(
'shift',
(e) => {
console.log('Shift down', e)
if (shouldHandleShortcut(e))
dimOtherNodes()
},