Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

2026-01-06 06:26:00 +00:00 · 2025-07-08 14:05:45 +08:00
parent db886ae3fb 467cd2c4c1
commit 6332fe795e
15 changed files with 313 additions and 323 deletions
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/index.tsx
@@ -0,0 +1,61 @@
+import React, { useCallback, useState } from 'react'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+import { RiMoreFill } from '@remixicon/react'
+import cn from '@/utils/classnames'
+import Menu from './menu'
+
+type DropdownProps = {
+  startIndex: number
+  breadcrumbs: string[]
+  onBreadcrumbClick: (index: number) => void
+}
+
+const Dropdown = ({
+  startIndex,
+  breadcrumbs,
+  onBreadcrumbClick,
+}: DropdownProps) => {
+  const [open, setOpen] = useState(false)
+
+  const handleTrigger = useCallback(() => {
+    setOpen(prev => !prev)
+  }, [])
+
+  return (
+    <PortalToFollowElem
+      open={open}
+      onOpenChange={setOpen}
+      placement='bottom-start'
+      offset={{
+        mainAxis: 4,
+        crossAxis: -13,
+      }}
+    >
+      <PortalToFollowElemTrigger onClick={handleTrigger}>
+        <button
+          type='button'
+          className={cn(
+            'rounded-md p-1',
+            open ? 'bg-state-base-hover' : 'hover:bg-state-base-hover',
+          )}
+        >
+          <RiMoreFill className='size-4 text-text-tertiary' />
+        </button>
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent className='z-[11]'>
+        <Menu
+          breadcrumbs={breadcrumbs}
+          startIndex={startIndex}
+          onBreadcrumbClick={onBreadcrumbClick}
+        />
+      </PortalToFollowElemContent>
+      <span className='system-xs-regular text-divider-deep'>/</span>
+    </PortalToFollowElem>
+  )
+}
+
+export default React.memo(Dropdown)
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/item.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/item.tsx
@@ -0,0 +1,28 @@
+import React, { useCallback } from 'react'
+
+type ItemProps = {
+  name: string
+  index: number
+  onBreadcrumbClick: (index: number) => void
+}
+
+const Item = ({
+  name,
+  index,
+  onBreadcrumbClick,
+}: ItemProps) => {
+  const handleClick = useCallback(() => {
+    onBreadcrumbClick(index)
+  }, [index, onBreadcrumbClick])
+
+  return (
+    <div
+      className='system-md-regular rounded-lg px-3 py-1.5 text-text-secondary hover:bg-state-base-hover'
+      onClick={handleClick}
+    >
+      {name}
+    </div>
+  )
+}
+
+export default React.memo(Item)
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx
@@ -0,0 +1,30 @@
+import React from 'react'
+import Item from './item'
+
+type MenuProps = {
+  breadcrumbs: string[]
+  startIndex: number
+  onBreadcrumbClick: (index: number) => void
+}
+
+const Menu = ({
+  breadcrumbs,
+  startIndex,
+  onBreadcrumbClick,
+}: MenuProps) => {
+  return (
+    <div className='flex w-[136px] flex-col rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]'>
+      {breadcrumbs.map((breadcrumb, index) => {
+        return (
+          <Item
+            name={breadcrumb}
+            index={startIndex + index}
+            onBreadcrumbClick={onBreadcrumbClick}
+          />
+        )
+      })}
+    </div>
+  )
+}
+
+export default React.memo(Menu)
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
@@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next'
 import { useDataSourceStore } from '../../../../store'
 import Bucket from './bucket'
 import BreadcrumbItem from './item'
+import Dropdown from './dropdown'

 type BreadcrumbsProps = {
  prefix: string[]
@@ -20,7 +21,7 @@ const Breadcrumbs = ({
  isInPipeline,
 }: BreadcrumbsProps) => {
  const { t } = useTranslation()
-  const { setFileList, setSelectedFileList, setPrefix, setBucket } = useDataSourceStore().getState()
+  const dataSourceStore = useDataSourceStore()
  const showSearchResult = !!keywords && searchResultsLength > 0
  const isRoot = prefix.length === 0 && bucket === ''

@@ -42,24 +43,27 @@ const Breadcrumbs = ({
  }, [displayBreadcrumbNum, prefix])

  const handleBackToBucketList = useCallback(() => {
+    const { setFileList, setSelectedFileList, setPrefix, setBucket } = dataSourceStore.getState()
    setFileList([])
    setSelectedFileList([])
    setBucket('')
    setPrefix([])
-  }, [setBucket, setFileList, setPrefix, setSelectedFileList])
+  }, [dataSourceStore])

  const handleClickBucketName = useCallback(() => {
+    const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState()
    setFileList([])
    setSelectedFileList([])
    setPrefix([])
-  }, [setFileList, setPrefix, setSelectedFileList])
+  }, [dataSourceStore])

  const handleClickBreadcrumb = useCallback((index: number) => {
-    const newPrefix = breadcrumbs.prefixBreadcrumbs.slice(0, index - 1)
+    const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState()
+    const newPrefix = prefix.slice(0, index - 1)
    setFileList([])
    setSelectedFileList([])
    setPrefix(newPrefix)
-  }, [breadcrumbs.prefixBreadcrumbs, setFileList, setPrefix, setSelectedFileList])
+  }, [dataSourceStore, prefix])

  return (
    <div className='flex grow items-center py-1'>
@@ -106,6 +110,33 @@ const Breadcrumbs = ({
              })}
            </>
          )}
+          {breadcrumbs.needCollapsed && (
+            <>
+              {breadcrumbs.prefixBreadcrumbs.map((breadcrumb, index) => {
+                return (
+                  <BreadcrumbItem
+                    key={`${breadcrumb}-${index}`}
+                    index={index}
+                    handleClick={handleClickBreadcrumb}
+                    name={breadcrumb}
+                  />
+                )
+              })}
+              <Dropdown
+                startIndex={breadcrumbs.prefixBreadcrumbs.length}
+                breadcrumbs={breadcrumbs.collapsedBreadcrumbs}
+                onBreadcrumbClick={handleClickBreadcrumb}
+              />
+              <BreadcrumbItem
+                index={prefix.length - 1}
+                handleClick={handleClickBreadcrumb}
+                name={breadcrumbs.lastBreadcrumb}
+                isActive={true}
+                disabled={true}
+                showSeparator={false}
+              />
+            </>
+          )}
        </div>
      )}
    </div>
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
@@ -17,6 +17,7 @@ type FileListProps = {
  handleSelectFile: (file: OnlineDriveFile) => void
  handleOpenFolder: (file: OnlineDriveFile) => void
  isLoading: boolean
+  isTruncated: boolean
 }

 const FileList = ({
@@ -32,6 +33,7 @@ const FileList = ({
  handleOpenFolder,
  isInPipeline,
  isLoading,
+  isTruncated,
 }: FileListProps) => {
  const [inputValue, setInputValue] = useState(keywords)

@@ -74,6 +76,7 @@ const FileList = ({
        handleSelectFile={handleSelectFile}
        isInPipeline={isInPipeline}
        isLoading={isLoading}
+        isTruncated={isTruncated}
      />
    </div>
  )
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
@@ -1,15 +1,21 @@
+import React, { useEffect, useMemo, useRef } from 'react'
 import type { OnlineDriveFile } from '@/models/pipeline'
 import Item from './item'
 import EmptyFolder from './empty-folder'
 import EmptySearchResult from './empty-search-result'
 import Loading from '@/app/components/base/loading'
 import { RiLoader2Line } from '@remixicon/react'
+import { useFileSupportTypes } from '@/service/use-common'
+import { isFile } from '../../utils'
+import { getFileExtension } from './utils'
+import { useDataSourceStore } from '../../../store'

 type FileListProps = {
  fileList: OnlineDriveFile[]
  selectedFileList: string[]
  keywords: string
  isInPipeline: boolean
+  isTruncated: boolean
  isLoading: boolean
  handleResetKeywords: () => void
  handleSelectFile: (file: OnlineDriveFile) => void
@@ -25,11 +31,35 @@ const List = ({
  handleOpenFolder,
  isInPipeline,
  isLoading,
+  isTruncated,
 }: FileListProps) => {
+  const anchorRef = useRef<HTMLDivElement>(null)
+  const observerRef = useRef<IntersectionObserver>()
+  const dataSourceStore = useDataSourceStore()
+
+  useEffect(() => {
+    if (anchorRef.current) {
+      observerRef.current = new IntersectionObserver((entries) => {
+        const { setStartAfter } = dataSourceStore.getState()
+        if (entries[0].isIntersecting && isTruncated && !isLoading)
+          setStartAfter(fileList[fileList.length - 1].key)
+      }, {
+        rootMargin: '100px',
+      })
+      observerRef.current.observe(anchorRef.current)
+    }
+    return () => observerRef.current?.disconnect()
+  }, [anchorRef, dataSourceStore, isTruncated, isLoading, fileList])
+
  const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0
  const isPartLoading = isLoading && fileList.length > 0
  const isEmptyFolder = !isLoading && fileList.length === 0 && keywords.length === 0
  const isSearchResultEmpty = !isLoading && fileList.length === 0 && keywords.length > 0
+  const { data: supportFileTypesRes } = useFileSupportTypes()
+  const supportedFileTypes = useMemo(() => {
+    if (!supportFileTypesRes) return []
+    return Array.from(new Set(supportFileTypesRes.allowed_extensions.map(item => item.toLowerCase())))
+  }, [supportFileTypesRes])

  return (
    <div className='grow overflow-hidden p-1 pt-0'>
@@ -53,11 +83,14 @@ const List = ({
          {
            fileList.map((file) => {
              const isSelected = selectedFileList.includes(file.key)
+              const extension = getFileExtension(file.key)
+              const disabled = isFile(file.key) && !supportedFileTypes.includes(extension)
              return (
                <Item
                  key={file.key}
                  file={file}
                  isSelected={isSelected}
+                  disabled={disabled}
                  onSelect={handleSelectFile}
                  onOpen={handleOpenFolder}
                  isMultipleChoice={!isInPipeline}
@@ -72,10 +105,11 @@ const List = ({
              </div>
            )
          }
+          <div ref={anchorRef} className='h-0' />
        </div>
      )}
    </div>
  )
 }

-export default List
+export default React.memo(List)
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx
@@ -51,15 +51,26 @@ const Item = ({
      onClick={handleClickItem}
    >
      {!isBucket && isMultipleChoice && (
-        <Checkbox className='shrink-0' id={file.key} checked={isSelected} onCheck={handleSelect} />
+        <Checkbox
+          className='shrink-0'
+          disabled={disabled}
+          id={file.key}
+          checked={isSelected}
+          onCheck={handleSelect}
+        />
      )}
      {!isBucket && !isMultipleChoice && (
-        <Radio className='shrink-0' isChecked={isSelected} onCheck={handleSelect} />
+        <Radio
+          className='shrink-0'
+          disabled={disabled}
+          isChecked={isSelected}
+          onCheck={handleSelect}
+        />
      )}
      <Wrapper
        popupContent={t('datasetPipeline.onlineDrive.notSupportedFileType')}
        position='top-end'
-        offset={{ mainAxis: 4, crossAxis: 104 }}
+        offset={{ mainAxis: 4, crossAxis: -104 }}
      >
        <div
          className={cn(
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
@@ -8,8 +8,8 @@ import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { ssePost } from '@/service/base'
 import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
 import Toast from '@/app/components/base/toast'
-import { useDataSourceStoreWithSelector } from '../store'
-import { convertOnlineDriveDataToFileList } from './utils'
+import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
+import { convertOnlineDriveData } from './utils'
 import produce from 'immer'

 type OnlineDriveProps = {
@@ -25,17 +25,13 @@ const OnlineDrive = ({
 }: OnlineDriveProps) => {
  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
  const prefix = useDataSourceStoreWithSelector(state => state.prefix)
-  const setPrefix = useDataSourceStoreWithSelector(state => state.setPrefix)
  const keywords = useDataSourceStoreWithSelector(state => state.keywords)
-  const setKeywords = useDataSourceStoreWithSelector(state => state.setKeywords)
  const bucket = useDataSourceStoreWithSelector(state => state.bucket)
-  const setBucket = useDataSourceStoreWithSelector(state => state.setBucket)
  const startAfter = useDataSourceStoreWithSelector(state => state.startAfter)
-  const setStartAfter = useDataSourceStoreWithSelector(state => state.setStartAfter)
  const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
-  const setSelectedFileList = useDataSourceStoreWithSelector(state => state.setSelectedFileList)
  const fileList = useDataSourceStoreWithSelector(state => state.fileList)
-  const setFileList = useDataSourceStoreWithSelector(state => state.setFileList)
+  const isTruncated = useDataSourceStoreWithSelector(state => state.isTruncated)
+  const dataSourceStore = useDataSourceStore()
  const [isLoading, setIsLoading] = useState(false)

  const datasourceNodeRunURL = !isInPipeline
@@ -60,8 +56,10 @@ const OnlineDrive = ({
      },
      {
        onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
-          const newFileList = convertOnlineDriveDataToFileList(documentsData.data)
+          const { setFileList, setIsTruncated } = dataSourceStore.getState()
+          const { fileList: newFileList, isTruncated } = convertOnlineDriveData(documentsData.data, prefix)
          setFileList([...fileList, ...newFileList])
+          setIsTruncated(isTruncated)
          setIsLoading(false)
        },
        onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
@@ -73,7 +71,7 @@ const OnlineDrive = ({
        },
      },
    )
-  }, [bucket, datasourceNodeRunURL, prefix, fileList, setFileList, startAfter])
+  }, [prefix, datasourceNodeRunURL, bucket, startAfter, dataSourceStore, fileList])

  useEffect(() => {
    getOnlineDrive()
@@ -87,14 +85,18 @@ const OnlineDrive = ({
  }, [fileList, keywords])

  const updateKeywords = useCallback((keywords: string) => {
+    const { setKeywords } = dataSourceStore.getState()
    setKeywords(keywords)
-  }, [setKeywords])
+  }, [dataSourceStore])

  const resetPrefix = useCallback(() => {
+    const { setKeywords } = dataSourceStore.getState()
+
    setKeywords('')
-  }, [setKeywords])
+  }, [dataSourceStore])

  const handleSelectFile = useCallback((file: OnlineDriveFile) => {
+    const { setSelectedFileList } = dataSourceStore.getState()
    if (file.type === OnlineDriveFileType.bucket) return
    const newSelectedFileList = produce(selectedFileList, (draft) => {
      if (draft.includes(file.key)) {
@@ -107,9 +109,10 @@ const OnlineDrive = ({
      }
    })
    setSelectedFileList(newSelectedFileList)
-  }, [isInPipeline, selectedFileList, setSelectedFileList])
+  }, [dataSourceStore, isInPipeline, selectedFileList])

  const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
+    const { setPrefix, setBucket, setFileList } = dataSourceStore.getState()
    if (file.type === OnlineDriveFileType.file) return
    setFileList([])
    if (file.type === OnlineDriveFileType.bucket) {
@@ -122,7 +125,7 @@ const OnlineDrive = ({
      })
      setPrefix(newPrefix)
    }
-  }, [prefix, setBucket, setFileList, setPrefix])
+  }, [dataSourceStore, prefix])

  return (
    <div className='flex flex-col gap-y-2'>
@@ -143,6 +146,7 @@ const OnlineDrive = ({
        handleOpenFolder={handleOpenFolder}
        isInPipeline={isInPipeline}
        isLoading={isLoading}
+        isTruncated={isTruncated}
      />
    </div>
  )
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
@@ -7,14 +7,19 @@ export const isFile = (path: string): boolean => {
  return filePathRegex.test(path)
 }

-export const hasBuckets = (data: OnlineDriveData[]): boolean => {
+export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[]): boolean => {
+  if (prefix.length > 0) return false
  return data.length > 1 || (data.length === 1 && data[0].files.length === 0)
 }

-export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): OnlineDriveFile[] => {
+export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[]): { fileList: OnlineDriveFile[], isTruncated: boolean } => {
  const fileList: OnlineDriveFile[] = []
+  let isTruncated = false

-  if (hasBuckets(data)) {
+  if (data.length === 0)
+    return { fileList, isTruncated }
+
+  if (isBucketListInitiation(data, prefix)) {
    data.forEach((item) => {
      fileList.push({
        key: item.bucket,
@@ -34,6 +39,7 @@ export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): Onlin
        type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
      })
    })
+    isTruncated = data[0].is_truncated ?? false
  }
-  return fileList
+  return { fileList, isTruncated }
 }
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
@@ -14,6 +14,8 @@ export type OnlineDriveSliceShape = {
  setFileList: (fileList: OnlineDriveFile[]) => void
  bucket: string
  setBucket: (bucket: string) => void
+  isTruncated: boolean
+  setIsTruncated: (isTruncated: boolean) => void
 }

 export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => {
@@ -42,5 +44,9 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
    setBucket: (bucket: string) => set(() => ({
      bucket,
    })),
+    isTruncated: false,
+    setIsTruncated: (isTruncated: boolean) => set(() => ({
+      isTruncated,
+    })),
  })
 }
--- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
@@ -1,12 +1,10 @@
 import { useTranslation } from 'react-i18next'
 import { AddDocumentsStep } from './types'
 import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
-import { useCallback, useMemo, useRef, useState } from 'react'
+import { useCallback, useMemo, useState } from 'react'
 import { BlockEnum, type Node } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
-import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
-import { CrawlStep } from '@/models/datasets'
-import { useDataSourceStoreWithSelector } from './data-source/store'
+import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store'

 export const useAddDocumentsSteps = () => {
  const { t } = useTranslation()
@@ -66,13 +64,14 @@ export const useLocalFile = () => {
  const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
  const previewFileRef = useDataSourceStoreWithSelector(state => state.previewLocalFileRef)
  const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile)
-  const setCurrentFile = useDataSourceStoreWithSelector(state => state.setCurrentLocalFile)
+  const dataSourceStore = useDataSourceStore()

  const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])

  const hidePreviewLocalFile = useCallback(() => {
-    setCurrentFile(undefined)
-  }, [setCurrentFile])
+    const { setCurrentLocalFile } = dataSourceStore.getState()
+    setCurrentLocalFile(undefined)
+  }, [dataSourceStore])

  return {
    fileList,
@@ -86,12 +85,13 @@ export const useLocalFile = () => {
 export const useOnlineDocuments = () => {
  const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
  const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef)
-  const setCurrentDocument = useDataSourceStoreWithSelector(state => state.setCurrentDocument)
  const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
+  const dataSourceStore = useDataSourceStore()

  const hidePreviewOnlineDocument = useCallback(() => {
+    const { setCurrentDocument } = dataSourceStore.getState()
    setCurrentDocument(undefined)
-  }, [setCurrentDocument])
+  }, [dataSourceStore])

  return {
    onlineDocuments,
@@ -102,44 +102,31 @@ export const useOnlineDocuments = () => {
 }

 export const useWebsiteCrawl = () => {
-  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
-  const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
-  const [crawlResult, setCrawlResult] = useState<CrawlResult | undefined>()
-  const [step, setStep] = useState<CrawlStep>(CrawlStep.init)
-  const [previewIndex, setPreviewIndex] = useState<number>(-1)
-
-  const previewWebsitePage = useRef<CrawlResultItem>(websitePages[0])
-
-  const updateCurrentWebsite = useCallback((website: CrawlResultItem, index: number) => {
-    setCurrentWebsite(website)
-    setPreviewIndex(index)
-  }, [])
+  const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
+  const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite)
+  const previewWebsitePageRef = useDataSourceStoreWithSelector(state => state.previewWebsitePageRef)
+  const dataSourceStore = useDataSourceStore()

  const hideWebsitePreview = useCallback(() => {
+    const { setCurrentWebsite, setPreviewIndex } = dataSourceStore.getState()
    setCurrentWebsite(undefined)
    setPreviewIndex(-1)
-  }, [])
-
-  const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
-    setWebsitePages(checkedCrawlResult)
-    previewWebsitePage.current = checkedCrawlResult[0]
-  }, [])
+  }, [dataSourceStore])

  return {
    websitePages,
-    crawlResult,
-    setCrawlResult,
-    step,
-    setStep,
-    previewWebsitePage,
-    updataCheckedCrawlResultChange,
+    previewWebsitePageRef,
    currentWebsite,
-    updateCurrentWebsite,
-    previewIndex,
    hideWebsitePreview,
  }
 }

 export const useOnlineDrive = () => {
-  return {}
+  const fileList = useDataSourceStoreWithSelector(state => state.fileList)
+  const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
+
+  return {
+    fileList,
+    selectedFileList,
+  }
 }
--- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx
@@ -27,8 +27,9 @@ import Processing from './processing'
 import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
-import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
+import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
 import DataSourceProvider from './data-source/store/provider'
+import { useDataSourceStore } from './data-source/store'

 const CreateFormPipeline = () => {
  const { t } = useTranslation()
@@ -39,6 +40,7 @@ const CreateFormPipeline = () => {
  const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
  const [batchId, setBatchId] = useState('')
  const [documents, setDocuments] = useState<InitialDocumentDetail[]>([])
+  const dataSourceStore = useDataSourceStore()

  const isPreview = useRef(false)
  const formRef = useRef<any>(null)
@@ -66,27 +68,44 @@ const CreateFormPipeline = () => {
  } = useOnlineDocuments()
  const {
    websitePages,
-    previewWebsitePage,
+    previewWebsitePageRef,
    currentWebsite,
    hideWebsitePreview,
  } = useWebsiteCrawl()
-  // const { } = useOnlineDrive()
+  const {
+    fileList: onlineDriveFileList,
+    selectedFileList,
+  } = useOnlineDrive()

-  const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
-  const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
-  const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
  const datasourceType = datasource?.nodeData.provider_type
+  const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
+  const isShowVectorSpaceFull = useMemo(() => {
+    if (!datasource)
+      return false
+    if (datasourceType === DatasourceType.localFile)
+      return allFileLoaded && isVectorSpaceFull && enableBilling
+    if (datasourceType === DatasourceType.onlineDocument)
+      return onlineDocuments.length > 0 && isVectorSpaceFull && enableBilling
+    if (datasourceType === DatasourceType.websiteCrawl)
+      return websitePages.length > 0 && isVectorSpaceFull && enableBilling
+    if (datasourceType === DatasourceType.onlineDrive)
+      return onlineDriveFileList.length > 0 && isVectorSpaceFull && enableBilling
+    return false
+  }, [allFileLoaded, datasource, datasourceType, enableBilling, isVectorSpaceFull, onlineDocuments.length, onlineDriveFileList.length, websitePages.length])
+  const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'

  const nextBtnDisabled = useMemo(() => {
    if (!datasource) return true
    if (datasourceType === DatasourceType.localFile)
-      return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
+      return isShowVectorSpaceFull || !fileList.length || !allFileLoaded
    if (datasourceType === DatasourceType.onlineDocument)
      return isShowVectorSpaceFull || !onlineDocuments.length
    if (datasourceType === DatasourceType.websiteCrawl)
      return isShowVectorSpaceFull || !websitePages.length
+    if (datasourceType === DatasourceType.onlineDrive)
+      return isShowVectorSpaceFull || !selectedFileList.length
    return false
-  }, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
+  }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileList.length])

  const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()

@@ -117,7 +136,7 @@ const CreateFormPipeline = () => {
      datasourceInfoList.push(documentInfo)
    }
    if (datasourceType === DatasourceType.websiteCrawl)
-      datasourceInfoList.push(previewWebsitePage.current)
+      datasourceInfoList.push(previewWebsitePageRef.current!)
    await runPublishedPipeline({
      pipeline_id: pipelineId!,
      inputs: data,
@@ -130,7 +149,7 @@ const CreateFormPipeline = () => {
        setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
      },
    })
-  }, [datasource, datasourceType, pipelineId, previewFileRef, previewOnlineDocumentRef, previewWebsitePage, runPublishedPipeline])
+  }, [datasource, datasourceType, previewWebsitePageRef, runPublishedPipeline, pipelineId, previewFileRef, previewOnlineDocumentRef])

  const handleProcess = useCallback(async (data: Record<string, any>) => {
    if (!datasource)
@@ -167,6 +186,17 @@ const CreateFormPipeline = () => {
        datasourceInfoList.push(websitePage)
      })
    }
+    if (datasourceType === DatasourceType.onlineDrive) {
+      if (datasourceType === DatasourceType.onlineDrive) {
+        const { bucket } = dataSourceStore.getState()
+        selectedFileList.forEach((key) => {
+          datasourceInfoList.push({
+            bucket,
+            key,
+          })
+        })
+      }
+    }
    await runPublishedPipeline({
      pipeline_id: pipelineId!,
      inputs: data,
@@ -181,7 +211,7 @@ const CreateFormPipeline = () => {
        handleNextStep()
      },
    })
-  }, [datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
+  }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileList, websitePages])

  const onClickProcess = useCallback(() => {
    isPreview.current = false
@@ -208,9 +238,9 @@ const CreateFormPipeline = () => {
  }, [onClickPreview, previewOnlineDocumentRef])

  const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
-    previewWebsitePage.current = website
+    previewWebsitePageRef.current = website
    onClickPreview()
-  }, [onClickPreview, previewWebsitePage])
+  }, [onClickPreview, previewWebsitePageRef])

  if (isFetchingPipelineInfo) {
    return (
--- a/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
+++ b/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
@@ -1,240 +0,0 @@
-'use client'
-
-import { useContext } from 'use-context-selector'
-import { useRouter } from 'next/navigation'
-import { useCallback, useEffect, useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiMoreFill } from '@remixicon/react'
-import cn from '@/utils/classnames'
-import Confirm from '@/app/components/base/confirm'
-import { ToastContext } from '@/app/components/base/toast'
-import { checkIsUsedInApp, deleteDataset } from '@/service/datasets'
-import type { DataSet } from '@/models/datasets'
-import Tooltip from '@/app/components/base/tooltip'
-import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
-import type { HtmlContentProps } from '@/app/components/base/popover'
-import CustomPopover from '@/app/components/base/popover'
-import Divider from '@/app/components/base/divider'
-import RenameDatasetModal from '@/app/components/datasets/rename-modal'
-import type { Tag } from '@/app/components/base/tag-management/constant'
-import TagSelector from '@/app/components/base/tag-management/selector'
-import CornerLabel from '@/app/components/base/corner-label'
-import { useAppContext } from '@/context/app-context'
-
-export type DatasetCardProps = {
-  dataset: DataSet
-  onSuccess?: () => void
-}
-
-const DatasetCard = ({
-  dataset,
-  onSuccess,
-}: DatasetCardProps) => {
-  const { t } = useTranslation()
-  const { notify } = useContext(ToastContext)
-  const { push } = useRouter()
-  const EXTERNAL_PROVIDER = 'external' as const
-
-  const { isCurrentWorkspaceDatasetOperator } = useAppContext()
-  const [tags, setTags] = useState<Tag[]>(dataset.tags)
-
-  const [showRenameModal, setShowRenameModal] = useState(false)
-  const [showConfirmDelete, setShowConfirmDelete] = useState(false)
-  const [confirmMessage, setConfirmMessage] = useState<string>('')
-  const isExternalProvider = (provider: string): boolean => provider === EXTERNAL_PROVIDER
-  const detectIsUsedByApp = useCallback(async () => {
-    try {
-      const { is_using: isUsedByApp } = await checkIsUsedInApp(dataset.id)
-      setConfirmMessage(isUsedByApp ? t('dataset.datasetUsedByApp')! : t('dataset.deleteDatasetConfirmContent')!)
-    }
-    catch (e: any) {
-      const res = await e.json()
-      notify({ type: 'error', message: res?.message || 'Unknown error' })
-    }
-
-    setShowConfirmDelete(true)
-  }, [dataset.id, notify, t])
-  const onConfirmDelete = useCallback(async () => {
-    try {
-      await deleteDataset(dataset.id)
-      notify({ type: 'success', message: t('dataset.datasetDeleted') })
-      if (onSuccess)
-        onSuccess()
-    }
-    catch {
-    }
-    setShowConfirmDelete(false)
-  }, [dataset.id, notify, onSuccess, t])
-
-  const Operations = (props: HtmlContentProps & { showDelete: boolean }) => {
-    const onMouseLeave = async () => {
-      props.onClose?.()
-    }
-    const onClickRename = async (e: React.MouseEvent<HTMLDivElement>) => {
-      e.stopPropagation()
-      props.onClick?.()
-      e.preventDefault()
-      setShowRenameModal(true)
-    }
-    const onClickDelete = async (e: React.MouseEvent<HTMLDivElement>) => {
-      e.stopPropagation()
-      props.onClick?.()
-      e.preventDefault()
-      detectIsUsedByApp()
-    }
-    return (
-      <div className="relative w-full py-1" onMouseLeave={onMouseLeave}>
-        <div className='mx-1 flex h-8 cursor-pointer items-center gap-2 rounded-lg px-3 py-[6px] hover:bg-state-base-hover' onClick={onClickRename}>
-          <span className='text-sm text-text-secondary'>{t('common.operation.settings')}</span>
-        </div>
-        {props.showDelete && (
-          <>
-            <Divider className="!my-1" />
-            <div
-              className='group mx-1 flex h-8 cursor-pointer items-center gap-2 rounded-lg px-3 py-[6px] hover:bg-state-destructive-hover'
-              onClick={onClickDelete}
-            >
-              <span className={cn('text-sm text-text-secondary', 'group-hover:text-text-destructive')}>
-                {t('common.operation.delete')}
-              </span>
-            </div>
-          </>
-        )}
-      </div>
-    )
-  }
-
-  useEffect(() => {
-    setTags(dataset.tags)
-  }, [dataset])
-
-  return (
-    <>
-      <div
-        className='group relative col-span-1 flex min-h-[171px] cursor-pointer flex-col rounded-xl border-[0.5px] border-solid border-components-card-border bg-components-card-bg shadow-sm transition-all duration-200 ease-in-out hover:shadow-lg'
-        data-disable-nprogress={true}
-        onClick={(e) => {
-          e.preventDefault()
-          isExternalProvider(dataset.provider)
-            ? push(`/datasets/${dataset.id}/hitTesting`)
-            : push(`/datasets/${dataset.id}/documents`)
-        }}
-      >
-        {isExternalProvider(dataset.provider) && <CornerLabel label='External' className='absolute right-0' labelClassName='rounded-tr-xl' />}
-        <div className='flex h-[66px] shrink-0 grow-0 items-center gap-3 px-[14px] pb-3 pt-[14px]'>
-          <div className={cn(
-            'flex shrink-0 items-center justify-center rounded-md border-[0.5px] border-[#E0EAFF] bg-[#F5F8FF] p-2.5',
-            !dataset.embedding_available && 'opacity-50 hover:opacity-100',
-          )}>
-            <Folder className='h-5 w-5 text-[#444CE7]' />
-          </div>
-          <div className='w-0 grow py-[1px]'>
-            <div className='flex items-center text-sm font-semibold leading-5 text-text-secondary'>
-              <div className={cn('truncate', !dataset.embedding_available && 'text-text-tertiary opacity-50 hover:opacity-100')} title={dataset.name}>{dataset.name}</div>
-              {!dataset.embedding_available && (
-                <Tooltip
-                  popupContent={t('dataset.unavailableTip')}
-                >
-                  <span className='ml-1 inline-flex w-max shrink-0 rounded-md border border-divider-regular px-1 text-xs font-normal leading-[18px] text-text-tertiary'>{t('dataset.unavailable')}</span>
-                </Tooltip>
-              )}
-            </div>
-            <div className='mt-[1px] flex items-center text-xs leading-[18px] text-text-tertiary'>
-              <div
-                className={cn('truncate', (!dataset.embedding_available || !dataset.document_count) && 'opacity-50')}
-                title={dataset.provider === 'external' ? `${dataset.app_count}${t('dataset.appCount')}` : `${dataset.document_count}${t('dataset.documentCount')} · ${Math.round(dataset.word_count / 1000)}${t('dataset.wordCount')} · ${dataset.app_count}${t('dataset.appCount')}`}
-              >
-                {dataset.provider === 'external'
-                  ? <>
-                    <span>{dataset.app_count}{t('dataset.appCount')}</span>
-                  </>
-                  : <>
-                    <span>{dataset.document_count}{t('dataset.documentCount')}</span>
-                    <span className='mx-0.5 w-1 shrink-0 text-text-tertiary'>·</span>
-                    <span>{Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}</span>
-                    <span className='mx-0.5 w-1 shrink-0 text-text-tertiary'>·</span>
-                    <span>{dataset.app_count}{t('dataset.appCount')}</span>
-                  </>
-                }
-              </div>
-            </div>
-          </div>
-        </div>
-        <div
-          className={cn(
-            'mb-2 max-h-[72px] grow px-[14px] text-xs leading-normal text-text-tertiary group-hover:line-clamp-2 group-hover:max-h-[36px]',
-            tags.length ? 'line-clamp-2' : 'line-clamp-4',
-            !dataset.embedding_available && 'opacity-50 hover:opacity-100',
-          )}
-          title={dataset.description}>
-          {dataset.description}
-        </div>
-        <div className={cn(
-          'mt-4 h-[42px] shrink-0 items-center pb-[6px] pl-[14px] pr-[6px] pt-1',
-          tags.length ? 'flex' : '!hidden group-hover:!flex',
-        )}>
-          <div className={cn('flex w-0 grow items-center gap-1', !dataset.embedding_available && 'opacity-50 hover:opacity-100')} onClick={(e) => {
-            e.stopPropagation()
-            e.preventDefault()
-          }}>
-            <div className={cn(
-              'mr-[41px] w-full grow group-hover:!mr-0 group-hover:!block',
-              tags.length ? '!block' : '!hidden',
-            )}>
-              <TagSelector
-                position='bl'
-                type='knowledge'
-                targetID={dataset.id}
-                value={tags.map(tag => tag.id)}
-                selectedTags={tags}
-                onCacheUpdate={setTags}
-                onChange={onSuccess}
-              />
-            </div>
-          </div>
-          <div className='mx-1 !hidden h-[14px] w-[1px] shrink-0 bg-divider-regular group-hover:!flex' />
-          <div className='!hidden shrink-0 group-hover:!flex'>
-            <CustomPopover
-              htmlContent={<Operations showDelete={!isCurrentWorkspaceDatasetOperator} />}
-              position="br"
-              trigger="click"
-              btnElement={
-                <div
-                  className='flex h-8 w-8 cursor-pointer items-center justify-center rounded-md'
-                >
-                  <RiMoreFill className='h-4 w-4 text-text-secondary' />
-                </div>
-              }
-              btnClassName={open =>
-                cn(
-                  open ? '!bg-black/5 !shadow-none' : '!bg-transparent',
-                  'h-8 w-8 rounded-md border-none !p-2 hover:!bg-black/5',
-                )
-              }
-              className={'!z-20 h-fit !w-[128px]'}
-            />
-          </div>
-        </div>
-      </div>
-      {showRenameModal && (
-        <RenameDatasetModal
-          show={showRenameModal}
-          dataset={dataset}
-          onClose={() => setShowRenameModal(false)}
-          onSuccess={onSuccess}
-        />
-      )}
-      {showConfirmDelete && (
-        <Confirm
-          title={t('dataset.deleteDatasetConfirmTitle')}
-          content={confirmMessage}
-          isShow={showConfirmDelete}
-          onConfirm={onConfirmDelete}
-          onCancel={() => setShowConfirmDelete(false)}
-        />
-      )}
-    </>
-  )
-}
-
-export default DatasetCard
--- a/web/app/components/datasets/list/datasets.tsx
+++ b/web/app/components/datasets/list/datasets.tsx
@@ -34,19 +34,17 @@ const Datasets = ({
    keyword: keywords,
  })
  const resetDatasetList = useResetDatasetList()
-  const loadingStateRef = useRef(false)
  const anchorRef = useRef<HTMLDivElement>(null)
  const observerRef = useRef<IntersectionObserver>()

  useEffect(() => {
-    loadingStateRef.current = isFetching
    document.title = `${t('dataset.knowledge')} - Dify`
-  }, [isFetching, t])
+  }, [t])

  useEffect(() => {
    if (anchorRef.current) {
      observerRef.current = new IntersectionObserver((entries) => {
-        if (entries[0].isIntersecting && hasNextPage)
+        if (entries[0].isIntersecting && hasNextPage && !isFetching)
          fetchNextPage()
      }, {
        rootMargin: '100px',
@@ -54,7 +52,7 @@ const Datasets = ({
      observerRef.current.observe(anchorRef.current)
    }
    return () => observerRef.current?.disconnect()
-  }, [anchorRef, datasetList, hasNextPage, fetchNextPage])
+  }, [anchorRef, datasetList, hasNextPage, fetchNextPage, isFetching])

  return (
    <>
--- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
@@ -24,7 +24,7 @@ const TestRunPanel = () => {
  const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
  const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
  const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
-  const { bucket } = useDataSourceStore().getState()
+  const dataSourceStore = useDataSourceStore()
  const [datasource, setDatasource] = useState<Datasource>()

  const {
@@ -84,6 +84,7 @@ const TestRunPanel = () => {
    if (datasourceType === DatasourceType.websiteCrawl)
      datasourceInfoList.push(websitePages[0])
    if (datasourceType === DatasourceType.onlineDrive) {
+      const { bucket } = dataSourceStore.getState()
      datasourceInfoList.push({
        bucket,
        key: selectedFileList[0],
@@ -95,7 +96,7 @@ const TestRunPanel = () => {
      datasource_type: datasourceType,
      datasource_info_list: datasourceInfoList,
    })
-  }, [bucket, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages])
+  }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages])

  return (
    <div