refactor: remove isTruncated state and update related logic to use mutable refs

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx

@@ -17,7 +17,6 @@ type FileListProps = {
   handleSelectFile: (file: OnlineDriveFile) => void
   handleOpenFolder: (file: OnlineDriveFile) => void
   isLoading: boolean
-  isTruncated: boolean
   getOnlineDriveFiles: (params: {
     prefix?: string[]
     bucket?: string
@@ -39,7 +38,6 @@ const FileList = ({
   handleOpenFolder,
   isInPipeline,
   isLoading,
-  isTruncated,
   getOnlineDriveFiles,
 }: FileListProps) => {
   const [inputValue, setInputValue] = useState(keywords)
@@ -84,7 +82,6 @@ const FileList = ({
         handleSelectFile={handleSelectFile}
         isInPipeline={isInPipeline}
         isLoading={isLoading}
-        isTruncated={isTruncated}
         getOnlineDriveFiles={getOnlineDriveFiles}
       />
     </div>

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx

@@ -12,7 +12,6 @@ type FileListProps = {
   selectedFileList: string[]
   keywords: string
   isInPipeline: boolean
-  isTruncated: boolean
   isLoading: boolean
   handleResetKeywords: () => void
   handleSelectFile: (file: OnlineDriveFile) => void
@@ -34,7 +33,6 @@ const List = ({
   handleOpenFolder,
   isInPipeline,
   isLoading,
-  isTruncated,
   getOnlineDriveFiles,
 }: FileListProps) => {
   const anchorRef = useRef<HTMLDivElement>(null)
@@ -44,9 +42,9 @@ const List = ({
   useEffect(() => {
     if (anchorRef.current) {
       observerRef.current = new IntersectionObserver((entries) => {
-        const { setStartAfter } = dataSourceStore.getState()
-        if (entries[0].isIntersecting && isTruncated && !isLoading) {
-          setStartAfter(fileList[fileList.length - 1].key)
+        const { startAfter, isTruncated } = dataSourceStore.getState()
+        if (entries[0].isIntersecting && isTruncated.current && !isLoading) {
+          startAfter.current = fileList[fileList.length - 1].key
           getOnlineDriveFiles({ startAfter: fileList[fileList.length - 1].key })
         }
       }, {

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx

@@ -27,10 +27,8 @@ const OnlineDrive = ({
   const prefix = useDataSourceStoreWithSelector(state => state.prefix)
   const keywords = useDataSourceStoreWithSelector(state => state.keywords)
   const bucket = useDataSourceStoreWithSelector(state => state.bucket)
-  const startAfter = useDataSourceStoreWithSelector(state => state.startAfter)
   const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
   const fileList = useDataSourceStoreWithSelector(state => state.fileList)
-  const isTruncated = useDataSourceStoreWithSelector(state => state.isTruncated)
   const dataSourceStore = useDataSourceStore()
   const [isLoading, setIsLoading] = useState(false)
 
@@ -44,9 +42,10 @@ const OnlineDrive = ({
     startAfter?: string
     fileList?: OnlineDriveFile[]
   }) => {
+    const { startAfter } = dataSourceStore.getState()
     const _prefix = params.prefix ?? prefix
     const _bucket = params.bucket ?? bucket
-    const _startAfter = params.startAfter ?? startAfter
+    const _startAfter = params.startAfter ?? startAfter.current
     const _fileList = params.fileList ?? fileList
     const prefixString = _prefix.length > 0 ? `${_prefix.join('/')}/` : ''
     setIsLoading(true)
@@ -65,10 +64,10 @@ const OnlineDrive = ({
       },
       {
         onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
-          const { setFileList, setIsTruncated } = dataSourceStore.getState()
-          const { fileList: newFileList, isTruncated } = convertOnlineDriveData(documentsData.data, _prefix)
+          const { setFileList, isTruncated } = dataSourceStore.getState()
+          const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, _prefix, _bucket)
           setFileList([..._fileList, ...newFileList])
-          setIsTruncated(isTruncated)
+          isTruncated.current = newIsTruncated
           setIsLoading(false)
         },
         onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
@@ -80,7 +79,7 @@ const OnlineDrive = ({
         },
       },
     )
-  }, [prefix, bucket, startAfter, datasourceNodeRunURL, dataSourceStore, fileList])
+  }, [prefix, bucket, datasourceNodeRunURL, dataSourceStore, fileList])
 
   useEffect(() => {
     if (fileList.length > 0) return
@@ -99,7 +98,7 @@ const OnlineDrive = ({
     setKeywords(keywords)
   }, [dataSourceStore])
 
-  const resetPrefix = useCallback(() => {
+  const resetKeywords = useCallback(() => {
     const { setKeywords } = dataSourceStore.getState()
 
     setKeywords('')
@@ -152,14 +151,13 @@ const OnlineDrive = ({
         prefix={prefix}
         keywords={keywords}
         bucket={bucket}
-        resetKeywords={resetPrefix}
+        resetKeywords={resetKeywords}
         updateKeywords={updateKeywords}
         searchResultsLength={onlineDriveFileList.length}
         handleSelectFile={handleSelectFile}
         handleOpenFolder={handleOpenFolder}
         isInPipeline={isInPipeline}
         isLoading={isLoading}
-        isTruncated={isTruncated}
         getOnlineDriveFiles={getOnlineDriveFiles}
       />
     </div>

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts

@@ -7,19 +7,19 @@ export const isFile = (path: string): boolean => {
   return filePathRegex.test(path)
 }
 
-export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[]): boolean => {
-  if (prefix.length > 0) return false
+export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[], bucket: string): boolean => {
+  if (bucket || prefix.length > 0) return false
   return data.length > 1 || (data.length === 1 && data[0].files.length === 0)
 }
 
-export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[]): { fileList: OnlineDriveFile[], isTruncated: boolean } => {
+export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[], bucket: string): { fileList: OnlineDriveFile[], isTruncated: boolean } => {
   const fileList: OnlineDriveFile[] = []
   let isTruncated = false
 
   if (data.length === 0)
     return { fileList, isTruncated }
 
-  if (isBucketListInitiation(data, prefix)) {
+  if (isBucketListInitiation(data, prefix, bucket)) {
     data.forEach((item) => {
       fileList.push({
         key: item.bucket,

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -6,16 +6,14 @@ export type OnlineDriveSliceShape = {
   setPrefix: (prefix: string[]) => void
   keywords: string
   setKeywords: (keywords: string) => void
-  startAfter: string
-  setStartAfter: (startAfter: string) => void
   selectedFileList: string[]
   setSelectedFileList: (selectedFileList: string[]) => void
   fileList: OnlineDriveFile[]
   setFileList: (fileList: OnlineDriveFile[]) => void
   bucket: string
   setBucket: (bucket: string) => void
-  isTruncated: boolean
-  setIsTruncated: (isTruncated: boolean) => void
+  startAfter: React.MutableRefObject<string>
+  isTruncated: React.MutableRefObject<boolean>
 }
 
 export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => {
@@ -28,10 +26,7 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
     setKeywords: (keywords: string) => set(() => ({
       keywords,
     })),
-    startAfter: '',
-    setStartAfter: (startAfter: string) => set(() => ({
-      startAfter,
-    })),
+    startAfter: { current: '' },
     selectedFileList: [],
     setSelectedFileList: (selectedFileList: string[]) => set(() => ({
       selectedFileList,
@@ -44,9 +39,6 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
     setBucket: (bucket: string) => set(() => ({
       bucket,
     })),
-    isTruncated: false,
-    setIsTruncated: (isTruncated: boolean) => set(() => ({
-      isTruncated,
-    })),
+    isTruncated: { current: false },
   })
 }