feat: add display status filtering to document list and API (#28342)

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>

web/app/components/base/sort/index.tsx

@@ -47,10 +47,10 @@ const Sort: FC<Props> = ({
             className='block'
           >
             <div className={cn(
-              'flex cursor-pointer items-center rounded-l-lg bg-components-input-bg-normal px-2 py-1 hover:bg-state-base-hover-alt',
+              'flex min-h-8 cursor-pointer items-center rounded-l-lg bg-components-input-bg-normal px-2 py-1 hover:bg-state-base-hover-alt',
               open && '!bg-state-base-hover-alt hover:bg-state-base-hover-alt',
             )}>
-              <div className='flex items-center gap-0.5 p-1'>
+              <div className='flex items-center gap-0.5 px-1'>
                 <div className='system-sm-regular text-text-tertiary'>{t('appLog.filter.sortBy')}</div>
                 <div className={cn('system-sm-regular text-text-tertiary', !!value && 'text-text-secondary')}>
                   {triggerContent}

web/app/components/datasets/documents/hooks/use-document-list-query-state.ts

@@ -1,16 +1,31 @@
 import { type ReadonlyURLSearchParams, usePathname, useRouter, useSearchParams } from 'next/navigation'
 import { useCallback, useMemo } from 'react'
+import { sanitizeStatusValue } from '../status-filter'
+import type { SortType } from '@/service/datasets'
+
+const ALLOWED_SORT_VALUES: SortType[] = ['-created_at', 'created_at', '-hit_count', 'hit_count']
+
+const sanitizeSortValue = (value?: string | null): SortType => {
+  if (!value)
+    return '-created_at'
+
+  return (ALLOWED_SORT_VALUES.includes(value as SortType) ? value : '-created_at') as SortType
+}
 
 export type DocumentListQuery = {
   page: number
   limit: number
   keyword: string
+  status: string
+  sort: SortType
 }
 
 const DEFAULT_QUERY: DocumentListQuery = {
   page: 1,
   limit: 10,
   keyword: '',
+  status: 'all',
+  sort: '-created_at',
 }
 
 // Parse the query parameters from the URL search string.
@@ -18,17 +33,21 @@ function parseParams(params: ReadonlyURLSearchParams): DocumentListQuery {
   const page = Number.parseInt(params.get('page') || '1', 10)
   const limit = Number.parseInt(params.get('limit') || '10', 10)
   const keyword = params.get('keyword') || ''
+  const status = sanitizeStatusValue(params.get('status'))
+  const sort = sanitizeSortValue(params.get('sort'))
 
   return {
     page: page > 0 ? page : 1,
     limit: (limit > 0 && limit <= 100) ? limit : 10,
     keyword: keyword ? decodeURIComponent(keyword) : '',
+    status,
+    sort,
   }
 }
 
 // Update the URL search string with the given query parameters.
 function updateSearchParams(query: DocumentListQuery, searchParams: URLSearchParams) {
-  const { page, limit, keyword } = query || {}
+  const { page, limit, keyword, status, sort } = query || {}
 
   const hasNonDefaultParams = (page && page > 1) || (limit && limit !== 10) || (keyword && keyword.trim())
 
@@ -45,6 +64,18 @@ function updateSearchParams(query: DocumentListQuery, searchParams: URLSearchPar
     searchParams.set('keyword', encodeURIComponent(keyword))
   else
     searchParams.delete('keyword')
+
+  const sanitizedStatus = sanitizeStatusValue(status)
+  if (sanitizedStatus && sanitizedStatus !== 'all')
+    searchParams.set('status', sanitizedStatus)
+  else
+    searchParams.delete('status')
+
+  const sanitizedSort = sanitizeSortValue(sort)
+  if (sanitizedSort !== '-created_at')
+    searchParams.set('sort', sanitizedSort)
+  else
+    searchParams.delete('sort')
 }
 
 function useDocumentListQueryState() {
@@ -57,6 +88,8 @@ function useDocumentListQueryState() {
   // Helper function to update specific query parameters
   const updateQuery = useCallback((updates: Partial<DocumentListQuery>) => {
     const newQuery = { ...query, ...updates }
+    newQuery.status = sanitizeStatusValue(newQuery.status)
+    newQuery.sort = sanitizeSortValue(newQuery.sort)
     const params = new URLSearchParams()
     updateSearchParams(newQuery, params)
     const search = params.toString()

web/app/components/datasets/documents/index.tsx

@@ -25,10 +25,12 @@ import useEditDocumentMetadata from '../metadata/hooks/use-edit-dataset-metadata
 import DatasetMetadataDrawer from '../metadata/metadata-dataset/dataset-metadata-drawer'
 import StatusWithAction from '../common/document-status-with-action/status-with-action'
 import { useDocLink } from '@/context/i18n'
-import { SimpleSelect } from '../../base/select'
-import StatusItem from './detail/completed/status-item'
+import Chip from '../../base/chip'
+import Sort from '../../base/sort'
+import type { SortType } from '@/service/datasets'
 import type { Item } from '@/app/components/base/select'
 import { useIndexStatus } from './status-item/hooks'
+import { normalizeStatusForQuery, sanitizeStatusValue } from './status-filter'
 
 const FolderPlusIcon = ({ className }: React.SVGProps<SVGElement>) => {
   return <svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
@@ -84,13 +86,12 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
   const docLink = useDocLink()
   const { plan } = useProviderContext()
   const isFreePlan = plan.type === 'sandbox'
+  const { query, updateQuery } = useDocumentListQueryState()
   const [inputValue, setInputValue] = useState<string>('') // the input value
   const [searchValue, setSearchValue] = useState<string>('')
-  const [statusFilter, setStatusFilter] = useState<Item>({ value: 'all', name: 'All Status' })
+  const [statusFilterValue, setStatusFilterValue] = useState<string>(() => sanitizeStatusValue(query.status))
+  const [sortValue, setSortValue] = useState<SortType>(query.sort)
   const DOC_INDEX_STATUS_MAP = useIndexStatus()
-
-  // Use the new hook for URL state management
-  const { query, updateQuery } = useDocumentListQueryState()
   const [currPage, setCurrPage] = React.useState<number>(query.page - 1) // Convert to 0-based index
   const [limit, setLimit] = useState<number>(query.limit)
 
@@ -104,7 +105,7 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
   const debouncedSearchValue = useDebounce(searchValue, { wait: 500 })
 
   const statusFilterItems: Item[] = useMemo(() => [
-    { value: 'all', name: 'All Status' },
+    { value: 'all', name: t('datasetDocuments.list.index.all') as string },
     { value: 'queuing', name: DOC_INDEX_STATUS_MAP.queuing.text },
     { value: 'indexing', name: DOC_INDEX_STATUS_MAP.indexing.text },
     { value: 'paused', name: DOC_INDEX_STATUS_MAP.paused.text },
@@ -114,6 +115,11 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
     { value: 'disabled', name: DOC_INDEX_STATUS_MAP.disabled.text },
     { value: 'archived', name: DOC_INDEX_STATUS_MAP.archived.text },
   ], [DOC_INDEX_STATUS_MAP, t])
+  const normalizedStatusFilterValue = useMemo(() => normalizeStatusForQuery(statusFilterValue), [statusFilterValue])
+  const sortItems: Item[] = useMemo(() => [
+    { value: 'created_at', name: t('datasetDocuments.list.sort.uploadTime') as string },
+    { value: 'hit_count', name: t('datasetDocuments.list.sort.hitCount') as string },
+  ], [t])
 
   // Initialize search value from URL on mount
   useEffect(() => {
@@ -131,12 +137,17 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
       setInputValue(query.keyword)
       setSearchValue(query.keyword)
     }
+    setStatusFilterValue((prev) => {
+      const nextValue = sanitizeStatusValue(query.status)
+      return prev === nextValue ? prev : nextValue
+    })
+    setSortValue(query.sort)
   }, [query])
 
   // Update URL when pagination changes
   const handlePageChange = (newPage: number) => {
     setCurrPage(newPage)
-    updateQuery({ page: newPage + 1 }) // Convert to 1-based index
+    updateQuery({ page: newPage + 1 }) // Pagination emits 0-based page, convert to 1-based for URL
   }
 
   // Update URL when limit changes
@@ -160,6 +171,8 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
       page: currPage + 1,
       limit,
       keyword: debouncedSearchValue,
+      status: normalizedStatusFilterValue,
+      sort: sortValue,
     },
     refetchInterval: timerCanRun ? 2500 : 0,
   })
@@ -211,8 +224,14 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
         percent,
       }
     })
-    setTimerCanRun(completedNum !== documentsRes?.data?.length)
-  }, [documentsRes])
+
+    const hasIncompleteDocuments = completedNum !== documentsRes?.data?.length
+    const transientStatuses = ['queuing', 'indexing', 'paused']
+    const shouldForcePolling = normalizedStatusFilterValue === 'all'
+      ? false
+      : transientStatuses.includes(normalizedStatusFilterValue)
+    setTimerCanRun(shouldForcePolling || hasIncompleteDocuments)
+  }, [documentsRes, normalizedStatusFilterValue])
   const total = documentsRes?.total || 0
 
   const routeToDocCreate = () => {
@@ -233,6 +252,10 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
       setSelectedIds([])
   }, [searchValue, query.keyword])
 
+  useEffect(() => {
+    setSelectedIds([])
+  }, [normalizedStatusFilterValue])
+
   const { run: handleSearch } = useDebounceFn(() => {
     setSearchValue(inputValue)
   }, { wait: 500 })
@@ -278,17 +301,24 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
       <div className='flex flex-1 flex-col px-6 py-4'>
         <div className='flex flex-wrap items-center justify-between'>
           <div className='flex items-center gap-2'>
-            <SimpleSelect
-              placeholder={t('datasetDocuments.list.table.header.status')}
-              onSelect={(item) => {
-                setStatusFilter(item)
-              }}
+            <Chip
+              className='w-[160px]'
+              showLeftIcon={false}
+              value={statusFilterValue}
               items={statusFilterItems}
-              defaultValue={statusFilter.value}
-              wrapperClassName='w-[160px] h-8'
-              renderOption={({ item, selected }) => <StatusItem item={item} selected={selected} />}
-              optionClassName='p-0'
-              notClearable
+              onSelect={(item) => {
+                const selectedValue = sanitizeStatusValue(item?.value ? String(item.value) : '')
+                setStatusFilterValue(selectedValue)
+                setCurrPage(0)
+                updateQuery({ status: selectedValue, page: 1 })
+              }}
+              onClear={() => {
+                if (statusFilterValue === 'all')
+                  return
+                setStatusFilterValue('all')
+                setCurrPage(0)
+                updateQuery({ status: 'all', page: 1 })
+              }}
             />
             <Input
               showLeftIcon
@@ -298,6 +328,20 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
               onChange={e => handleInputChange(e.target.value)}
               onClear={() => handleInputChange('')}
             />
+            <div className='h-3.5 w-px bg-divider-regular'></div>
+            <Sort
+              order={sortValue.startsWith('-') ? '-' : ''}
+              value={sortValue.replace('-', '')}
+              items={sortItems}
+              onSelect={(value) => {
+                const next = String(value) as SortType
+                if (next === sortValue)
+                  return
+                setSortValue(next)
+                setCurrPage(0)
+                updateQuery({ sort: next, page: 1 })
+              }}
+            />
           </div>
           <div className='flex !h-8 items-center justify-center gap-2'>
             {!isFreePlan && <AutoDisabledDocument datasetId={datasetId} />}
@@ -343,7 +387,8 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
                 onUpdate={handleUpdate}
                 selectedIds={selectedIds}
                 onSelectedIdChange={setSelectedIds}
-                statusFilter={statusFilter}
+                statusFilterValue={normalizedStatusFilterValue}
+                remoteSortValue={sortValue}
                 pagination={{
                   total,
                   limit,

web/app/components/datasets/documents/list.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback, useMemo, useState } from 'react'
+import React, { useCallback, useEffect, useMemo, useState } from 'react'
 import { useBoolean } from 'ahooks'
 import { ArrowDownIcon } from '@heroicons/react/24/outline'
 import { pick, uniq } from 'lodash-es'
@@ -18,7 +18,6 @@ import BatchAction from './detail/completed/common/batch-action'
 import cn from '@/utils/classnames'
 import Tooltip from '@/app/components/base/tooltip'
 import Toast from '@/app/components/base/toast'
-import type { Item } from '@/app/components/base/select'
 import { asyncRunSafe } from '@/utils'
 import { formatNumber } from '@/utils/format'
 import NotionIcon from '@/app/components/base/notion-icon'
@@ -37,6 +36,7 @@ import EditMetadataBatchModal from '@/app/components/datasets/metadata/edit-meta
 import StatusItem from './status-item'
 import Operations from './operations'
 import { DatasourceType } from '@/models/pipeline'
+import { normalizeStatusForQuery } from '@/app/components/datasets/documents/status-filter'
 
 export const renderTdValue = (value: string | number | null, isEmptyStyle = false) => {
   return (
@@ -66,7 +66,8 @@ type IDocumentListProps = {
   pagination: PaginationProps
   onUpdate: () => void
   onManageMetadata: () => void
-  statusFilter: Item
+  statusFilterValue: string
+  remoteSortValue: string
 }
 
 /**
@@ -81,7 +82,8 @@ const DocumentList: FC<IDocumentListProps> = ({
   pagination,
   onUpdate,
   onManageMetadata,
-  statusFilter,
+  statusFilterValue,
+  remoteSortValue,
 }) => {
   const { t } = useTranslation()
   const { formatTime } = useTimestamp()
@@ -90,9 +92,14 @@ const DocumentList: FC<IDocumentListProps> = ({
   const chunkingMode = datasetConfig?.doc_form
   const isGeneralMode = chunkingMode !== ChunkingMode.parentChild
   const isQAMode = chunkingMode === ChunkingMode.qa
-  const [sortField, setSortField] = useState<'name' | 'word_count' | 'hit_count' | 'created_at' | null>('created_at')
+  const [sortField, setSortField] = useState<'name' | 'word_count' | 'hit_count' | 'created_at' | null>(null)
   const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc')
 
+  useEffect(() => {
+    setSortField(null)
+    setSortOrder('desc')
+  }, [remoteSortValue])
+
   const {
     isShowEditModal,
     showEditModal,
@@ -109,11 +116,10 @@ const DocumentList: FC<IDocumentListProps> = ({
   const localDocs = useMemo(() => {
     let filteredDocs = documents
 
-    if (statusFilter.value !== 'all') {
+    if (statusFilterValue && statusFilterValue !== 'all') {
       filteredDocs = filteredDocs.filter(doc =>
         typeof doc.display_status === 'string'
-        && typeof statusFilter.value === 'string'
-        && doc.display_status.toLowerCase() === statusFilter.value.toLowerCase(),
+        && normalizeStatusForQuery(doc.display_status) === statusFilterValue,
       )
     }
 
@@ -156,7 +162,7 @@ const DocumentList: FC<IDocumentListProps> = ({
     })
 
     return sortedDocs
-  }, [documents, sortField, sortOrder, statusFilter])
+  }, [documents, sortField, sortOrder, statusFilterValue])
 
   const handleSort = (field: 'name' | 'word_count' | 'hit_count' | 'created_at') => {
     if (sortField === field) {

web/app/components/datasets/documents/status-filter.ts

@@ -0,0 +1,33 @@
+import { DisplayStatusList } from '@/models/datasets'
+
+const KNOWN_STATUS_VALUES = new Set<string>([
+  'all',
+  ...DisplayStatusList.map(item => item.toLowerCase()),
+])
+
+const URL_STATUS_ALIASES: Record<string, string> = {
+  active: 'available',
+}
+
+const QUERY_STATUS_ALIASES: Record<string, string> = {
+  enabled: 'available',
+}
+
+export const sanitizeStatusValue = (value?: string | null) => {
+  if (!value)
+    return 'all'
+
+  const normalized = value.toLowerCase()
+  if (URL_STATUS_ALIASES[normalized])
+    return URL_STATUS_ALIASES[normalized]
+
+  return KNOWN_STATUS_VALUES.has(normalized) ? normalized : 'all'
+}
+
+export const normalizeStatusForQuery = (value?: string | null) => {
+  const sanitized = sanitizeStatusValue(value)
+  if (sanitized === 'all')
+    return 'all'
+
+  return QUERY_STATUS_ALIASES[sanitized] || sanitized
+}

web/i18n/en-US/dataset-documents.ts

@@ -40,6 +40,10 @@ const translation = {
       enableTip: 'The file can be indexed',
       disableTip: 'The file cannot be indexed',
     },
+    sort: {
+      uploadTime: 'Upload Time',
+      hitCount: 'Retrieval Count',
+    },
     status: {
       queuing: 'Queuing',
       indexing: 'Indexing',

web/i18n/zh-Hans/dataset-documents.ts

@@ -40,6 +40,10 @@ const translation = {
       enableTip: '该文件可以被索引',
       disableTip: '该文件无法被索引',
     },
+    sort: {
+      uploadTime: '上传时间',
+      hitCount: '召回次数',
+    },
     status: {
       queuing: '排队中',
       indexing: '索引中',

web/service/knowledge/use-document.ts

@@ -9,6 +9,7 @@ import { pauseDocIndexing, resumeDocIndexing } from '../datasets'
 import type { DocumentDetailResponse, DocumentListResponse, UpdateDocumentBatchParams } from '@/models/datasets'
 import { DocumentActionType } from '@/models/datasets'
 import type { CommonResponse } from '@/models/common'
+import { normalizeStatusForQuery } from '@/app/components/datasets/documents/status-filter'
 
 const NAME_SPACE = 'knowledge/document'
 
@@ -20,15 +21,26 @@ export const useDocumentList = (payload: {
     page: number
     limit: number
     sort?: SortType
+    status?: string
   },
   refetchInterval?: number | false
 }) => {
   const { query, datasetId, refetchInterval } = payload
-  const { keyword, page, limit, sort } = query
+  const { keyword, page, limit, sort, status } = query
+  const normalizedStatus = normalizeStatusForQuery(status)
+  const params: Record<string, number | string> = {
+    keyword,
+    page,
+    limit,
+  }
+  if (sort)
+    params.sort = sort
+  if (normalizedStatus && normalizedStatus !== 'all')
+    params.status = normalizedStatus
   return useQuery<DocumentListResponse>({
-    queryKey: [...useDocumentListKey, datasetId, keyword, page, limit, sort],
+    queryKey: [...useDocumentListKey, datasetId, keyword, page, limit, sort, normalizedStatus],
     queryFn: () => get<DocumentListResponse>(`/datasets/${datasetId}/documents`, {
-      params: query,
+      params,
     }),
     refetchInterval,
   })