Feature/mutil embedding model (#908)

Co-authored-by: JzoNg <jzongcode@gmail.com>
Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
This commit is contained in:
Jyong
2023-08-18 17:37:31 +08:00
committed by GitHub
parent 4420281d96
commit db7156dafd
54 changed files with 1704 additions and 278 deletions

View File

@@ -5,13 +5,14 @@ import Link from 'next/link'
import type { MouseEventHandler } from 'react'
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import classNames from 'classnames'
import cn from 'classnames'
import style from '../list.module.css'
import Confirm from '@/app/components/base/confirm'
import { ToastContext } from '@/app/components/base/toast'
import { deleteDataset } from '@/service/datasets'
import AppIcon from '@/app/components/base/app-icon'
import type { DataSet } from '@/models/datasets'
import Tooltip from '@/app/components/base/tooltip'
export type DatasetCardProps = {
dataset: DataSet
@@ -45,26 +46,36 @@ const DatasetCard = ({
return (
<>
<Link href={`/datasets/${dataset.id}/documents`} className={style.listItem}>
<Link href={`/datasets/${dataset.id}/documents`} className={cn(style.listItem)}>
<div className={style.listItemTitle}>
<AppIcon size='small' />
<div className={style.listItemHeading}>
<div className={style.listItemHeadingContent}>{dataset.name}</div>
<AppIcon size='small' className={cn(!dataset.embedding_available && style.unavailable)} />
<div className={cn(style.listItemHeading, !dataset.embedding_available && style.unavailable)}>
<div className={style.listItemHeadingContent}>
{dataset.name}
</div>
</div>
{!dataset.embedding_available && (
<Tooltip
selector={`dataset-tag-${dataset.id}`}
htmlContent={t('dataset.unavailableTip')}
>
<span className='px-1 border boder-gray-200 rounded-md text-gray-500 text-xs font-normal leading-[18px]'>{t('dataset.unavailable')}</span>
</Tooltip>
)}
<span className={style.deleteDatasetIcon} onClick={onDeleteClick} />
</div>
<div className={style.listItemDescription}>{dataset.description}</div>
<div className={classNames(style.listItemFooter, style.datasetCardFooter)}>
<div className={cn(style.listItemDescription, !dataset.embedding_available && style.unavailable)}>{dataset.description}</div>
<div className={cn(style.listItemFooter, style.datasetCardFooter, !dataset.embedding_available && style.unavailable)}>
<span className={style.listItemStats}>
<span className={classNames(style.listItemFooterIcon, style.docIcon)} />
<span className={cn(style.listItemFooterIcon, style.docIcon)} />
{dataset.document_count}{t('dataset.documentCount')}
</span>
<span className={style.listItemStats}>
<span className={classNames(style.listItemFooterIcon, style.textIcon)} />
<span className={cn(style.listItemFooterIcon, style.textIcon)} />
{Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}
</span>
<span className={style.listItemStats}>
<span className={classNames(style.listItemFooterIcon, style.applicationIcon)} />
<span className={cn(style.listItemFooterIcon, style.applicationIcon)} />
{dataset.app_count}{t('dataset.appCount')}
</span>
</div>

View File

@@ -1,13 +1,7 @@
import classNames from 'classnames'
import { getLocaleOnServer } from '@/i18n/server'
import { useTranslation } from '@/i18n/i18next-serverside-config'
import Datasets from './Datasets'
import DatasetFooter from './DatasetFooter'
const AppList = async () => {
const locale = getLocaleOnServer()
const { t } = await useTranslation(locale, 'dataset')
return (
<div className='flex flex-col overflow-auto bg-gray-100 shrink-0 grow'>
<Datasets />

View File

@@ -192,3 +192,11 @@
@apply inline-flex items-center mb-2 text-sm font-medium;
}
/* #endregion new app dialog */
.unavailable {
@apply opacity-50;
}
.listItem:hover .unavailable {
@apply opacity-100;
}

View File

@@ -7,6 +7,7 @@ import TypeIcon from '../type-icon'
import RemoveIcon from '../../base/icons/remove-icon'
import s from './style.module.css'
import { formatNumber } from '@/utils/format'
import Tooltip from '@/app/components/base/tooltip'
export type ICardItemProps = {
className?: string
@@ -36,10 +37,22 @@ const CardItem: FC<ICardItemProps> = ({
'flex items-center justify-between rounded-xl px-3 py-2.5 bg-white border border-gray-200 cursor-pointer')
}>
<div className='shrink-0 flex items-center space-x-2'>
<TypeIcon type="upload_file" />
<div className={cn(!config.embedding_available && 'opacity-50')}>
<TypeIcon type="upload_file" />
</div>
<div>
<div className='w-[160px] text-[13px] leading-[18px] font-medium text-gray-800 overflow-hidden text-ellipsis whitespace-nowrap'>{config.name}</div>
<div className='flex text-xs text-gray-500'>
<div className='flex items-center w-[160px] mr-1'>
<div className={cn('text-[13px] leading-[18px] font-medium text-gray-800 overflow-hidden text-ellipsis whitespace-nowrap', !config.embedding_available && 'opacity-50')}>{config.name}</div>
{!config.embedding_available && (
<Tooltip
selector={`unavailable-tag-${config.id}`}
htmlContent={t('dataset.unavailableTip')}
>
<span className='shrink-0 px-1 border boder-gray-200 rounded-md text-gray-500 text-xs font-normal leading-[18px]'>{t('dataset.unavailable')}</span>
</Tooltip>
)}
</div>
<div className={cn('flex text-xs text-gray-500', !config.embedding_available && 'opacity-50')}>
{formatNumber(config.word_count)} {t('appDebug.feature.dataSet.words')} · {formatNumber(config.document_count)} {t('appDebug.feature.dataSet.textBlocks')}
</div>
</div>

View File

@@ -120,15 +120,24 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
{datasets.map(item => (
<div
key={item.id}
className={cn(s.item, selected.some(i => i.id === item.id) && s.selected, 'flex justify-between items-center h-10 px-2 rounded-lg bg-white border border-gray-200 cursor-pointer')}
onClick={() => toggleSelect(item)}
className={cn(s.item, selected.some(i => i.id === item.id) && s.selected, 'flex justify-between items-center h-10 px-2 rounded-lg bg-white border border-gray-200 cursor-pointer', !item.embedding_available && s.disabled)}
onClick={() => {
if (!item.embedding_available)
return
toggleSelect(item)
}}
>
<div className='flex items-center space-x-2'>
<TypeIcon type="upload_file" size='md' />
<div className='max-w-[200px] text-[13px] font-medium text-gray-800 overflow-hidden text-ellipsis whitespace-nowrap'>{item.name}</div>
<div className='mr-1 flex items-center'>
<div className={cn('mr-2', !item.embedding_available && 'opacity-50')}>
<TypeIcon type="upload_file" size='md' />
</div>
<div className={cn('max-w-[200px] text-[13px] font-medium text-gray-800 overflow-hidden text-ellipsis whitespace-nowrap', !item.embedding_available && 'opacity-50 !max-w-[120px]')}>{item.name}</div>
{!item.embedding_available && (
<span className='ml-1 shrink-0 px-1 border boder-gray-200 rounded-md text-gray-500 text-xs font-normal leading-[18px]'>{t('dataset.unavailable')}</span>
)}
</div>
<div className='flex text-xs text-gray-500 overflow-hidden whitespace-nowrap'>
<div className={cn('shrink-0 flex text-xs text-gray-500 overflow-hidden whitespace-nowrap', !item.embedding_available && 'opacity-50')}>
<span className='max-w-[100px] overflow-hidden text-ellipsis whitespace-nowrap'>{formatNumber(item.word_count)}</span>
{t('appDebug.feature.dataSet.words')}
<span className='px-0.5'>·</span>

View File

@@ -6,4 +6,8 @@
.item.selected {
background: #F5F8FF;
border-color: #528BFF;
}
}
.item.disabled {
@apply bg-white border-gray-200 cursor-default;
}

View File

@@ -1,9 +1,9 @@
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<g id="Icon">
<g id="Icon_2">
<path d="M6 6.5C6.27614 6.5 6.5 6.27614 6.5 6C6.5 5.72386 6.27614 5.5 6 5.5C5.72386 5.5 5.5 5.72386 5.5 6C5.5 6.27614 5.72386 6.5 6 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M9.5 6.5C9.77614 6.5 10 6.27614 10 6C10 5.72386 9.77614 5.5 9.5 5.5C9.22386 5.5 9 5.72386 9 6C9 6.27614 9.22386 6.5 9.5 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M2.5 6.5C2.77614 6.5 3 6.27614 3 6C3 5.72386 2.77614 5.5 2.5 5.5C2.22386 5.5 2 5.72386 2 6C2 6.27614 2.22386 6.5 2.5 6.5Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M8.00008 8.66634C8.36827 8.66634 8.66675 8.36786 8.66675 7.99967C8.66675 7.63148 8.36827 7.33301 8.00008 7.33301C7.63189 7.33301 7.33341 7.63148 7.33341 7.99967C7.33341 8.36786 7.63189 8.66634 8.00008 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M12.6667 8.66634C13.0349 8.66634 13.3334 8.36786 13.3334 7.99967C13.3334 7.63148 13.0349 7.33301 12.6667 7.33301C12.2986 7.33301 12.0001 7.63148 12.0001 7.99967C12.0001 8.36786 12.2986 8.66634 12.6667 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
<path d="M3.33341 8.66634C3.7016 8.66634 4.00008 8.36786 4.00008 7.99967C4.00008 7.63148 3.7016 7.33301 3.33341 7.33301C2.96522 7.33301 2.66675 7.63148 2.66675 7.99967C2.66675 8.36786 2.96522 8.66634 3.33341 8.66634Z" stroke="#344054" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 803 B

After

Width:  |  Height:  |  Size: 1.0 KiB

View File

@@ -4,9 +4,9 @@
"isRootNode": true,
"name": "svg",
"attributes": {
"width": "12",
"height": "12",
"viewBox": "0 0 12 12",
"width": "16",
"height": "16",
"viewBox": "0 0 16 16",
"fill": "none",
"xmlns": "http://www.w3.org/2000/svg"
},
@@ -29,7 +29,7 @@
"type": "element",
"name": "path",
"attributes": {
"d": "M6 6.5C6.27614 6.5 6.5 6.27614 6.5 6C6.5 5.72386 6.27614 5.5 6 5.5C5.72386 5.5 5.5 5.72386 5.5 6C5.5 6.27614 5.72386 6.5 6 6.5Z",
"d": "M8.00008 8.66634C8.36827 8.66634 8.66675 8.36786 8.66675 7.99967C8.66675 7.63148 8.36827 7.33301 8.00008 7.33301C7.63189 7.33301 7.33341 7.63148 7.33341 7.99967C7.33341 8.36786 7.63189 8.66634 8.00008 8.66634Z",
"stroke": "currentColor",
"stroke-width": "1.5",
"stroke-linecap": "round",
@@ -41,7 +41,7 @@
"type": "element",
"name": "path",
"attributes": {
"d": "M9.5 6.5C9.77614 6.5 10 6.27614 10 6C10 5.72386 9.77614 5.5 9.5 5.5C9.22386 5.5 9 5.72386 9 6C9 6.27614 9.22386 6.5 9.5 6.5Z",
"d": "M12.6667 8.66634C13.0349 8.66634 13.3334 8.36786 13.3334 7.99967C13.3334 7.63148 13.0349 7.33301 12.6667 7.33301C12.2986 7.33301 12.0001 7.63148 12.0001 7.99967C12.0001 8.36786 12.2986 8.66634 12.6667 8.66634Z",
"stroke": "currentColor",
"stroke-width": "1.5",
"stroke-linecap": "round",
@@ -53,7 +53,7 @@
"type": "element",
"name": "path",
"attributes": {
"d": "M2.5 6.5C2.77614 6.5 3 6.27614 3 6C3 5.72386 2.77614 5.5 2.5 5.5C2.22386 5.5 2 5.72386 2 6C2 6.27614 2.22386 6.5 2.5 6.5Z",
"d": "M3.33341 8.66634C3.7016 8.66634 4.00008 8.36786 4.00008 7.99967C4.00008 7.63148 3.7016 7.33301 3.33341 7.33301C2.96522 7.33301 2.66675 7.63148 2.66675 7.99967C2.66675 8.36786 2.96522 8.66634 3.33341 8.66634Z",
"stroke": "currentColor",
"stroke-width": "1.5",
"stroke-linecap": "round",
@@ -68,4 +68,4 @@
]
},
"name": "DotsHorizontal"
}
}

View File

@@ -9,6 +9,7 @@ type IPopover = {
position?: 'bottom' | 'br'
btnElement?: string | React.ReactNode
btnClassName?: string | ((open: boolean) => string)
manualClose?: boolean
}
const timeoutDuration = 100
@@ -20,6 +21,7 @@ export default function CustomPopover({
btnElement,
className,
btnClassName,
manualClose,
}: IPopover) {
const buttonRef = useRef<HTMLButtonElement>(null)
const timeOutRef = useRef<NodeJS.Timeout | null>(null)
@@ -62,17 +64,14 @@ export default function CustomPopover({
</Popover.Button>
<Transition as={Fragment}>
<Popover.Panel
className={`${s.popupPanel} ${
position === 'br'
? 'right-0'
: 'transform -translate-x-1/2 left-1/2'
} ${className}`}
className={`${s.popupPanel} ${position === 'br' ? 'right-0' : 'translate-x-1/2 left-1/2'} ${className}`}
{...(trigger !== 'hover'
? {}
: {
onMouseLeave: () => onMouseLeave(open),
onMouseEnter: () => onMouseEnter(open),
})}
})
}
>
{({ close }) => (
<div
@@ -82,10 +81,16 @@ export default function CustomPopover({
: {
onMouseLeave: () => onMouseLeave(open),
onMouseEnter: () => onMouseEnter(open),
})}
})
}
>
{cloneElement(htmlContent as React.ReactElement, {
onClose: () => close(),
onClose: () => onMouseLeave(open),
...(manualClose
? {
onClick: close,
}
: {}),
})}
</div>
)}

View File

@@ -29,7 +29,7 @@ const ACCEPTS = [
'.txt',
// '.xls',
'.xlsx',
'.csv',
// '.csv',
]
const FileUploader = ({

View File

@@ -2,12 +2,14 @@
'use client'
import React, { useEffect, useLayoutEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import { useBoolean } from 'ahooks'
import { XMarkIcon } from '@heroicons/react/20/solid'
import cn from 'classnames'
import Link from 'next/link'
import { groupBy } from 'lodash-es'
import PreviewItem, { PreviewType } from './preview-item'
import LanguageSelect from './language-select'
import s from './index.module.css'
import type { CreateDocumentReq, CustomFile, FullDocumentDetail, FileIndexingEstimateResponse as IndexingEstimateResponse, NotionInfo, PreProcessingRule, Rules, createDocumentResponse } from '@/models/datasets'
import {
@@ -22,11 +24,13 @@ import Loading from '@/app/components/base/loading'
import Toast from '@/app/components/base/toast'
import { formatNumber } from '@/utils/format'
import type { DataSourceNotionPage } from '@/models/common'
import { DataSourceType } from '@/models/datasets'
import { DataSourceType, DocForm } from '@/models/datasets'
import NotionIcon from '@/app/components/base/notion-icon'
import Switch from '@/app/components/base/switch'
import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
import { XClose } from '@/app/components/base/icons/src/vender/line/general'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import I18n from '@/context/i18n'
import { IS_CE_EDITION } from '@/config'
type Page = DataSourceNotionPage & { workspace_id: string }
@@ -56,10 +60,6 @@ enum IndexingType {
QUALIFIED = 'high_quality',
ECONOMICAL = 'economy',
}
enum DocForm {
TEXT = 'text_model',
QA = 'qa_model',
}
const StepTwo = ({
isSetting,
@@ -78,6 +78,8 @@ const StepTwo = ({
onCancel,
}: StepTwoProps) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
const { mutateDatasetRes } = useDatasetDetailContext()
const scrollRef = useRef<HTMLDivElement>(null)
const [scrolled, setScrolled] = useState(false)
@@ -98,6 +100,8 @@ const StepTwo = ({
const [docForm, setDocForm] = useState<DocForm | string>(
datasetId && documentDetail ? documentDetail.doc_form : DocForm.TEXT,
)
const [docLanguage, setDocLanguage] = useState<string>(locale === 'en' ? 'English' : 'Chinese')
const [QATipHide, setQATipHide] = useState(false)
const [previewSwitched, setPreviewSwitched] = useState(false)
const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean()
const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState<IndexingEstimateResponse | null>(null)
@@ -230,6 +234,8 @@ const StepTwo = ({
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
dataset_id: datasetId,
}
}
if (dataSourceType === DataSourceType.NOTION) {
@@ -241,6 +247,8 @@ const StepTwo = ({
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
dataset_id: datasetId,
}
}
return params
@@ -252,6 +260,7 @@ const StepTwo = ({
params = {
original_document_id: documentDetail?.id,
doc_form: docForm,
doc_language: docLanguage,
process_rule: getProcessRule(),
} as CreateDocumentReq
}
@@ -266,6 +275,7 @@ const StepTwo = ({
indexing_technique: getIndexing_technique(),
process_rule: getProcessRule(),
doc_form: docForm,
doc_language: docLanguage,
} as CreateDocumentReq
if (dataSourceType === DataSourceType.FILE) {
params.data_source.info_list.file_info_list = {
@@ -348,6 +358,10 @@ const StepTwo = ({
setDocForm(DocForm.TEXT)
}
const handleSelect = (language: string) => {
setDocLanguage(language)
}
const changeToEconomicalType = () => {
if (!hasSetIndexType) {
setIndexType(IndexingType.ECONOMICAL)
@@ -574,21 +588,32 @@ const StepTwo = ({
</div>
)}
{IS_CE_EDITION && indexType === IndexingType.QUALIFIED && (
<div className='flex justify-between items-center mt-3 px-5 py-4 rounded-xl bg-gray-50 border border-gray-100'>
<div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
<MessageChatSquare className='w-4 h-4' />
</div>
<div className='grow mx-3'>
<div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
<div className='text-[13px] leading-[18px] text-gray-500'>{t('datasetCreation.stepTwo.QATip')}</div>
</div>
<div className='shrink-0'>
<Switch
defaultValue={docForm === DocForm.QA}
onChange={handleSwitch}
size='md'
/>
<div className='mt-3 rounded-xl bg-gray-50 border border-gray-100'>
<div className='flex justify-between items-center px-5 py-4'>
<div className='flex justify-center items-center w-8 h-8 rounded-lg bg-indigo-50'>
<MessageChatSquare className='w-4 h-4' />
</div>
<div className='grow mx-3'>
<div className='mb-[2px] text-md font-medium text-gray-900'>{t('datasetCreation.stepTwo.QATitle')}</div>
<div className='inline-flex items-center text-[13px] leading-[18px] text-gray-500'>
<span className='pr-1'>{t('datasetCreation.stepTwo.QALanguage')}</span>
<LanguageSelect currentLanguage={docLanguage} onSelect={handleSelect} />
</div>
</div>
<div className='shrink-0'>
<Switch
defaultValue={docForm === DocForm.QA}
onChange={handleSwitch}
size='md'
/>
</div>
</div>
{docForm === DocForm.QA && !QATipHide && (
<div className='flex justify-between items-center px-5 py-2 bg-orange-50 border-t border-amber-100 rounded-b-xl text-[13px] leading-[18px] text-medium text-amber-500'>
{t('datasetCreation.stepTwo.QATip')}
<XClose className='w-4 h-4 text-gray-500 cursor-pointer' onClick={() => setQATipHide(true)} />
</div>
)}
</div>
)}
<div className={s.source}>

View File

@@ -0,0 +1,38 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from 'classnames'
import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows'
import Popover from '@/app/components/base/popover'
export type ILanguageSelectProps = {
currentLanguage: string
onSelect: (language: string) => void
}
const LanguageSelect: FC<ILanguageSelectProps> = ({
currentLanguage,
onSelect,
}) => {
return (
<Popover
manualClose
trigger='click'
htmlContent={
<div className='w-full py-1'>
<div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('English')}>English</div>
<div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={() => onSelect('Chinese')}></div>
</div>
}
btnElement={
<div className='inline-flex items-center'>
<span className='pr-[2px] text-xs leading-[18px] font-medium'>{currentLanguage === 'English' ? 'English' : '简体中文'}</span>
<ChevronDown className='w-3 h-3 opacity-60' />
</div>
}
btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')}
className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]'
/>
)
}
export default React.memo(LanguageSelect)

View File

@@ -0,0 +1,108 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import {
useCSVDownloader,
} from 'react-papaparse'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import { Download02 as DownloadIcon } from '@/app/components/base/icons/src/vender/solid/general'
import { DocForm } from '@/models/datasets'
import I18n from '@/context/i18n'
const CSV_TEMPLATE_QA_EN = [
['question', 'answer'],
['question1', 'answer1'],
['question2', 'answer2'],
]
const CSV_TEMPLATE_QA_CN = [
['问题', '答案'],
['问题 1', '答案 1'],
['问题 2', '答案 2'],
]
const CSV_TEMPLATE_EN = [
['segment content'],
['content1'],
['content2'],
]
const CSV_TEMPLATE_CN = [
['分段内容'],
['内容 1'],
['内容 2'],
]
const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => {
const { t } = useTranslation()
const { locale } = useContext(I18n)
const { CSVDownloader, Type } = useCSVDownloader()
const getTemplate = () => {
if (locale === 'en') {
if (docForm === DocForm.QA)
return CSV_TEMPLATE_QA_EN
return CSV_TEMPLATE_EN
}
if (docForm === DocForm.QA)
return CSV_TEMPLATE_QA_CN
return CSV_TEMPLATE_CN
}
return (
<div className='mt-6'>
<div className='text-sm text-gray-900 font-medium'>{t('share.generation.csvStructureTitle')}</div>
<div className='mt-2 max-h-[500px] overflow-auto'>
{docForm === DocForm.QA && (
<table className='table-fixed w-full border-separate border-spacing-0 border border-gray-200 rounded-lg text-xs'>
<thead className='text-gray-500'>
<tr>
<td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.question')}</td>
<td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.answer')}</td>
</tr>
</thead>
<tbody className='text-gray-700'>
<tr>
<td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 1</td>
<td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 1</td>
</tr>
<tr>
<td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.question')} 2</td>
<td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.answer')} 2</td>
</tr>
</tbody>
</table>
)}
{docForm === DocForm.TEXT && (
<table className='table-fixed w-full border-separate border-spacing-0 border border-gray-200 rounded-lg text-xs'>
<thead className='text-gray-500'>
<tr>
<td className='h-9 pl-3 pr-2 border-b border-gray-200'>{t('datasetDocuments.list.batchModal.contentTitle')}</td>
</tr>
</thead>
<tbody className='text-gray-700'>
<tr>
<td className='h-9 pl-3 pr-2 border-b border-gray-100 text-[13px]'>{t('datasetDocuments.list.batchModal.content')} 1</td>
</tr>
<tr>
<td className='h-9 pl-3 pr-2 text-[13px]'>{t('datasetDocuments.list.batchModal.content')} 2</td>
</tr>
</tbody>
</table>
)}
</div>
<CSVDownloader
className="block mt-2 cursor-pointer"
type={Type.Link}
filename={'template'}
bom={true}
data={getTemplate()}
>
<div className='flex items-center h-[18px] space-x-1 text-[#155EEF] text-xs font-medium'>
<DownloadIcon className='w-3 h-3 mr-1' />
{t('datasetDocuments.list.batchModal.template')}
</div>
</CSVDownloader>
</div>
)
}
export default React.memo(CSVDownload)

View File

@@ -0,0 +1,126 @@
'use client'
import type { FC } from 'react'
import React, { useEffect, useRef, useState } from 'react'
import cn from 'classnames'
import { useTranslation } from 'react-i18next'
import { useContext } from 'use-context-selector'
import { Csv as CSVIcon } from '@/app/components/base/icons/src/public/files'
import { ToastContext } from '@/app/components/base/toast'
import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
import Button from '@/app/components/base/button'
export type Props = {
file: File | undefined
updateFile: (file?: File) => void
}
const CSVUploader: FC<Props> = ({
file,
updateFile,
}) => {
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const [dragging, setDragging] = useState(false)
const dropRef = useRef<HTMLDivElement>(null)
const dragRef = useRef<HTMLDivElement>(null)
const fileUploader = useRef<HTMLInputElement>(null)
const handleDragEnter = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
e.target !== dragRef.current && setDragging(true)
}
const handleDragOver = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
}
const handleDragLeave = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
e.target === dragRef.current && setDragging(false)
}
const handleDrop = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
setDragging(false)
if (!e.dataTransfer)
return
const files = [...e.dataTransfer.files]
if (files.length > 1) {
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.count') })
return
}
updateFile(files[0])
}
const selectHandle = () => {
if (fileUploader.current)
fileUploader.current.click()
}
const removeFile = () => {
if (fileUploader.current)
fileUploader.current.value = ''
updateFile()
}
const fileChangeHandle = (e: React.ChangeEvent<HTMLInputElement>) => {
const currentFile = e.target.files?.[0]
updateFile(currentFile)
}
useEffect(() => {
dropRef.current?.addEventListener('dragenter', handleDragEnter)
dropRef.current?.addEventListener('dragover', handleDragOver)
dropRef.current?.addEventListener('dragleave', handleDragLeave)
dropRef.current?.addEventListener('drop', handleDrop)
return () => {
dropRef.current?.removeEventListener('dragenter', handleDragEnter)
dropRef.current?.removeEventListener('dragover', handleDragOver)
dropRef.current?.removeEventListener('dragleave', handleDragLeave)
dropRef.current?.removeEventListener('drop', handleDrop)
}
}, [])
return (
<div className='mt-6'>
<input
ref={fileUploader}
style={{ display: 'none' }}
type="file"
id="fileUploader"
accept='.csv'
onChange={fileChangeHandle}
/>
<div ref={dropRef}>
{!file && (
<div className={cn('flex items-center h-20 rounded-xl bg-gray-50 border border-dashed border-gray-200 text-sm font-normal', dragging && 'bg-[#F5F8FF] border border-[#B2CCFF]')}>
<div className='w-full flex items-center justify-center space-x-2'>
<CSVIcon className="shrink-0" />
<div className='text-gray-500'>
{t('datasetDocuments.list.batchModal.csvUploadTitle')}
<span className='text-primary-400 cursor-pointer' onClick={selectHandle}>{t('datasetDocuments.list.batchModal.browse')}</span>
</div>
</div>
{dragging && <div ref={dragRef} className='absolute w-full h-full top-0 left-0'/>}
</div>
)}
{file && (
<div className={cn('flex items-center h-20 px-6 rounded-xl bg-gray-50 border border-gray-200 text-sm font-normal group', 'hover:bg-[#F5F8FF] hover:border-[#B2CCFF]')}>
<CSVIcon className="shrink-0" />
<div className='flex ml-2 w-0 grow'>
<span className='max-w-[calc(100%_-_30px)] text-ellipsis whitespace-nowrap overflow-hidden text-gray-800'>{file.name.replace(/.csv$/, '')}</span>
<span className='shrink-0 text-gray-500'>.csv</span>
</div>
<div className='hidden group-hover:flex items-center'>
<Button className='!h-8 !px-3 !py-[6px] bg-white !text-[13px] !leading-[18px] text-gray-700' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.change')}</Button>
<div className='mx-2 w-px h-4 bg-gray-200' />
<div className='p-2 cursor-pointer' onClick={removeFile}>
<Trash03 className='w-4 h-4 text-gray-500' />
</div>
</div>
</div>
)}
</div>
</div>
)
}
export default React.memo(CSVUploader)

View File

@@ -0,0 +1,65 @@
'use client'
import type { FC } from 'react'
import React, { useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import CSVUploader from './csv-uploader'
import CSVDownloader from './csv-downloader'
import Button from '@/app/components/base/button'
import Modal from '@/app/components/base/modal'
import { XClose } from '@/app/components/base/icons/src/vender/line/general'
import type { DocForm } from '@/models/datasets'
export type IBatchModalProps = {
isShow: boolean
docForm: DocForm
onCancel: () => void
onConfirm: (file: File) => void
}
const BatchModal: FC<IBatchModalProps> = ({
isShow,
docForm,
onCancel,
onConfirm,
}) => {
const { t } = useTranslation()
const [currentCSV, setCurrentCSV] = useState<File>()
const handleFile = (file?: File) => setCurrentCSV(file)
const handleSend = () => {
if (!currentCSV)
return
onCancel()
onConfirm(currentCSV)
}
useEffect(() => {
if (!isShow)
setCurrentCSV(undefined)
}, [isShow])
return (
<Modal isShow={isShow} onClose={() => {}} className='px-8 py-6 !max-w-[520px] !rounded-xl'>
<div className='relative pb-1 text-xl font-medium leading-[30px] text-gray-900'>{t('datasetDocuments.list.batchModal.title')}</div>
<div className='absolute right-4 top-4 p-2 cursor-pointer' onClick={onCancel}>
<XClose className='w-4 h-4 text-gray-500' />
</div>
<CSVUploader
file={currentCSV}
updateFile={handleFile}
/>
<CSVDownloader
docForm={docForm}
/>
<div className='mt-[28px] pt-6 flex justify-end'>
<Button className='mr-2 text-gray-700 text-sm font-medium' onClick={onCancel}>
{t('datasetDocuments.list.batchModal.cancel')}
</Button>
<Button className='text-sm font-medium' type="primary" onClick={handleSend} disabled={!currentCSV}>
{t('datasetDocuments.list.batchModal.run')}
</Button>
</div>
</Modal>
)
}
export default React.memo(BatchModal)

View File

@@ -13,6 +13,9 @@ type IInfiniteVirtualListProps = {
loadNextPage: () => Promise<any> // Callback function responsible for loading the next page of items.
onClick: (detail: SegmentDetailModel) => void
onChangeSwitch: (segId: string, enabled: boolean) => Promise<void>
onDelete: (segId: string) => Promise<void>
archived?: boolean
}
const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
@@ -22,6 +25,8 @@ const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
loadNextPage,
onClick: onClickCard,
onChangeSwitch,
onDelete,
archived,
}) => {
// If there are more items to be loaded then add an extra row to hold a loading indicator.
const itemCount = hasNextPage ? items.length + 1 : items.length
@@ -52,7 +57,9 @@ const InfiniteVirtualList: FC<IInfiniteVirtualListProps> = ({
detail={segItem}
onClick={() => onClickCard(segItem)}
onChangeSwitch={onChangeSwitch}
onDelete={onDelete}
loading={false}
archived={archived}
/>
))
}

View File

@@ -1,5 +1,5 @@
import type { FC } from 'react'
import React from 'react'
import React, { useState } from 'react'
import cn from 'classnames'
import { ArrowUpRightIcon } from '@heroicons/react/24/outline'
import { useTranslation } from 'react-i18next'
@@ -7,11 +7,15 @@ import { StatusItem } from '../../list'
import { DocumentTitle } from '../index'
import s from './style.module.css'
import { SegmentIndexTag } from './index'
import Modal from '@/app/components/base/modal'
import Button from '@/app/components/base/button'
import Switch from '@/app/components/base/switch'
import Divider from '@/app/components/base/divider'
import Indicator from '@/app/components/header/indicator'
import { formatNumber } from '@/utils/format'
import type { SegmentDetailModel } from '@/models/datasets'
import { AlertCircle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
const ProgressBar: FC<{ percent: number; loading: boolean }> = ({ percent, loading }) => {
return (
@@ -35,8 +39,10 @@ type ISegmentCardProps = {
score?: number
onClick?: () => void
onChangeSwitch?: (segId: string, enabled: boolean) => Promise<void>
onDelete?: (segId: string) => Promise<void>
scene?: UsageScene
className?: string
archived?: boolean
}
const SegmentCard: FC<ISegmentCardProps> = ({
@@ -44,9 +50,11 @@ const SegmentCard: FC<ISegmentCardProps> = ({
score,
onClick,
onChangeSwitch,
onDelete,
loading = true,
scene = 'doc',
className = '',
archived,
}) => {
const { t } = useTranslation()
const {
@@ -60,6 +68,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
answer,
} = detail as any
const isDocScene = scene === 'doc'
const [showModal, setShowModal] = useState(false)
const renderContent = () => {
if (answer) {
@@ -86,7 +95,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
s.segWrapper,
(isDocScene && !enabled) ? 'bg-gray-25' : '',
'group',
!loading ? 'pb-4' : '',
!loading ? 'pb-4 hover:pb-[10px]' : '',
className,
)}
onClick={() => onClick?.()}
@@ -116,6 +125,7 @@ const SegmentCard: FC<ISegmentCardProps> = ({
>
<Switch
size='md'
disabled={archived}
defaultValue={enabled}
onChange={async (val) => {
await onChangeSwitch?.(id, val)
@@ -159,10 +169,18 @@ const SegmentCard: FC<ISegmentCardProps> = ({
<div className={cn(s.commonIcon, s.targetIcon)} />
<div className={s.segDataText}>{formatNumber(hit_count)}</div>
</div>
<div className="flex items-center">
<div className="grow flex items-center">
<div className={cn(s.commonIcon, s.bezierCurveIcon)} />
<div className={s.segDataText}>{index_node_hash}</div>
</div>
{!archived && (
<div className='shrink-0 w-6 h-6 flex items-center justify-center rounded-md hover:bg-red-100 hover:text-red-600 cursor-pointer group/delete' onClick={(e) => {
e.stopPropagation()
setShowModal(true)
}}>
<Trash03 className='w-[14px] h-[14px] text-gray-500 group-hover/delete:text-red-600' />
</div>
)}
</div>
</>
: <>
@@ -187,6 +205,26 @@ const SegmentCard: FC<ISegmentCardProps> = ({
</div>
</>
)}
{showModal && <Modal isShow={showModal} onClose={() => setShowModal(false)} className={s.delModal} closable>
<div>
<div className={s.warningWrapper}>
<AlertCircle className='w-6 h-6 text-red-600' />
</div>
<div className='text-xl font-semibold text-gray-900 mb-1'>{t('datasetDocuments.segment.delete')}</div>
<div className='flex gap-2 justify-end'>
<Button onClick={() => setShowModal(false)}>{t('common.operation.cancel')}</Button>
<Button
type='warning'
onClick={async () => {
await onDelete?.(id)
}}
className='border-red-700 border-[0.5px]'
>
{t('common.operation.sure')}
</Button>
</div>
</div>
</Modal>}
</div>
)
}

View File

@@ -8,6 +8,7 @@ import { debounce, isNil, omitBy } from 'lodash-es'
import cn from 'classnames'
import { StatusItem } from '../../list'
import { DocumentContext } from '../index'
import { ProcessStatus } from '../segment-add'
import s from './style.module.css'
import InfiniteVirtualList from './InfiniteVirtualList'
import { formatNumber } from '@/utils/format'
@@ -18,7 +19,7 @@ import Input from '@/app/components/base/input'
import { ToastContext } from '@/app/components/base/toast'
import type { Item } from '@/app/components/base/select'
import { SimpleSelect } from '@/app/components/base/select'
import { disableSegment, enableSegment, fetchSegments, updateSegment } from '@/service/datasets'
import { deleteSegment, disableSegment, enableSegment, fetchSegments, updateSegment } from '@/service/datasets'
import type { SegmentDetailModel, SegmentUpdator, SegmentsQuery, SegmentsResponse } from '@/models/datasets'
import { asyncRunSafe } from '@/utils'
import type { CommonResponse } from '@/models/common'
@@ -48,12 +49,14 @@ type ISegmentDetailProps = {
onChangeSwitch?: (segId: string, enabled: boolean) => Promise<void>
onUpdate: (segmentId: string, q: string, a: string, k: string[]) => void
onCancel: () => void
archived?: boolean
}
/**
* Show all the contents of the segment
*/
export const SegmentDetail: FC<ISegmentDetailProps> = memo(({
segInfo,
archived,
onChangeSwitch,
onUpdate,
onCancel,
@@ -116,31 +119,30 @@ export const SegmentDetail: FC<ISegmentDetailProps> = memo(({
return (
<div className={'flex flex-col relative'}>
<div className='absolute right-0 top-0 flex items-center h-7'>
{
isEditing
? (
<>
<Button
className='mr-2 !h-7 !px-3 !py-[5px] text-xs font-medium text-gray-700 !rounded-md'
onClick={handleCancel}>
{t('common.operation.cancel')}
</Button>
<Button
type='primary'
className='!h-7 !px-3 !py-[5px] text-xs font-medium !rounded-md'
onClick={handleSave}>
{t('common.operation.save')}
</Button>
</>
)
: (
<div className='group relative flex justify-center items-center w-6 h-6 hover:bg-gray-100 rounded-md cursor-pointer'>
<div className={cn(s.editTip, 'hidden items-center absolute -top-10 px-3 h-[34px] bg-white rounded-lg whitespace-nowrap text-xs font-semibold text-gray-700 group-hover:flex')}>{t('common.operation.edit')}</div>
<Edit03 className='w-4 h-4 text-gray-500' onClick={() => setIsEditing(true)} />
</div>
)
}
<div className='mx-3 w-[1px] h-3 bg-gray-200' />
{isEditing && (
<>
<Button
className='mr-2 !h-7 !px-3 !py-[5px] text-xs font-medium text-gray-700 !rounded-md'
onClick={handleCancel}>
{t('common.operation.cancel')}
</Button>
<Button
type='primary'
className='!h-7 !px-3 !py-[5px] text-xs font-medium !rounded-md'
onClick={handleSave}>
{t('common.operation.save')}
</Button>
</>
)}
{!isEditing && !archived && (
<>
<div className='group relative flex justify-center items-center w-6 h-6 hover:bg-gray-100 rounded-md cursor-pointer'>
<div className={cn(s.editTip, 'hidden items-center absolute -top-10 px-3 h-[34px] bg-white rounded-lg whitespace-nowrap text-xs font-semibold text-gray-700 group-hover:flex')}>{t('common.operation.edit')}</div>
<Edit03 className='w-4 h-4 text-gray-500' onClick={() => setIsEditing(true)} />
</div>
<div className='mx-3 w-[1px] h-3 bg-gray-200' />
</>
)}
<div className='flex justify-center items-center w-6 h-6 cursor-pointer' onClick={onCancel}>
<XClose className='w-4 h-4 text-gray-500' />
</div>
@@ -176,6 +178,7 @@ export const SegmentDetail: FC<ISegmentDetailProps> = memo(({
onChange={async (val) => {
await onChangeSwitch?.(segInfo?.id || '', val)
}}
disabled={archived}
/>
</div>
</div>
@@ -195,13 +198,20 @@ export const splitArray = (arr: any[], size = 3) => {
type ICompletedProps = {
showNewSegmentModal: boolean
onNewSegmentModalChange: (state: boolean) => void
importStatus: ProcessStatus | string | undefined
archived?: boolean
// data: Array<{}> // all/part segments
}
/**
* Embedding done, show list of all segments
* Support search and filter
*/
const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModalChange }) => {
const Completed: FC<ICompletedProps> = ({
showNewSegmentModal,
onNewSegmentModalChange,
importStatus,
archived,
}) => {
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const { datasetId = '', documentId = '', docForm } = useContext(DocumentContext)
@@ -250,11 +260,6 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
getSegments(false)
}
useEffect(() => {
if (lastSegmentsRes !== undefined)
getSegments(false)
}, [selectedStatus, searchValue])
const onClickCard = (detail: SegmentDetailModel) => {
setCurrSegment({ segInfo: detail, showModal: true })
}
@@ -281,6 +286,17 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
}
}
const onDelete = async (segId: string) => {
const [e] = await asyncRunSafe<CommonResponse>(deleteSegment({ datasetId, documentId, segmentId: segId }) as Promise<CommonResponse>)
if (!e) {
notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
resetList()
}
else {
notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
}
}
const handleUpdateSegment = async (segmentId: string, question: string, answer: string, keywords: string[]) => {
const params: SegmentUpdator = { content: '' }
if (docForm === 'qa_model') {
@@ -321,6 +337,16 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
setAllSegments([...allSegments])
}
useEffect(() => {
if (lastSegmentsRes !== undefined)
getSegments(false)
}, [selectedStatus, searchValue])
useEffect(() => {
if (importStatus === ProcessStatus.COMPLETED)
resetList()
}, [importStatus])
return (
<>
<div className={s.docSearchWrapper}>
@@ -343,7 +369,9 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
items={allSegments}
loadNextPage={getSegments}
onChangeSwitch={onChangeSwitch}
onDelete={onDelete}
onClick={onClickCard}
archived={archived}
/>
<Modal isShow={currSegment.showModal} onClose={() => {}} className='!max-w-[640px] !overflow-visible'>
<SegmentDetail
@@ -351,6 +379,7 @@ const Completed: FC<ICompletedProps> = ({ showNewSegmentModal, onNewSegmentModal
onChangeSwitch={onChangeSwitch}
onUpdate={handleUpdateSegment}
onCancel={onCloseModal}
archived={archived}
/>
</Modal>
<NewSegmentModal

View File

@@ -132,3 +132,24 @@
.editTip {
box-shadow: 0px 4px 6px -2px rgba(16, 24, 40, 0.03), 0px 12px 16px -4px rgba(16, 24, 40, 0.08);
}
.delModal {
background: linear-gradient(
180deg,
rgba(217, 45, 32, 0.05) 0%,
rgba(217, 45, 32, 0) 24.02%
),
#f9fafb;
box-shadow: 0px 20px 24px -4px rgba(16, 24, 40, 0.08),
0px 8px 8px -4px rgba(16, 24, 40, 0.03);
@apply rounded-2xl p-8;
}
.warningWrapper {
box-shadow: 0px 20px 24px -4px rgba(16, 24, 40, 0.08),
0px 8px 8px -4px rgba(16, 24, 40, 0.03);
background: rgba(255, 255, 255, 0.9);
@apply h-12 w-12 border-[0.5px] border-gray-100 rounded-xl mb-3 flex items-center justify-center;
}
.warningIcon {
@apply w-[22px] h-[22px] fill-current text-red-600;
}

View File

@@ -3,7 +3,7 @@ import type { FC } from 'react'
import React, { useState } from 'react'
import useSWR from 'swr'
import { ArrowLeftIcon } from '@heroicons/react/24/solid'
import { createContext } from 'use-context-selector'
import { createContext, useContext } from 'use-context-selector'
import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { omit } from 'lodash-es'
@@ -13,19 +13,15 @@ import s from '../style.module.css'
import Completed from './completed'
import Embedding from './embedding'
import Metadata from './metadata'
import SegmentAdd, { ProcessStatus } from './segment-add'
import BatchModal from './batch-modal'
import style from './style.module.css'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import type { MetadataType } from '@/service/datasets'
import { fetchDocumentDetail } from '@/service/datasets'
export const BackCircleBtn: FC<{ onClick: () => void }> = ({ onClick }) => {
return (
<div onClick={onClick} className={'rounded-full w-8 h-8 flex justify-center items-center border-gray-100 cursor-pointer border hover:border-gray-300 shadow-lg'}>
<ArrowLeftIcon className='text-primary-600 fill-current stroke-current h-4 w-4' />
</div>
)
}
import { checkSegmentBatchImportProgress, fetchDocumentDetail, segmentBatchImport } from '@/service/datasets'
import { ToastContext } from '@/app/components/base/toast'
import type { DocForm } from '@/models/datasets'
export const DocumentContext = createContext<{ datasetId?: string; documentId?: string; docForm: string }>({ docForm: '' })
@@ -51,10 +47,45 @@ type Props = {
}
const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
const { t } = useTranslation()
const router = useRouter()
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const [showMetadata, setShowMetadata] = useState(true)
const [showNewSegmentModal, setShowNewSegmentModal] = useState(false)
const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false)
const [batchModalVisible, setBatchModalVisible] = useState(false)
const [importStatus, setImportStatus] = useState<ProcessStatus | string>()
const showNewSegmentModal = () => setNewSegmentModalVisible(true)
const showBatchModal = () => setBatchModalVisible(true)
const hideBatchModal = () => setBatchModalVisible(false)
const resetProcessStatus = () => setImportStatus('')
const checkProcess = async (jobID: string) => {
try {
const res = await checkSegmentBatchImportProgress({ jobID })
setImportStatus(res.job_status)
if (res.job_status === ProcessStatus.WAITING || res.job_status === ProcessStatus.PROCESSING)
setTimeout(() => checkProcess(res.job_id), 2500)
if (res.job_status === ProcessStatus.ERROR)
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
}
catch (e: any) {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
}
}
const runBatch = async (csv: File) => {
const formData = new FormData()
formData.append('file', csv)
try {
const res = await segmentBatchImport({
url: `/datasets/${datasetId}/documents/${documentId}/segments/batch_import`,
body: formData,
})
setImportStatus(res.job_status)
checkProcess(res.job_id)
}
catch (e: any) {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
}
}
const { data: documentDetail, error, mutate: detailMutate } = useSWR({
action: 'fetchDocumentDetail',
@@ -91,22 +122,32 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
<DocumentContext.Provider value={{ datasetId, documentId, docForm: documentDetail?.doc_form || '' }}>
<div className='flex flex-col h-full'>
<div className='flex h-16 border-b-gray-100 border-b items-center p-4'>
<BackCircleBtn onClick={backToPrev} />
<div onClick={backToPrev} className={'rounded-full w-8 h-8 flex justify-center items-center border-gray-100 cursor-pointer border hover:border-gray-300 shadow-[0px_12px_16px_-4px_rgba(16,24,40,0.08),0px_4px_6px_-2px_rgba(16,24,40,0.03)]'}>
<ArrowLeftIcon className='text-primary-600 fill-current stroke-current h-4 w-4' />
</div>
<Divider className='!h-4' type='vertical' />
<DocumentTitle extension={documentDetail?.data_source_info?.upload_file?.extension} name={documentDetail?.name} />
<StatusItem status={documentDetail?.display_status || 'available'} scene='detail' />
{documentDetail && !documentDetail.archived && (
<SegmentAdd
importStatus={importStatus}
clearProcessStatus={resetProcessStatus}
showNewSegmentModal={showNewSegmentModal}
showBatchModal={showBatchModal}
/>
)}
<OperationAction
scene='detail'
detail={{
enabled: documentDetail?.enabled || false,
archived: documentDetail?.archived || false,
id: documentId,
data_source_type: documentDetail?.data_source_type || '',
doc_form: documentDetail?.doc_form || '',
}}
datasetId={datasetId}
onUpdate={handleOperate}
className='!w-[216px]'
showNewSegmentModal={() => setShowNewSegmentModal(true)}
/>
<button
className={cn(style.layoutRightIcon, showMetadata ? style.iconShow : style.iconClose)}
@@ -120,8 +161,10 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
{embedding
? <Embedding detail={documentDetail} detailUpdate={detailMutate} />
: <Completed
showNewSegmentModal={showNewSegmentModal}
onNewSegmentModalChange={setShowNewSegmentModal}
showNewSegmentModal={newSegmentModalVisible}
onNewSegmentModalChange={setNewSegmentModalVisible}
importStatus={importStatus}
archived={documentDetail?.archived}
/>
}
</div>
@@ -132,6 +175,12 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
onUpdate={metadataMutate}
/>}
</div>
<BatchModal
isShow={batchModalVisible}
onCancel={hideBatchModal}
onConfirm={runBatch}
docForm={documentDetail?.doc_form as DocForm}
/>
</div>
</DocumentContext.Provider>
)

View File

@@ -0,0 +1,84 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import cn from 'classnames'
import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
import { Loading02 } from '@/app/components/base/icons/src/vender/line/general'
import { AlertCircle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
import { CheckCircle } from '@/app/components/base/icons/src/vender/solid/general'
import Popover from '@/app/components/base/popover'
export type ISegmentAddProps = {
importStatus: ProcessStatus | string | undefined
clearProcessStatus: () => void
showNewSegmentModal: () => void
showBatchModal: () => void
}
export enum ProcessStatus {
WAITING = 'waiting',
PROCESSING = 'processing',
COMPLETED = 'completed',
ERROR = 'error',
}
const SegmentAdd: FC<ISegmentAddProps> = ({
importStatus,
clearProcessStatus,
showNewSegmentModal,
showBatchModal,
}) => {
const { t } = useTranslation()
if (importStatus) {
return (
<>
{(importStatus === ProcessStatus.WAITING || importStatus === ProcessStatus.PROCESSING) && (
<div className='relative overflow-hidden inline-flex items-center mr-2 px-3 py-[6px] text-blue-700 bg-[#F5F8FF] rounded-lg border border-black/5'>
{importStatus === ProcessStatus.WAITING && <div className='absolute left-0 top-0 w-3/12 h-full bg-[#D1E0FF] z-0'/>}
{importStatus === ProcessStatus.PROCESSING && <div className='absolute left-0 top-0 w-2/3 h-full bg-[#D1E0FF] z-0'/>}
<Loading02 className='animate-spin mr-2 w-4 h-4' />
<span className='font-medium text-[13px] leading-[18px] z-10'>{t('datasetDocuments.list.batchModal.processing')}</span>
</div>
)}
{importStatus === ProcessStatus.COMPLETED && (
<div className='inline-flex items-center mr-2 px-3 py-[6px] text-gray-700 bg-[#F6FEF9] rounded-lg border border-black/5'>
<CheckCircle className='mr-2 w-4 h-4 text-[#039855]' />
<span className='font-medium text-[13px] leading-[18px]'>{t('datasetDocuments.list.batchModal.completed')}</span>
<span className='pl-2 font-medium text-[13px] leading-[18px] text-[#155EEF] cursor-pointer' onClick={clearProcessStatus}>{t('datasetDocuments.list.batchModal.ok')}</span>
</div>
)}
{importStatus === ProcessStatus.ERROR && (
<div className='inline-flex items-center mr-2 px-3 py-[6px] text-red-600 bg-red-100 rounded-lg border border-black/5'>
<AlertCircle className='mr-2 w-4 h-4 text-[#D92D20]' />
<span className='font-medium text-[13px] leading-[18px]'>{t('datasetDocuments.list.batchModal.error')}</span>
<span className='pl-2 font-medium text-[13px] leading-[18px] text-[#155EEF] cursor-pointer' onClick={clearProcessStatus}>{t('datasetDocuments.list.batchModal.ok')}</span>
</div>
)}
</>
)
}
return (
<Popover
manualClose
trigger='click'
htmlContent={
<div className='w-full py-1'>
<div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={showNewSegmentModal}>{t('datasetDocuments.list.action.add')}</div>
<div className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' onClick={showBatchModal}>{t('datasetDocuments.list.action.batchAdd')}</div>
</div>
}
btnElement={
<div className='inline-flex items-center'>
<FilePlus02 className='w-4 h-4 text-gray-700' />
<span className='pl-1'>{t('datasetDocuments.list.action.addButton')}</span>
</div>
}
btnClassName={open => cn('mr-2 !py-[6px] !text-[13px] !leading-[18px] hover:bg-gray-50 border border-gray-200 hover:border-gray-300 hover:shadow-[0_1px_2px_rgba(16,24,40,0.05)]', open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')}
className='!w-[132px] h-fit !z-20 !translate-x-0 !left-0'
/>
)
}
export default React.memo(SegmentAdd)

View File

@@ -22,12 +22,12 @@ import type { IndicatorProps } from '@/app/components/header/indicator'
import Indicator from '@/app/components/header/indicator'
import { asyncRunSafe } from '@/utils'
import { formatNumber } from '@/utils/format'
import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument } from '@/service/datasets'
import { archiveDocument, deleteDocument, disableDocument, enableDocument, syncDocument, unArchiveDocument } from '@/service/datasets'
import NotionIcon from '@/app/components/base/notion-icon'
import ProgressBar from '@/app/components/base/progress-bar'
import { DataSourceType, type DocumentDisplayStatus, type SimpleDocumentDetail } from '@/models/datasets'
import type { CommonResponse } from '@/models/common'
import { FilePlus02 } from '@/app/components/base/icons/src/vender/line/files'
import { DotsHorizontal } from '@/app/components/base/icons/src/vender/line/general'
export const SettingsIcon: FC<{ className?: string }> = ({ className }) => {
return <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" className={className ?? ''}>
@@ -86,7 +86,7 @@ export const StatusItem: FC<{
</div>
}
type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync'
type OperationName = 'delete' | 'archive' | 'enable' | 'disable' | 'sync' | 'un_archive'
// operation action for list and detail
export const OperationAction: FC<{
@@ -101,8 +101,7 @@ export const OperationAction: FC<{
onUpdate: (operationName?: string) => void
scene?: 'list' | 'detail'
className?: string
showNewSegmentModal?: () => void
}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '', showNewSegmentModal }) => {
}> = ({ datasetId, detail, onUpdate, scene = 'list', className = '' }) => {
const { id, enabled = false, archived = false, data_source_type } = detail || {}
const [showModal, setShowModal] = useState(false)
const { notify } = useContext(ToastContext)
@@ -117,6 +116,9 @@ export const OperationAction: FC<{
case 'archive':
opApi = archiveDocument
break
case 'un_archive':
opApi = unArchiveDocument
break
case 'enable':
opApi = enableDocument
break
@@ -218,10 +220,72 @@ export const OperationAction: FC<{
<Divider className='!ml-4 !mr-2 !h-3' type='vertical' />
</>}
<Popover
htmlContent={<Operations />}
htmlContent={
<div className='w-full py-1'>
{!isListScene && <>
<div className='flex justify-between items-center mx-4 pt-2'>
<span className={cn(s.actionName, 'font-medium')}>
{!archived && enabled ? t('datasetDocuments.list.index.enable') : t('datasetDocuments.list.index.disable')}
</span>
<Tooltip
selector={`detail-switch-${id}`}
content={t('datasetDocuments.list.action.enableWarning') as string}
className='!font-semibold'
disabled={!archived}
>
<div>
<Switch
defaultValue={archived ? false : enabled}
onChange={v => !archived && onOperate(v ? 'enable' : 'disable')}
disabled={archived}
size='md'
/>
</div>
</Tooltip>
</div>
<div className='mx-4 pb-1 pt-0.5 text-xs text-gray-500'>
{!archived && enabled ? t('datasetDocuments.list.index.enableTip') : t('datasetDocuments.list.index.disableTip')}
</div>
<Divider />
</>}
{!archived && (
<>
<div className={s.actionItem} onClick={() => router.push(`/datasets/${datasetId}/documents/${detail.id}/settings`)}>
<SettingsIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.settings')}</span>
</div>
{data_source_type === 'notion_import' && (
<div className={s.actionItem} onClick={() => onOperate('sync')}>
<SyncIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.sync')}</span>
</div>
)}
<Divider className='my-1' />
</>
)}
{!archived && <div className={s.actionItem} onClick={() => onOperate('archive')}>
<ArchiveIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.archive')}</span>
</div>}
{archived && (
<div className={s.actionItem} onClick={() => onOperate('un_archive')}>
<ArchiveIcon />
<span className={s.actionName}>{t('datasetDocuments.list.action.unarchive')}</span>
</div>
)}
<div className={cn(s.actionItem, s.deleteActionItem, 'group')} onClick={() => setShowModal(true)}>
<TrashIcon className={'w-4 h-4 stroke-current text-gray-500 stroke-2 group-hover:text-red-500'} />
<span className={cn(s.actionName, 'group-hover:text-red-500')}>{t('datasetDocuments.list.action.delete')}</span>
</div>
</div>
}
trigger='click'
position='br'
btnElement={<div className={cn(s.actionIcon, s.commonIcon)} />}
btnElement={
<div className={cn(s.commonIcon)}>
<DotsHorizontal className='w-4 h-4 text-gray-700' />
</div>
}
btnClassName={open => cn(isListScene ? s.actionIconWrapperList : s.actionIconWrapperDetail, open ? '!bg-gray-100 !shadow-none' : '!bg-transparent')}
className={`!w-[200px] h-fit !z-20 ${className}`}
/>

View File

@@ -10,6 +10,10 @@ import { ToastContext } from '@/app/components/base/toast'
import Button from '@/app/components/base/button'
import { fetchDataDetail, updateDatasetSetting } from '@/service/datasets'
import type { DataSet } from '@/models/datasets'
import ModelSelector from '@/app/components/header/account-setting/model-page/model-selector'
import type { ProviderEnum } from '@/app/components/header/account-setting/model-page/declarations'
import { ModelType } from '@/app/components/header/account-setting/model-page/declarations'
import AccountSetting from '@/app/components/header/account-setting'
const rowClass = `
flex justify-between py-4
@@ -41,7 +45,7 @@ const Form = ({
const [description, setDescription] = useState(currentDataset?.description ?? '')
const [permission, setPermission] = useState(currentDataset?.permission)
const [indexMethod, setIndexMethod] = useState(currentDataset?.indexing_technique)
const [showSetAPIKeyModal, setShowSetAPIKeyModal] = useState(false)
const handleSave = async () => {
if (loading)
return
@@ -128,6 +132,32 @@ const Form = ({
/>
</div>
</div>
<div className={rowClass}>
<div className={labelClass}>
<div>{t('datasetSettings.form.embeddingModel')}</div>
</div>
<div className='w-[480px]'>
{currentDataset && (
<>
<div className='w-full h-9 rounded-lg bg-gray-100 opacity-60'>
<ModelSelector
readonly
value={{
providerName: currentDataset.embedding_model_provider as ProviderEnum,
modelName: currentDataset.embedding_model,
}}
modelType={ModelType.embeddings}
onChange={() => {}}
/>
</div>
<div className='mt-2 w-full text-xs leading-6 text-gray-500'>
{t('datasetSettings.form.embeddingModelTip')}
<span className='text-[#155eef] cursor-pointer' onClick={() => setShowSetAPIKeyModal(true)}>{t('datasetSettings.form.embeddingModelTipLink')}</span>
</div>
</>
)}
</div>
</div>
<div className={rowClass}>
<div className={labelClass} />
<div className='w-[480px]'>
@@ -140,6 +170,11 @@ const Form = ({
</Button>
</div>
</div>
{showSetAPIKeyModal && (
<AccountSetting activeTab="provider" onCancel={async () => {
setShowSetAPIKeyModal(false)
}} />
)}
</div>
)
}

View File

@@ -75,6 +75,7 @@ const translation = {
economicalTip: 'Use offline vector engines, keyword indexes, etc. to reduce accuracy without spending tokens',
QATitle: 'Segmenting in Question & Answer format',
QATip: 'Enable this option will consume more tokens',
QALanguage: 'Segment using',
emstimateCost: 'Estimation',
emstimateSegment: 'Estimated segments',
segmentCount: 'segments',

View File

@@ -75,6 +75,7 @@ const translation = {
economicalTip: '使用离线的向量引擎、关键词索引等方式,降低了准确度但无需花费 Token',
QATitle: '采用 Q&A 分段模式',
QATip: '开启后将会消耗额外的 token',
QALanguage: '分段使用',
emstimateCost: '执行嵌入预估消耗',
emstimateSegment: '预估分段数',
segmentCount: '段',

View File

@@ -17,8 +17,11 @@ const translation = {
action: {
uploadFile: 'Upload new file',
settings: 'Segment settings',
add: 'Add new segment',
addButton: 'Add segment',
add: 'Add a segment',
batchAdd: 'Batch add',
archive: 'Archive',
unarchive: 'Unarchive',
delete: 'Delete',
enableWarning: 'Archived file cannot be enabled',
sync: 'Sync',
@@ -53,6 +56,24 @@ const translation = {
title: 'Are you sure Delete?',
content: 'If you need to resume processing later, you will continue from where you left off',
},
batchModal: {
title: 'Batch add segments',
csvUploadTitle: 'Drag and drop your CSV file here, or ',
browse: 'browse',
tip: 'The CSV file must conform to the following structure:',
question: 'question',
answer: 'answer',
contentTitle: 'segment content',
content: 'content',
template: 'Download the template here',
cancel: 'Cancel',
run: 'Run Batch',
runError: 'Run batch failed',
processing: 'In batch processing',
completed: 'Import completed',
error: 'Import Error',
ok: 'OK',
},
},
metadata: {
title: 'Metadata',
@@ -321,6 +342,7 @@ const translation = {
contentEmpty: 'Content can not be empty',
newTextSegment: 'New Text Segment',
newQaSegment: 'New Q&A Segment',
delete: 'Delete this segment ?',
},
}

View File

@@ -17,8 +17,11 @@ const translation = {
action: {
uploadFile: '上传新文件',
settings: '分段设置',
addButton: '添加分段',
add: '添加新分段',
batchAdd: '批量添加',
archive: '归档',
unarchive: '撤销归档',
delete: '删除',
enableWarning: '归档的文件无法启用',
sync: '同步',
@@ -53,6 +56,24 @@ const translation = {
title: '确定删除吗?',
content: '如果您需要稍后恢复处理,您将从您离开的地方继续',
},
batchModal: {
title: '批量添加分段',
csvUploadTitle: '将您的 CSV 文件拖放到此处,或',
browse: '选择文件',
tip: 'CSV 文件必须符合以下结构:',
question: '问题',
answer: '回答',
contentTitle: '分段内容',
content: '内容',
template: '下载模板',
cancel: '取消',
run: '导入',
runError: '批量导入失败',
processing: '批量处理中',
completed: '导入完成',
error: '导入出错',
ok: '确定',
},
},
metadata: {
title: '元数据',
@@ -320,6 +341,7 @@ const translation = {
contentEmpty: '内容不能为空',
newTextSegment: '新文本分段',
newQaSegment: '新问答分段',
delete: '删除这个分段?',
},
}

View File

@@ -15,6 +15,9 @@ const translation = {
indexMethodHighQualityTip: 'Call OpenAI\'s embedding interface for processing to provide higher accuracy when users query.',
indexMethodEconomy: 'Economical',
indexMethodEconomyTip: 'Use offline vector engines, keyword indexes, etc. to reduce accuracy without spending tokens',
embeddingModel: 'Embedding Model',
embeddingModelTip: 'Change the embedded model, please go to ',
embeddingModelTipLink: 'Settings',
save: 'Save',
},
}

View File

@@ -15,6 +15,9 @@ const translation = {
indexMethodHighQualityTip: '调用 OpenAI 的嵌入接口进行处理,以在用户查询时提供更高的准确度',
indexMethodEconomy: '经济',
indexMethodEconomyTip: '使用离线的向量引擎、关键词索引等方式,降低了准确度但无需花费 Token',
embeddingModel: 'Embedding 模型',
embeddingModelTip: '修改 Embedding 模型,请去',
embeddingModelTipLink: '设置',
save: '保存',
},
}

View File

@@ -16,6 +16,8 @@ const translation = {
intro4: 'or it ',
intro5: 'can be created',
intro6: ' as a standalone ChatGPT index plug-in to publish',
unavailable: 'Unavailable',
unavailableTip: 'Embedding model is not available, the default embedding model needs to be configured',
}
export default translation

View File

@@ -16,6 +16,8 @@ const translation = {
intro4: '或可以',
intro5: '创建',
intro6: '为独立的 ChatGPT 插件发布使用',
unavailable: '不可用',
unavailableTip: '由于 embedding 模型不可用,需要配置默认 embedding 模型',
}
export default translation

View File

@@ -22,6 +22,9 @@ export type DataSet = {
app_count: number
document_count: number
word_count: number
embedding_model: string
embedding_model_provider: string
embedding_available: boolean
}
export type CustomFile = File & {
@@ -184,6 +187,7 @@ export type CreateDocumentReq = {
original_document_id?: string
indexing_technique?: string
doc_form: 'text_model' | 'qa_model'
doc_language: string
data_source: DataSource
process_rule: ProcessRule
}
@@ -390,3 +394,8 @@ export type SegmentUpdator = {
answer?: string
keywords?: string[]
}
export enum DocForm {
TEXT = 'text_model',
QA = 'qa_model',
}

View File

@@ -118,6 +118,10 @@ export const archiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ dataset
return patch(`/datasets/${datasetId}/documents/${documentId}/status/archive`) as Promise<CommonResponse>
}
export const unArchiveDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
return patch(`/datasets/${datasetId}/documents/${documentId}/status/un_archive`) as Promise<CommonResponse>
}
export const enableDocument: Fetcher<CommonResponse, CommonDocReq> = ({ datasetId, documentId }) => {
return patch(`/datasets/${datasetId}/documents/${documentId}/status/enable`) as Promise<CommonResponse>
}
@@ -138,10 +142,6 @@ export const modifyDocMetadata: Fetcher<CommonResponse, CommonDocReq & { body: {
return put(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body }) as Promise<CommonResponse>
}
export const getDatasetIndexingStatus: Fetcher<{ data: IndexingStatusResponse[] }, string> = (datasetId) => {
return get(`/datasets/${datasetId}/indexing-status`) as Promise<{ data: IndexingStatusResponse[] }>
}
// apis for segments in a document
export const fetchSegments: Fetcher<SegmentsResponse, CommonDocReq & { params: SegmentsQuery }> = ({ datasetId, documentId, params }) => {
@@ -164,6 +164,18 @@ export const addSegment: Fetcher<{ data: SegmentDetailModel; doc_form: string },
return post(`/datasets/${datasetId}/documents/${documentId}/segment`, { body }) as Promise<{ data: SegmentDetailModel; doc_form: string }>
}
export const deleteSegment: Fetcher<CommonResponse, { datasetId: string; documentId: string; segmentId: string }> = ({ datasetId, documentId, segmentId }) => {
return del(`/datasets/${datasetId}/documents/${documentId}/segments/${segmentId}`) as Promise<CommonResponse>
}
export const segmentBatchImport: Fetcher<{ job_id: string; job_status: string }, { url: string; body: FormData }> = ({ url, body }) => {
return post(url, { body }, { bodyStringify: false, deleteContentType: true }) as Promise<{ job_id: string; job_status: string }>
}
export const checkSegmentBatchImportProgress: Fetcher<{ job_id: string; job_status: string }, { jobID: string }> = ({ jobID }) => {
return get(`/datasets/batch_import_status/${jobID}`) as Promise<{ job_id: string; job_status: string }>
}
// hit testing
export const hitTesting: Fetcher<HitTestingResponse, { datasetId: string; queryText: string }> = ({ datasetId, queryText }) => {
return post(`/datasets/${datasetId}/hit-testing`, { body: { query: queryText } }) as Promise<HitTestingResponse>