mirror of
https://github.com/langgenius/dify.git
synced 2026-01-06 06:26:00 +00:00
Refactor: dataset creation components and implement Firecrawl functionality
This commit is contained in:
twwu
@@ -6,7 +6,7 @@ import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } f
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
import NotionConnector from '../notion-connector'
|
||||
import { usePreImportNotionPages } from '@/service/knowledge/use-import'
|
||||
import Header from './header'
|
||||
import Header from '../../datasets/create/website/base/header'
|
||||
|
||||
type NotionPageSelectorProps = {
|
||||
value?: string[]
|
||||
@@ -99,7 +99,11 @@ const NotionPageSelector = ({
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
isInPipeline={isInPipeline}
|
||||
handleConfigureNotion={handleConfigureNotion}
|
||||
onClickConfiguration={handleConfigureNotion}
|
||||
title={'Choose notion pages'}
|
||||
buttonText={'Configure Notion'}
|
||||
docTitle={'Notion docs'}
|
||||
docLink={'https://www.notion.so/docs'}
|
||||
/>
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
import React from 'react'
|
||||
import Divider from '../../divider'
|
||||
import Button from '../../button'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import Button from '@/app/components/base/button'
|
||||
import cn from '@/utils/classnames'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
|
||||
type HeaderProps = {
|
||||
isInPipeline?: boolean
|
||||
handleConfigureNotion: () => void
|
||||
onClickConfiguration: () => void
|
||||
title: string
|
||||
buttonText: string
|
||||
docTitle: string
|
||||
docLink: string
|
||||
}
|
||||
|
||||
const Header = ({
|
||||
isInPipeline = false,
|
||||
handleConfigureNotion,
|
||||
onClickConfiguration,
|
||||
title,
|
||||
buttonText,
|
||||
docTitle,
|
||||
docLink,
|
||||
}: HeaderProps) => {
|
||||
return (
|
||||
<div className='flex items-center gap-x-2'>
|
||||
@@ -20,7 +28,7 @@ const Header = ({
|
||||
'text-text-secondary',
|
||||
isInPipeline ? 'system-sm-semibold' : 'system-md-semibold',
|
||||
)}>
|
||||
Choose notion pages
|
||||
{title}
|
||||
</div>
|
||||
<Divider type='vertical' className='mx-1 h-3.5' />
|
||||
<Button
|
||||
@@ -30,23 +38,23 @@ const Header = ({
|
||||
>
|
||||
<RiEqualizer2Line
|
||||
className='h-4 w-4'
|
||||
onClick={handleConfigureNotion}
|
||||
onClick={onClickConfiguration}
|
||||
/>
|
||||
{!isInPipeline && (
|
||||
<span className='system-xs-medium'>
|
||||
Configure Notion
|
||||
{buttonText}
|
||||
</span>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
className='system-xs-medium flex items-center gap-x-1 text-text-accent'
|
||||
href='https://www.notion.so/docs'
|
||||
className='system-xs-medium flex shrink-0 items-center gap-x-1 text-text-accent'
|
||||
href={docLink}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
>
|
||||
<RiBookOpenLine className='size-3.5' />
|
||||
<span>Notion docs</span>
|
||||
<span>{docTitle}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onSetting: () => void
|
||||
}
|
||||
|
||||
const Header: FC<Props> = ({
|
||||
onSetting,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex h-6 items-center justify-between'>
|
||||
<div className='flex items-center'>
|
||||
<div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
|
||||
<div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
|
||||
<Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
|
||||
<RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
|
||||
<span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
|
||||
{t(`${I18N_PREFIX}.configureFirecrawl`)}
|
||||
</span>
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
href='https://docs.firecrawl.dev/introduction'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
|
||||
>
|
||||
<RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
|
||||
<span>{t(`${I18N_PREFIX}.firecrawlDoc`)}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Header)
|
||||
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
|
||||
import CrawledResult from '../base/crawled-result'
|
||||
import Crawling from '../base/crawling'
|
||||
import ErrorMessage from '../base/error-message'
|
||||
import Header from './header'
|
||||
import Options from './options'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
|
||||
import { sleep } from '@/utils'
|
||||
import Header from '../base/header'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'common.errorMsg'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
@@ -48,7 +48,7 @@ const FireCrawl: FC<Props> = ({
|
||||
if (step !== Step.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
const { setShowAccountSettingModal } = useModalContext()
|
||||
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
|
||||
const handleSetting = useCallback(() => {
|
||||
setShowAccountSettingModal({
|
||||
payload: 'data-source',
|
||||
@@ -131,7 +131,7 @@ const FireCrawl: FC<Props> = ({
|
||||
},
|
||||
}
|
||||
}
|
||||
}, [crawlOptions.limit])
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
const handleRun = useCallback(async (url: string) => {
|
||||
const { isValid, errorMsg } = checkValid(url)
|
||||
@@ -173,11 +173,17 @@ const FireCrawl: FC<Props> = ({
|
||||
finally {
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
|
||||
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header onSetting={handleSetting} />
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.firecrawlTitle`)}
|
||||
buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
|
||||
docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
|
||||
docLink={'https://docs.firecrawl.dev/introduction'}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
title: string
|
||||
errorMsg?: string
|
||||
}
|
||||
|
||||
const ErrorMessage: FC<Props> = ({
|
||||
className,
|
||||
title,
|
||||
errorMsg,
|
||||
}) => {
|
||||
return (
|
||||
<div className={cn(className, 'border-t border-gray-200 bg-[#FFFAEB] px-4 py-2')}>
|
||||
<div className='flex h-5 items-center'>
|
||||
<AlertTriangle className='mr-2 h-4 w-4 text-[#F79009]' />
|
||||
<div className='text-sm font-medium text-[#DC6803]'>{title}</div>
|
||||
</div>
|
||||
{errorMsg && (
|
||||
<div className='mt-1 pl-6 text-xs font-normal leading-[18px] text-gray-700'>{errorMsg}</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(ErrorMessage)
|
||||
@@ -1,54 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import Input from './input'
|
||||
import cn from '@/utils/classnames'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
label: string
|
||||
labelClassName?: string
|
||||
value: string | number
|
||||
onChange: (value: string | number) => void
|
||||
isRequired?: boolean
|
||||
placeholder?: string
|
||||
isNumber?: boolean
|
||||
tooltip?: string
|
||||
}
|
||||
|
||||
const Field: FC<Props> = ({
|
||||
className,
|
||||
label,
|
||||
labelClassName,
|
||||
value,
|
||||
onChange,
|
||||
isRequired = false,
|
||||
placeholder = '',
|
||||
isNumber = false,
|
||||
tooltip,
|
||||
}) => {
|
||||
return (
|
||||
<div className={cn(className)}>
|
||||
<div className='flex py-[7px]'>
|
||||
<div className={cn(labelClassName, 'flex h-[18px] items-center text-[13px] font-medium text-gray-900')}>{label} </div>
|
||||
{isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
|
||||
{tooltip && (
|
||||
<Tooltip
|
||||
popupContent={
|
||||
<div className='w-[200px]'>{tooltip}</div>
|
||||
}
|
||||
triggerClassName='ml-0.5 w-4 h-4'
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<Input
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
placeholder={placeholder}
|
||||
isNumber={isNumber}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Field)
|
||||
@@ -1,58 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback } from 'react'
|
||||
|
||||
type Props = {
|
||||
value: string | number
|
||||
onChange: (value: string | number) => void
|
||||
placeholder?: string
|
||||
isNumber?: boolean
|
||||
}
|
||||
|
||||
const MIN_VALUE = 0
|
||||
|
||||
const Input: FC<Props> = ({
|
||||
value,
|
||||
onChange,
|
||||
placeholder = '',
|
||||
isNumber = false,
|
||||
}) => {
|
||||
const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const value = e.target.value
|
||||
if (isNumber) {
|
||||
let numberValue = Number.parseInt(value, 10) // integer only
|
||||
if (isNaN(numberValue)) {
|
||||
onChange('')
|
||||
return
|
||||
}
|
||||
if (numberValue < MIN_VALUE)
|
||||
numberValue = MIN_VALUE
|
||||
|
||||
onChange(numberValue)
|
||||
return
|
||||
}
|
||||
onChange(value)
|
||||
}, [isNumber, onChange])
|
||||
|
||||
const otherOption = (() => {
|
||||
if (isNumber) {
|
||||
return {
|
||||
min: MIN_VALUE,
|
||||
}
|
||||
}
|
||||
return {
|
||||
|
||||
}
|
||||
})()
|
||||
return (
|
||||
<input
|
||||
type={isNumber ? 'number' : 'text'}
|
||||
{...otherOption}
|
||||
value={value}
|
||||
onChange={handleChange}
|
||||
className='flex h-9 w-full rounded-lg bg-gray-100 px-2 py-1 text-xs leading-normal caret-primary-600 placeholder:text-gray-400 hover:bg-gray-100 focus:bg-gray-50 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none'
|
||||
placeholder={placeholder}
|
||||
/>
|
||||
)
|
||||
}
|
||||
export default React.memo(Input)
|
||||
@@ -1,55 +0,0 @@
|
||||
'use client'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import type { FC } from 'react'
|
||||
import React, { useEffect } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
children: React.ReactNode
|
||||
controlFoldOptions?: number
|
||||
}
|
||||
|
||||
const OptionsWrap: FC<Props> = ({
|
||||
className = '',
|
||||
children,
|
||||
controlFoldOptions,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const [fold, {
|
||||
toggle: foldToggle,
|
||||
setTrue: foldHide,
|
||||
}] = useBoolean(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (controlFoldOptions)
|
||||
foldHide()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [controlFoldOptions])
|
||||
return (
|
||||
<div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
|
||||
<div
|
||||
className='flex h-[26px] cursor-pointer select-none items-center justify-between py-1'
|
||||
onClick={foldToggle}
|
||||
>
|
||||
<div className='flex items-center text-gray-700'>
|
||||
<Settings04 className='mr-1 h-4 w-4' />
|
||||
<div className='text-[13px] font-semibold uppercase text-gray-800'>{t(`${I18N_PREFIX}.options`)}</div>
|
||||
</div>
|
||||
<ChevronRight className={cn(!fold && 'rotate-90', 'h-4 w-4 text-gray-500')} />
|
||||
</div>
|
||||
{!fold && (
|
||||
<div className='mb-4'>
|
||||
{children}
|
||||
</div>
|
||||
)}
|
||||
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(OptionsWrap)
|
||||
@@ -1,48 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Input from './input'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
isRunning: boolean
|
||||
onRun: (url: string) => void
|
||||
}
|
||||
|
||||
const UrlInput: FC<Props> = ({
|
||||
isRunning,
|
||||
onRun,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [url, setUrl] = useState('')
|
||||
const handleUrlChange = useCallback((url: string | number) => {
|
||||
setUrl(url as string)
|
||||
}, [])
|
||||
const handleOnRun = useCallback(() => {
|
||||
if (isRunning)
|
||||
return
|
||||
onRun(url)
|
||||
}, [isRunning, onRun, url])
|
||||
|
||||
return (
|
||||
<div className='flex items-center justify-between'>
|
||||
<Input
|
||||
value={url}
|
||||
onChange={handleUrlChange}
|
||||
placeholder='https://docs.dify.ai'
|
||||
/>
|
||||
<Button
|
||||
variant='primary'
|
||||
onClick={handleOnRun}
|
||||
className='ml-2'
|
||||
loading={isRunning}
|
||||
>
|
||||
{!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(UrlInput)
|
||||
@@ -1,43 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onSetting: () => void
|
||||
}
|
||||
|
||||
const Header: FC<Props> = ({
|
||||
onSetting,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex h-6 items-center justify-between'>
|
||||
<div className='flex items-center'>
|
||||
<div className='text-sm font-semibold text-text-secondary'>{t(`${I18N_PREFIX}.jinaReaderTitle`)}</div>
|
||||
<div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
|
||||
<Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
|
||||
<RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
|
||||
<span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
|
||||
{t(`${I18N_PREFIX}.configureJinaReader`)}
|
||||
</span>
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
href='https://jina.ai/reader'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
|
||||
>
|
||||
<RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
|
||||
<span>{t(`${I18N_PREFIX}.jinaReaderDoc`)}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Header)
|
||||
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
|
||||
import CrawledResult from '../base/crawled-result'
|
||||
import Crawling from '../base/crawling'
|
||||
import ErrorMessage from '../base/error-message'
|
||||
import Header from './header'
|
||||
import Options from './options'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
|
||||
import { sleep } from '@/utils'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import Header from '../base/header'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'common.errorMsg'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
@@ -190,7 +190,13 @@ const JinaReader: FC<Props> = ({
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header onSetting={handleSetting} />
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.jinaReaderTitle`)}
|
||||
buttonText={t(`${I18N_PREFIX}.configureJinaReader`)}
|
||||
docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`)}
|
||||
docLink={'https://jina.ai/reader'}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onSetting: () => void
|
||||
}
|
||||
|
||||
const Header: FC<Props> = ({
|
||||
onSetting,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex h-6 items-center justify-between'>
|
||||
<div className='flex items-center'>
|
||||
<div className='text-base font-medium text-text-secondary'>{t(`${I18N_PREFIX}.watercrawlTitle`)}</div>
|
||||
<div className='ml-2 mr-2 h-3.5 w-px bg-divider-regular' />
|
||||
<Button className='flex h-6 items-center gap-x-[1px] px-1.5' onClick={onSetting}>
|
||||
<RiEqualizer2Line className='h-3.5 w-3.5 text-components-button-secondary-text' />
|
||||
<span className='px-[3px] text-xs font-medium text-components-button-secondary-text'>
|
||||
{t(`${I18N_PREFIX}.configureWatercrawl`)}
|
||||
</span>
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
href='https://docs.watercrawl.dev/'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
className='inline-flex items-center gap-x-1 text-xs font-medium text-text-accent'
|
||||
>
|
||||
<RiBookOpenLine className='h-3.5 w-3.5 text-text-accent' />
|
||||
<span>{t(`${I18N_PREFIX}.watercrawlDoc`)}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Header)
|
||||
@@ -7,13 +7,13 @@ import OptionsWrap from '../base/options-wrap'
|
||||
import CrawledResult from '../base/crawled-result'
|
||||
import Crawling from '../base/crawling'
|
||||
import ErrorMessage from '../base/error-message'
|
||||
import Header from './header'
|
||||
import Options from './options'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets'
|
||||
import { sleep } from '@/utils'
|
||||
import Header from '../base/header'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'common.errorMsg'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
@@ -177,7 +177,13 @@ const WaterCrawl: FC<Props> = ({
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header onSetting={handleSetting} />
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.watercrawlTitle`)}
|
||||
buttonText={t(`${I18N_PREFIX}.configureWatercrawl`)}
|
||||
docTitle={t(`${I18N_PREFIX}.watercrawlDoc`)}
|
||||
docLink={'https://docs.watercrawl.dev/'}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0'>
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
|
||||
@@ -35,12 +35,13 @@ const InputFieldForm = ({
|
||||
const result = schema.safeParse(value)
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues
|
||||
const firstIssue = issues[0].message
|
||||
const firstIssue = issues[0]
|
||||
const errorMessage = `"${firstIssue.path.join('.')}" ${firstIssue.message}`
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: firstIssue,
|
||||
message: errorMessage,
|
||||
})
|
||||
return firstIssue
|
||||
return errorMessage
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import type { CrawlOptions } from '@/models/datasets'
|
||||
|
||||
export const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
|
||||
crawl_sub_pages: true,
|
||||
only_main_content: true,
|
||||
includes: '',
|
||||
excludes: '',
|
||||
limit: 10,
|
||||
max_depth: '',
|
||||
use_sitemap: true,
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
|
||||
type Props = {
|
||||
type CheckboxWithLabelProps = {
|
||||
className?: string
|
||||
isChecked: boolean
|
||||
onChange: (isChecked: boolean) => void
|
||||
@@ -14,18 +13,18 @@ type Props = {
|
||||
tooltip?: string
|
||||
}
|
||||
|
||||
const CheckboxWithLabel: FC<Props> = ({
|
||||
const CheckboxWithLabel = ({
|
||||
className = '',
|
||||
isChecked,
|
||||
onChange,
|
||||
label,
|
||||
labelClassName,
|
||||
tooltip,
|
||||
}) => {
|
||||
}: CheckboxWithLabelProps) => {
|
||||
return (
|
||||
<label className={cn(className, 'flex h-7 items-center space-x-2')}>
|
||||
<label className={cn('flex items-center space-x-2', className)}>
|
||||
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
|
||||
<div className={cn('text-sm font-normal text-gray-800', labelClassName)}>{label}</div>
|
||||
<div className={cn('system-sm-medium text-text-secondary', labelClassName)}>{label}</div>
|
||||
{tooltip && (
|
||||
<Tooltip
|
||||
popupContent={
|
||||
@@ -0,0 +1,45 @@
|
||||
'use client'
|
||||
import React, { useCallback } from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
|
||||
type CrawledResultItemProps = {
|
||||
payload: CrawlResultItemType
|
||||
isChecked: boolean
|
||||
onCheckChange: (checked: boolean) => void
|
||||
}
|
||||
|
||||
const CrawledResultItem = ({
|
||||
payload,
|
||||
isChecked,
|
||||
onCheckChange,
|
||||
}: CrawledResultItemProps) => {
|
||||
const handleCheckChange = useCallback(() => {
|
||||
onCheckChange(!isChecked)
|
||||
}, [isChecked, onCheckChange])
|
||||
return (
|
||||
<div className={cn('group flex cursor-pointer gap-x-2 rounded-lg p-2 hover:bg-state-base-hover')}>
|
||||
<Checkbox
|
||||
className='shrink-0'
|
||||
checked={isChecked}
|
||||
onCheck={handleCheckChange}
|
||||
/>
|
||||
<div className='flex min-w-0 grow flex-col gap-y-0.5'>
|
||||
<div
|
||||
className='system-sm-medium truncate text-text-secondary'
|
||||
title={payload.title}
|
||||
>
|
||||
{payload.title}
|
||||
</div>
|
||||
<div
|
||||
className='system-xs-regular truncate text-text-tertiary'
|
||||
title={payload.source_url}
|
||||
>
|
||||
{payload.source_url}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResultItem)
|
||||
@@ -0,0 +1,77 @@
|
||||
'use client'
|
||||
import React, { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import CheckboxWithLabel from './checkbox-with-label'
|
||||
import CrawledResultItem from './crawled-result-item'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type CrawledResultProps = {
|
||||
className?: string
|
||||
list: CrawlResultItem[]
|
||||
checkedList: CrawlResultItem[]
|
||||
onSelectedChange: (selected: CrawlResultItem[]) => void
|
||||
usedTime: number
|
||||
}
|
||||
|
||||
const CrawledResult = ({
|
||||
className = '',
|
||||
list,
|
||||
checkedList,
|
||||
onSelectedChange,
|
||||
usedTime,
|
||||
}: CrawledResultProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const isCheckAll = checkedList.length === list.length
|
||||
|
||||
const handleCheckedAll = useCallback(() => {
|
||||
if (!isCheckAll)
|
||||
onSelectedChange(list)
|
||||
|
||||
else
|
||||
onSelectedChange([])
|
||||
}, [isCheckAll, list, onSelectedChange])
|
||||
|
||||
const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
|
||||
return (checked: boolean) => {
|
||||
if (checked)
|
||||
onSelectedChange([...checkedList, item])
|
||||
|
||||
else
|
||||
onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
|
||||
}
|
||||
}, [checkedList, onSelectedChange])
|
||||
|
||||
return (
|
||||
<div className={cn('flex flex-col gap-y-2', className)}>
|
||||
<div className='system-sm-medium pt-2 text-text-primary'>
|
||||
{t(`${I18N_PREFIX}.scrapTimeInfo`, {
|
||||
total: list.length,
|
||||
time: usedTime.toFixed(1),
|
||||
})}
|
||||
</div>
|
||||
<div className='rounded-xl border border-components-panel-border bg-components-panel-bg'>
|
||||
<div className='flex items-center px-4 py-2'>
|
||||
<CheckboxWithLabel
|
||||
isChecked={isCheckAll}
|
||||
onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
|
||||
/>
|
||||
</div>
|
||||
<div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
|
||||
{list.map(item => (
|
||||
<CrawledResultItem
|
||||
key={item.source_url}
|
||||
payload={item}
|
||||
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
|
||||
onCheckChange={handleItemCheckChange(item)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResult)
|
||||
@@ -0,0 +1,89 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
|
||||
type CrawlingProps = {
|
||||
className?: string
|
||||
crawledNum: number
|
||||
totalNum: number
|
||||
}
|
||||
|
||||
type BlockProps = {
|
||||
className?: string
|
||||
}
|
||||
|
||||
type ItemProps = {
|
||||
firstLineWidth: string
|
||||
secondLineWidth: string
|
||||
}
|
||||
|
||||
const Block = React.memo(({
|
||||
className,
|
||||
}: BlockProps) => {
|
||||
return <div className={cn('bg-text-quaternary opacity-20', className)} />
|
||||
})
|
||||
|
||||
const Item = React.memo(({
|
||||
firstLineWidth,
|
||||
secondLineWidth,
|
||||
}: ItemProps) => {
|
||||
return (
|
||||
<div className='flex gap-x-2 px-2 py-[5px]'>
|
||||
<div className='py-0.5'>
|
||||
<Block className='size-4 rounded-[4px]' />
|
||||
</div>
|
||||
<div className='flex grow flex-col'>
|
||||
<div className='flex h-5 w-full items-center'>
|
||||
<Block className={cn('h-2.5 rounded-sm', firstLineWidth)} />
|
||||
</div>
|
||||
<div className='flex h-[18px] w-full items-center'>
|
||||
<Block className={cn('h-1.5 rounded-sm', secondLineWidth)} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})
|
||||
|
||||
const Crawling = ({
|
||||
className = '',
|
||||
crawledNum,
|
||||
totalNum,
|
||||
}: CrawlingProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const itemsConfig = [{
|
||||
firstLineWidth: 'w-[35%]',
|
||||
secondLineWidth: 'w-[50%]',
|
||||
}, {
|
||||
firstLineWidth: 'w-[40%]',
|
||||
secondLineWidth: 'w-[45%]',
|
||||
}, {
|
||||
firstLineWidth: 'w-[30%]',
|
||||
secondLineWidth: 'w-[36%]',
|
||||
}]
|
||||
|
||||
return (
|
||||
<div className={cn('mt-2 flex flex-col gap-y-2 pt-2', className)}>
|
||||
<div className='system-sm-medium text-text-primary'>
|
||||
{t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
|
||||
<div className='flex items-center gap-x-2 px-4 py-2'>
|
||||
<Block className='size-4 rounded-[4px]' />
|
||||
<Block className='h-2.5 w-14 rounded-sm' />
|
||||
</div>
|
||||
<div className='flex flex-col gap-px border-t border-divider-subtle bg-background-default-subtle p-2'>
|
||||
{itemsConfig.map((item, index) => (
|
||||
<Item
|
||||
key={index}
|
||||
firstLineWidth={item.firstLineWidth}
|
||||
secondLineWidth={item.secondLineWidth}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Crawling)
|
||||
@@ -0,0 +1,34 @@
|
||||
import React from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import { RiErrorWarningFill } from '@remixicon/react'
|
||||
|
||||
type ErrorMessageProps = {
|
||||
className?: string
|
||||
title: string
|
||||
errorMsg?: string
|
||||
}
|
||||
|
||||
const ErrorMessage = ({
|
||||
className,
|
||||
title,
|
||||
errorMsg,
|
||||
}: ErrorMessageProps) => {
|
||||
return (
|
||||
// eslint-disable-next-line tailwindcss/migration-from-tailwind-2
|
||||
<div className={cn(
|
||||
'flex gap-x-0.5 rounded-xl border-[0.5px] border-components-panel-border bg-opacity-40 bg-toast-error-bg p-2 shadow-xs shadow-shadow-shadow-3',
|
||||
className,
|
||||
)}>
|
||||
<div className='flex size-6 items-center justify-center'>
|
||||
<RiErrorWarningFill className='h-4 w-4 text-text-destructive' />
|
||||
</div>
|
||||
<div className='flex flex-col gap-y-0.5 py-1'>
|
||||
<div className='system-xs-medium text-text-primary'>{title}</div>
|
||||
{errorMsg && (
|
||||
<div className='system-xs-regular text-text-secondary'>{errorMsg}</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(ErrorMessage)
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
|
||||
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { FormData } from './options'
|
||||
import { z } from 'zod'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'common.errorMsg'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export const useConfigurations = () => {
|
||||
const { t } = useTranslation()
|
||||
const configurations: BaseConfiguration<FormData>[] = [
|
||||
{
|
||||
type: BaseFieldType.textInput,
|
||||
variable: 'url',
|
||||
label: 'URL',
|
||||
required: true,
|
||||
showConditions: [],
|
||||
placeholder: 'https://docs.dify.ai',
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.numberInput,
|
||||
variable: 'limit',
|
||||
label: t(`${I18N_PREFIX}.limit`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.numberInput,
|
||||
variable: 'max_depth',
|
||||
label: t(`${I18N_PREFIX}.maxDepth`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.textInput,
|
||||
variable: 'excludes',
|
||||
label: t(`${I18N_PREFIX}.excludePaths`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
placeholder: 'blog/*, /about/*',
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.textInput,
|
||||
variable: 'includes',
|
||||
label: t(`${I18N_PREFIX}.includeOnlyPaths`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
placeholder: 'articles/*',
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.checkbox,
|
||||
variable: 'crawl_sub_pages',
|
||||
label: t(`${I18N_PREFIX}.crawlSubPage`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
},
|
||||
{
|
||||
type: BaseFieldType.checkbox,
|
||||
variable: 'only_main_content',
|
||||
label: t(`${I18N_PREFIX}.extractOnlyMainContent`),
|
||||
required: false,
|
||||
showConditions: [],
|
||||
},
|
||||
]
|
||||
|
||||
return configurations
|
||||
}
|
||||
|
||||
export const useSchema = () => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const Schema = z.object({
|
||||
url: z.string().nonempty({
|
||||
message: t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
field: 'url',
|
||||
}),
|
||||
}).regex(/^https?:\/\//, {
|
||||
message: t(`${ERROR_I18N_PREFIX}.urlError`),
|
||||
}),
|
||||
limit: z.number().positive({
|
||||
message: t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
field: t(`${I18N_PREFIX}.limit`),
|
||||
}),
|
||||
}).int(),
|
||||
}).passthrough()
|
||||
|
||||
return Schema
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
|
||||
import { sleep } from '@/utils'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import type { FormData } from './options'
|
||||
import Options from './options'
|
||||
import { useConfigurations } from './hooks'
|
||||
import Crawling from '../base/crawling'
|
||||
import ErrorMessage from '../base/error-message'
|
||||
import CrawledResult from '../base/crawled-result'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
onJobIdChange: (jobId: string) => void
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
enum Step {
|
||||
init = 'init',
|
||||
running = 'running',
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
const FireCrawl: FC<Props> = ({
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const configurations = useConfigurations()
|
||||
useEffect(() => {
|
||||
if (step !== Step.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
|
||||
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
|
||||
const handleSetting = useCallback(() => {
|
||||
setShowAccountSettingModal({
|
||||
payload: 'data-source',
|
||||
})
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
const [crawlResult, setCrawlResult] = useState<{
|
||||
current: number
|
||||
total: number
|
||||
data: CrawlResultItem[]
|
||||
time_consuming: number | string
|
||||
} | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string) => {
|
||||
try {
|
||||
const res = await checkFirecrawlTaskStatus(jobId) as any
|
||||
if (res.status === 'completed') {
|
||||
return {
|
||||
isError: false,
|
||||
data: {
|
||||
...res,
|
||||
total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
|
||||
},
|
||||
}
|
||||
}
|
||||
if (res.status === 'error' || !res.status) {
|
||||
// can't get the error message from the firecrawl api
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage: res.message,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
// update the progress
|
||||
setCrawlResult({
|
||||
...res,
|
||||
total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
|
||||
})
|
||||
onCheckedCrawlResultChange(res.data || []) // default select the crawl result
|
||||
await sleep(2500)
|
||||
return await waitForCrawlFinished(jobId)
|
||||
}
|
||||
catch (e: any) {
|
||||
const errorBody = await e.json()
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage: errorBody.message,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
const handleRun = useCallback(async (value: FormData) => {
|
||||
const { url, ...crawlOptions } = value
|
||||
onCrawlOptionsChange(crawlOptions)
|
||||
setStep(Step.running)
|
||||
try {
|
||||
const passToServerCrawlOptions: any = {
|
||||
...crawlOptions,
|
||||
}
|
||||
if (crawlOptions.max_depth === '')
|
||||
delete passToServerCrawlOptions.max_depth
|
||||
|
||||
const res = await createFirecrawlTask({
|
||||
url,
|
||||
options: passToServerCrawlOptions,
|
||||
}) as any
|
||||
const jobId = res.job_id
|
||||
onJobIdChange(jobId)
|
||||
const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
|
||||
if (isError) {
|
||||
setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
|
||||
}
|
||||
else {
|
||||
setCrawlResult(data)
|
||||
onCheckedCrawlResultChange(data.data || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
|
||||
console.log(e)
|
||||
}
|
||||
finally {
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [onCrawlOptionsChange, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header
|
||||
isInPipeline
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.firecrawlTitle`)}
|
||||
buttonText={t(`${I18N_PREFIX}.configureFirecrawl`)}
|
||||
docTitle={t(`${I18N_PREFIX}.firecrawlDoc`)}
|
||||
docLink={'https://docs.firecrawl.dev/introduction'}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<Options
|
||||
initialData={{
|
||||
...crawlOptions,
|
||||
url: '',
|
||||
}}
|
||||
configurations={configurations}
|
||||
isRunning={isRunning}
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
onSubmit={(value) => {
|
||||
handleRun(value)
|
||||
console.log('submit')
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{!isInit && (
|
||||
<div className='relative'>
|
||||
{isRunning && (
|
||||
<Crawling
|
||||
crawledNum={crawlResult?.current || 0}
|
||||
totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage
|
||||
className='mt-2'
|
||||
title={t(`${I18N_PREFIX}.exceptionErrorTitle`)}
|
||||
errorMsg={crawlErrorMessage}
|
||||
/>
|
||||
)}
|
||||
{isCrawlFinished && !showError && (
|
||||
<CrawledResult
|
||||
className='mt-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(FireCrawl)
|
||||
@@ -0,0 +1,118 @@
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useAppForm } from '@/app/components/base/form'
|
||||
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
|
||||
import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
|
||||
import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general'
|
||||
import type { CrawlOptions } from '@/models/datasets'
|
||||
import cn from '@/utils/classnames'
|
||||
import { RiPlayLargeLine } from '@remixicon/react'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import { useEffect } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useSchema } from './hooks'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export type FormData = {
|
||||
url: string
|
||||
} & CrawlOptions
|
||||
|
||||
type OptionsProps = {
|
||||
initialData: FormData
|
||||
configurations: BaseConfiguration<FormData>[]
|
||||
isRunning: boolean
|
||||
controlFoldOptions?: number
|
||||
onSubmit: (data: FormData) => void
|
||||
}
|
||||
|
||||
const Options = ({
|
||||
initialData,
|
||||
configurations,
|
||||
isRunning,
|
||||
controlFoldOptions,
|
||||
onSubmit,
|
||||
}: OptionsProps) => {
|
||||
const schema = useSchema()
|
||||
const form = useAppForm({
|
||||
defaultValues: initialData,
|
||||
validators: {
|
||||
onSubmit: ({ value }) => {
|
||||
const result = schema.safeParse(value)
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues
|
||||
const firstIssue = issues[0]
|
||||
const errorMessage = `"${firstIssue.path.join('.')}" ${firstIssue.message}`
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: errorMessage,
|
||||
})
|
||||
return errorMessage
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
},
|
||||
onSubmit: ({ value }) => {
|
||||
onSubmit(value)
|
||||
},
|
||||
})
|
||||
|
||||
const { t } = useTranslation()
|
||||
|
||||
const [fold, {
|
||||
toggle: foldToggle,
|
||||
setTrue: foldHide,
|
||||
}] = useBoolean(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (controlFoldOptions !== 0)
|
||||
foldHide()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [controlFoldOptions])
|
||||
|
||||
return (
|
||||
<form
|
||||
className='w-full'
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
form.handleSubmit()
|
||||
}}
|
||||
>
|
||||
<div className='flex items-center gap-x-1 px-4 py-2'>
|
||||
<div
|
||||
className='flex grow cursor-pointer select-none items-center gap-x-0.5'
|
||||
onClick={foldToggle}
|
||||
>
|
||||
<span className='system-sm-semibold-uppercase text-text-secondary'>
|
||||
{t(`${I18N_PREFIX}.options`)}
|
||||
</span>
|
||||
<ArrowDownRoundFill className={cn('h-4 w-4 shrink-0 text-text-tertiary', fold && '-rotate-90')} />
|
||||
</div>
|
||||
<Button
|
||||
variant='primary'
|
||||
onClick={form.handleSubmit}
|
||||
loading={isRunning}
|
||||
className='shrink-0 gap-x-0.5'
|
||||
spinnerClassName='!ml-0'
|
||||
>
|
||||
<RiPlayLargeLine className='size-4' />
|
||||
<span className='px-0.5'>{!isRunning ? t(`${I18N_PREFIX}.run`) : t(`${I18N_PREFIX}.running`)}</span>
|
||||
</Button>
|
||||
</div>
|
||||
{!fold && (
|
||||
<div className='flex flex-col gap-3 border-t border-divider-subtle px-4 py-3'>
|
||||
{configurations.map((config, index) => {
|
||||
const FieldComponent = BaseField<FormData>({
|
||||
initialData,
|
||||
config,
|
||||
})
|
||||
return <FieldComponent key={index} form={form} />
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
export default Options
|
||||
@@ -4,30 +4,35 @@ import { useCallback, useMemo, useState } from 'react'
|
||||
import StepIndicator from './step-indicator'
|
||||
import { useTestRunSteps } from './hooks'
|
||||
import DataSourceOptions from './data-source-options'
|
||||
import type { FileItem } from '@/models/datasets'
|
||||
import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import LocalFile from './data-source/local-file'
|
||||
import produce from 'immer'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useProviderContextSelector } from '@/context/provider-context'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import { DataSourceProvider, type NotionPage } from '@/models/common'
|
||||
import Notion from './data-source/notion'
|
||||
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
|
||||
import { DEFAULT_CRAWL_OPTIONS } from './consts'
|
||||
import Firecrawl from './data-source/website/firecrawl'
|
||||
|
||||
const TestRunPanel = () => {
|
||||
const { t } = useTranslation()
|
||||
const [currentStep, setCurrentStep] = useState(1)
|
||||
const [dataSourceType, setDataSourceType] = useState<string>(DataSourceType.FILE)
|
||||
const [dataSource, setDataSource] = useState<string>(DataSourceProvider.fireCrawl)
|
||||
const [fileList, setFiles] = useState<FileItem[]>([])
|
||||
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
|
||||
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
|
||||
const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
|
||||
const [crawlOptions, setCrawlOptions] = useState<CrawlOptions>(DEFAULT_CRAWL_OPTIONS)
|
||||
|
||||
const setShowTestRunPanel = useStore(s => s.setShowTestRunPanel)
|
||||
const plan = useProviderContextSelector(state => state.plan)
|
||||
const enableBilling = useProviderContextSelector(state => state.enableBilling)
|
||||
|
||||
const steps = useTestRunSteps()
|
||||
const dataSources = ['upload_file', 'notion_import', 'firecrawl', 'jinareader', 'watercrawl'] // TODO: replace with real data sources
|
||||
const dataSources = ['upload_file', 'notion_import', 'firecrawl', 'jinareader', 'watercrawl'] // TODO: replace with real data sources from API
|
||||
|
||||
const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
|
||||
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
|
||||
@@ -42,19 +47,19 @@ const TestRunPanel = () => {
|
||||
}, [fileList, isShowVectorSpaceFull])
|
||||
|
||||
const nextBtnDisabled = useMemo(() => {
|
||||
if (dataSourceType === DataSourceType.FILE)
|
||||
if (dataSource === DataSourceType.FILE)
|
||||
return nextDisabled
|
||||
if (dataSourceType === DataSourceType.NOTION)
|
||||
if (dataSource === DataSourceType.NOTION)
|
||||
return isShowVectorSpaceFull || !notionPages.length
|
||||
return false
|
||||
}, [dataSourceType, nextDisabled, isShowVectorSpaceFull, notionPages.length])
|
||||
}, [dataSource, nextDisabled, isShowVectorSpaceFull, notionPages.length])
|
||||
|
||||
const handleClose = () => {
|
||||
setShowTestRunPanel?.(false)
|
||||
}
|
||||
|
||||
const handleDataSourceSelect = useCallback((option: string) => {
|
||||
setDataSourceType(option)
|
||||
setDataSource(option)
|
||||
}, [])
|
||||
|
||||
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
@@ -81,7 +86,9 @@ const TestRunPanel = () => {
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div className='relative flex h-full w-[480px] flex-col rounded-l-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl shadow-shadow-shadow-1'>
|
||||
<div
|
||||
className='relative flex h-full w-[480px] flex-col rounded-l-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl shadow-shadow-shadow-1'
|
||||
>
|
||||
<button
|
||||
type='button'
|
||||
className='absolute right-2.5 top-2.5 flex size-8 items-center justify-center p-1.5'
|
||||
@@ -95,41 +102,52 @@ const TestRunPanel = () => {
|
||||
</div>
|
||||
<StepIndicator steps={steps} currentStep={currentStep} />
|
||||
</div>
|
||||
{
|
||||
currentStep === 1 && (
|
||||
<>
|
||||
<div className='flex flex-col gap-y-4 px-4 py-2'>
|
||||
<DataSourceOptions
|
||||
dataSources={dataSources}
|
||||
dataSourceType={dataSourceType}
|
||||
onSelect={handleDataSourceSelect}
|
||||
/>
|
||||
{dataSourceType === DataSourceType.FILE && (
|
||||
<LocalFile
|
||||
files={fileList}
|
||||
updateFile={updateFile}
|
||||
updateFileList={updateFileList}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
<div className='grow overflow-y-auto'>
|
||||
{
|
||||
currentStep === 1 && (
|
||||
<>
|
||||
<div className='flex flex-col gap-y-4 px-4 py-2'>
|
||||
<DataSourceOptions
|
||||
dataSources={dataSources}
|
||||
dataSourceType={dataSource}
|
||||
onSelect={handleDataSourceSelect}
|
||||
/>
|
||||
)}
|
||||
{dataSourceType === DataSourceType.NOTION && (
|
||||
<Notion
|
||||
notionPages={notionPages}
|
||||
updateNotionPages={updateNotionPages}
|
||||
/>
|
||||
)}
|
||||
{isShowVectorSpaceFull && (
|
||||
<VectorSpaceFull />
|
||||
)}
|
||||
</div>
|
||||
<div className='flex justify-end p-4 pt-2'>
|
||||
<Button disabled={nextBtnDisabled} variant='primary' onClick={handleNextStep}>
|
||||
<span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
{dataSource === DataSourceType.FILE && (
|
||||
<LocalFile
|
||||
files={fileList}
|
||||
updateFile={updateFile}
|
||||
updateFileList={updateFileList}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
/>
|
||||
)}
|
||||
{dataSource === DataSourceType.NOTION && (
|
||||
<Notion
|
||||
notionPages={notionPages}
|
||||
updateNotionPages={updateNotionPages}
|
||||
/>
|
||||
)}
|
||||
{dataSource === DataSourceProvider.fireCrawl && (
|
||||
<Firecrawl
|
||||
checkedCrawlResult={websitePages}
|
||||
onCheckedCrawlResultChange={setWebsitePages}
|
||||
onJobIdChange={setWebsiteCrawlJobId}
|
||||
crawlOptions={crawlOptions}
|
||||
onCrawlOptionsChange={setCrawlOptions}
|
||||
/>
|
||||
)}
|
||||
{isShowVectorSpaceFull && (
|
||||
<VectorSpaceFull />
|
||||
)}
|
||||
</div>
|
||||
<div className='flex justify-end p-4 pt-2'>
|
||||
<Button disabled={nextBtnDisabled} variant='primary' onClick={handleNextStep}>
|
||||
<span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -78,6 +78,7 @@ const translation = {
|
||||
configureWatercrawl: 'Configure Watercrawl',
|
||||
configureJinaReader: 'Configure Jina Reader',
|
||||
run: 'Run',
|
||||
running: 'Running',
|
||||
firecrawlTitle: 'Extract web content with 🔥Firecrawl',
|
||||
firecrawlDoc: 'Firecrawl docs',
|
||||
firecrawlDocLink: 'https://docs.dify.ai/guides/knowledge-base/sync-from-website',
|
||||
|
||||
@@ -77,6 +77,7 @@ const translation = {
|
||||
configureFirecrawl: '配置 Firecrawl',
|
||||
configureJinaReader: '配置 Jina Reader',
|
||||
run: '运行',
|
||||
running: '运行中',
|
||||
firecrawlTitle: '使用 🔥Firecrawl 提取网页内容',
|
||||
firecrawlDoc: 'Firecrawl 文档',
|
||||
firecrawlDocLink: 'https://docs.dify.ai/v/zh-hans/guides/knowledge-base/sync-from-website',
|
||||
|
||||
Reference in New Issue
Block a user