Compare commits

...

5 Commits

Author SHA1 Message Date
Stephen Zhou
5f74392c30 tweaks 2026-03-22 21:37:53 +08:00
Stephen Zhou
9f4cf431a4 tweaks 2026-03-22 21:35:40 +08:00
Stephen Zhou
89cdbca4f9 chore: update deps 2026-03-22 21:26:34 +08:00
wangxiaolei
40846c262c perf: tidb_on_qdrant_vector delete_by_ids use batch delete (#33846) 2026-03-22 21:09:43 +09:00
ckstck
c6e317a00b fix: test error by matching pkg versioin with docker compose (#33857) 2026-03-22 18:33:24 +09:00
9 changed files with 1644 additions and 1260 deletions

View File

@@ -284,27 +284,29 @@ class TidbOnQdrantVector(BaseVector):
from qdrant_client.http import models
from qdrant_client.http.exceptions import UnexpectedResponse
for node_id in ids:
try:
filter = models.Filter(
must=[
models.FieldCondition(
key="metadata.doc_id",
match=models.MatchValue(value=node_id),
),
],
)
self._client.delete(
collection_name=self._collection_name,
points_selector=FilterSelector(filter=filter),
)
except UnexpectedResponse as e:
# Collection does not exist, so return
if e.status_code == 404:
return
# Some other error occurred, so re-raise the exception
else:
raise e
if not ids:
return
try:
filter = models.Filter(
must=[
models.FieldCondition(
key="metadata.doc_id",
match=models.MatchAny(any=ids),
),
],
)
self._client.delete(
collection_name=self._collection_name,
points_selector=FilterSelector(filter=filter),
)
except UnexpectedResponse as e:
# Collection does not exist, so return
if e.status_code == 404:
return
# Some other error occurred, so re-raise the exception
else:
raise e
def text_exists(self, id: str) -> bool:
all_collection_name = []

View File

@@ -165,8 +165,9 @@ class DifyTestContainers:
# Start Dify Sandbox container for code execution environment
# Dify Sandbox provides a secure environment for executing user code
# Use pinned version 0.2.12 to match production docker-compose configuration
logger.info("Initializing Dify Sandbox container...")
self.dify_sandbox = DockerContainer(image="langgenius/dify-sandbox:latest").with_network(self.network)
self.dify_sandbox = DockerContainer(image="langgenius/dify-sandbox:0.2.12").with_network(self.network)
self.dify_sandbox.with_exposed_ports(8194)
self.dify_sandbox.env = {
"API_KEY": "test_api_key",

View File

@@ -0,0 +1,160 @@
from unittest.mock import patch
import httpx
import pytest
from qdrant_client.http import models as rest
from qdrant_client.http.exceptions import UnexpectedResponse
from core.rag.datasource.vdb.tidb_on_qdrant.tidb_on_qdrant_vector import (
TidbOnQdrantConfig,
TidbOnQdrantVector,
)
class TestTidbOnQdrantVectorDeleteByIds:
"""Unit tests for TidbOnQdrantVector.delete_by_ids method."""
@pytest.fixture
def vector_instance(self):
"""Create a TidbOnQdrantVector instance for testing."""
config = TidbOnQdrantConfig(
endpoint="http://localhost:6333",
api_key="test_api_key",
)
with patch("core.rag.datasource.vdb.tidb_on_qdrant.tidb_on_qdrant_vector.qdrant_client.QdrantClient"):
vector = TidbOnQdrantVector(
collection_name="test_collection",
group_id="test_group",
config=config,
)
return vector
def test_delete_by_ids_with_multiple_ids(self, vector_instance):
"""Test batch deletion with multiple document IDs."""
ids = ["doc1", "doc2", "doc3"]
vector_instance.delete_by_ids(ids)
# Verify that delete was called once with MatchAny filter
vector_instance._client.delete.assert_called_once()
call_args = vector_instance._client.delete.call_args
# Check collection name
assert call_args[1]["collection_name"] == "test_collection"
# Verify filter uses MatchAny with all IDs
filter_selector = call_args[1]["points_selector"]
filter_obj = filter_selector.filter
assert len(filter_obj.must) == 1
field_condition = filter_obj.must[0]
assert field_condition.key == "metadata.doc_id"
assert isinstance(field_condition.match, rest.MatchAny)
assert set(field_condition.match.any) == {"doc1", "doc2", "doc3"}
def test_delete_by_ids_with_single_id(self, vector_instance):
"""Test deletion with a single document ID."""
ids = ["doc1"]
vector_instance.delete_by_ids(ids)
# Verify that delete was called once
vector_instance._client.delete.assert_called_once()
call_args = vector_instance._client.delete.call_args
# Verify filter uses MatchAny with single ID
filter_selector = call_args[1]["points_selector"]
filter_obj = filter_selector.filter
field_condition = filter_obj.must[0]
assert isinstance(field_condition.match, rest.MatchAny)
assert field_condition.match.any == ["doc1"]
def test_delete_by_ids_with_empty_list(self, vector_instance):
"""Test deletion with empty ID list returns early without API call."""
vector_instance.delete_by_ids([])
# Verify that delete was NOT called
vector_instance._client.delete.assert_not_called()
def test_delete_by_ids_with_404_error(self, vector_instance):
"""Test that 404 errors (collection not found) are handled gracefully."""
ids = ["doc1", "doc2"]
# Mock a 404 error
error = UnexpectedResponse(
status_code=404,
reason_phrase="Not Found",
content=b"Collection not found",
headers=httpx.Headers(),
)
vector_instance._client.delete.side_effect = error
# Should not raise an exception
vector_instance.delete_by_ids(ids)
# Verify delete was called
vector_instance._client.delete.assert_called_once()
def test_delete_by_ids_with_unexpected_error(self, vector_instance):
"""Test that non-404 errors are re-raised."""
ids = ["doc1", "doc2"]
# Mock a 500 error
error = UnexpectedResponse(
status_code=500,
reason_phrase="Internal Server Error",
content=b"Server error",
headers=httpx.Headers(),
)
vector_instance._client.delete.side_effect = error
# Should re-raise the exception
with pytest.raises(UnexpectedResponse) as exc_info:
vector_instance.delete_by_ids(ids)
assert exc_info.value.status_code == 500
def test_delete_by_ids_with_large_batch(self, vector_instance):
"""Test deletion with a large batch of IDs."""
# Create 1000 IDs
ids = [f"doc_{i}" for i in range(1000)]
vector_instance.delete_by_ids(ids)
# Verify single delete call with all IDs
vector_instance._client.delete.assert_called_once()
call_args = vector_instance._client.delete.call_args
filter_selector = call_args[1]["points_selector"]
filter_obj = filter_selector.filter
field_condition = filter_obj.must[0]
# Verify all 1000 IDs are in the batch
assert len(field_condition.match.any) == 1000
assert "doc_0" in field_condition.match.any
assert "doc_999" in field_condition.match.any
def test_delete_by_ids_filter_structure(self, vector_instance):
"""Test that the filter structure is correctly constructed."""
ids = ["doc1", "doc2"]
vector_instance.delete_by_ids(ids)
call_args = vector_instance._client.delete.call_args
filter_selector = call_args[1]["points_selector"]
filter_obj = filter_selector.filter
# Verify Filter structure
assert isinstance(filter_obj, rest.Filter)
assert filter_obj.must is not None
assert len(filter_obj.must) == 1
# Verify FieldCondition structure
field_condition = filter_obj.must[0]
assert isinstance(field_condition, rest.FieldCondition)
assert field_condition.key == "metadata.doc_id"
# Verify MatchAny structure
assert isinstance(field_condition.match, rest.MatchAny)
assert field_condition.match.any == ids

View File

@@ -21,7 +21,6 @@ const config: KnipConfig = {
'@storybook/addon-onboarding',
'@voidzero-dev/vite-plus-core',
],
rules: {
files: 'warn',
@@ -32,7 +31,6 @@ const config: KnipConfig = {
unresolved: 'warn',
exports: 'warn',
nsExports: 'warn',
classMembers: 'warn',
types: 'warn',
nsTypes: 'warn',
enumMembers: 'warn',

View File

@@ -58,8 +58,8 @@
"uglify-embed": "node ./bin/uglify-embed"
},
"dependencies": {
"@amplitude/analytics-browser": "2.36.7",
"@amplitude/plugin-session-replay-browser": "1.26.4",
"@amplitude/analytics-browser": "2.37.0",
"@amplitude/plugin-session-replay-browser": "1.27.1",
"@base-ui/react": "1.3.0",
"@emoji-mart/data": "1.2.1",
"@floating-ui/react": "0.27.19",
@@ -67,28 +67,28 @@
"@headlessui/react": "2.2.9",
"@heroicons/react": "2.2.0",
"@hono/node-server": "1.19.11",
"@lexical/code": "0.41.0",
"@lexical/link": "0.41.0",
"@lexical/list": "0.41.0",
"@lexical/react": "0.41.0",
"@lexical/selection": "0.41.0",
"@lexical/text": "0.41.0",
"@lexical/utils": "0.41.0",
"@lexical/code": "0.42.0",
"@lexical/link": "0.42.0",
"@lexical/list": "0.42.0",
"@lexical/react": "0.42.0",
"@lexical/selection": "0.42.0",
"@lexical/text": "0.42.0",
"@lexical/utils": "0.42.0",
"@monaco-editor/react": "4.7.0",
"@octokit/core": "7.0.6",
"@octokit/request-error": "7.1.0",
"@orpc/client": "1.13.8",
"@orpc/contract": "1.13.8",
"@orpc/openapi-client": "1.13.8",
"@orpc/tanstack-query": "1.13.8",
"@orpc/client": "1.13.9",
"@orpc/contract": "1.13.9",
"@orpc/openapi-client": "1.13.9",
"@orpc/tanstack-query": "1.13.9",
"@remixicon/react": "4.9.0",
"@sentry/react": "10.44.0",
"@sentry/react": "10.45.0",
"@streamdown/math": "1.0.2",
"@svgdotjs/svg.js": "3.2.5",
"@t3-oss/env-nextjs": "0.13.10",
"@tailwindcss/typography": "0.5.19",
"@tanstack/react-form": "1.28.5",
"@tanstack/react-query": "5.91.0",
"@tanstack/react-query": "5.94.5",
"abcjs": "6.6.2",
"ahooks": "3.9.6",
"class-variance-authority": "0.7.1",
@@ -111,7 +111,7 @@
"hono": "4.12.8",
"html-entities": "2.6.0",
"html-to-image": "1.11.13",
"i18next": "25.8.18",
"i18next": "25.10.4",
"i18next-resources-to-backend": "1.2.1",
"immer": "11.1.4",
"jotai": "2.18.1",
@@ -119,15 +119,15 @@
"js-cookie": "3.0.5",
"js-yaml": "4.1.1",
"jsonschema": "1.5.0",
"katex": "0.16.38",
"katex": "0.16.40",
"ky": "1.14.3",
"lamejs": "1.2.1",
"lexical": "0.41.0",
"lexical": "0.42.0",
"mermaid": "11.13.0",
"mime": "4.1.0",
"mitt": "3.0.1",
"negotiator": "1.0.0",
"next": "16.2.0",
"next": "16.2.1",
"next-themes": "0.4.6",
"nuqs": "2.8.9",
"pinyin-pro": "3.28.0",
@@ -138,7 +138,7 @@
"react-dom": "19.2.4",
"react-easy-crop": "5.5.6",
"react-hotkeys-hook": "5.2.4",
"react-i18next": "16.5.8",
"react-i18next": "16.6.1",
"react-multi-email": "1.0.25",
"react-papaparse": "4.4.0",
"react-pdf-highlighter": "8.0.0-rc.0",
@@ -157,7 +157,7 @@
"streamdown": "2.5.0",
"string-ts": "2.3.1",
"tailwind-merge": "2.6.1",
"tldts": "7.0.26",
"tldts": "7.0.27",
"unist-util-visit": "5.1.0",
"use-context-selector": "2.0.0",
"uuid": "13.0.0",
@@ -167,7 +167,7 @@
},
"devDependencies": {
"@antfu/eslint-config": "7.7.3",
"@chromatic-com/storybook": "5.0.1",
"@chromatic-com/storybook": "5.0.2",
"@egoist/tailwindcss-icons": "1.9.2",
"@eslint-react/eslint-plugin": "2.13.0",
"@iconify-json/heroicons": "1.2.3",
@@ -175,19 +175,19 @@
"@mdx-js/loader": "3.1.1",
"@mdx-js/react": "3.1.1",
"@mdx-js/rollup": "3.1.1",
"@next/eslint-plugin-next": "16.2.0",
"@next/mdx": "16.2.0",
"@next/eslint-plugin-next": "16.2.1",
"@next/mdx": "16.2.1",
"@rgrove/parse-xml": "4.2.0",
"@storybook/addon-docs": "10.3.0",
"@storybook/addon-links": "10.3.0",
"@storybook/addon-onboarding": "10.3.0",
"@storybook/addon-themes": "10.3.0",
"@storybook/nextjs-vite": "10.3.0",
"@storybook/react": "10.3.0",
"@tanstack/eslint-plugin-query": "5.91.5",
"@storybook/addon-docs": "10.3.1",
"@storybook/addon-links": "10.3.1",
"@storybook/addon-onboarding": "10.3.1",
"@storybook/addon-themes": "10.3.1",
"@storybook/nextjs-vite": "10.3.1",
"@storybook/react": "10.3.1",
"@tanstack/eslint-plugin-query": "5.94.5",
"@tanstack/react-devtools": "0.10.0",
"@tanstack/react-form-devtools": "0.2.19",
"@tanstack/react-query-devtools": "5.91.3",
"@tanstack/react-query-devtools": "5.94.5",
"@testing-library/dom": "10.4.1",
"@testing-library/jest-dom": "6.9.1",
"@testing-library/react": "16.3.2",
@@ -208,14 +208,14 @@
"@types/react-window": "1.8.8",
"@types/sortablejs": "1.15.9",
"@typescript-eslint/parser": "8.57.1",
"@typescript/native-preview": "7.0.0-dev.20260318.1",
"@typescript/native-preview": "7.0.0-dev.20260322.1",
"@vitejs/plugin-react": "6.0.1",
"@vitejs/plugin-rsc": "0.5.21",
"@vitest/coverage-v8": "4.1.0",
"agentation": "2.3.3",
"autoprefixer": "10.4.27",
"code-inspector-plugin": "1.4.4",
"eslint": "10.0.3",
"code-inspector-plugin": "1.4.5",
"eslint": "10.1.0",
"eslint-markdown": "0.6.0",
"eslint-plugin-better-tailwindcss": "4.3.2",
"eslint-plugin-hyoban": "0.14.1",
@@ -223,29 +223,29 @@
"eslint-plugin-react-hooks": "7.0.1",
"eslint-plugin-react-refresh": "0.5.2",
"eslint-plugin-sonarjs": "4.0.2",
"eslint-plugin-storybook": "10.3.0",
"eslint-plugin-storybook": "10.3.1",
"husky": "9.1.7",
"iconify-import-svg": "0.1.2",
"jsdom": "29.0.0",
"jsdom": "29.0.1",
"jsdom-testing-mocks": "1.16.0",
"knip": "5.88.0",
"knip": "6.0.1",
"lint-staged": "16.4.0",
"nock": "14.0.11",
"postcss": "8.5.8",
"postcss-js": "5.1.0",
"react-server-dom-webpack": "19.2.4",
"sass": "1.98.0",
"storybook": "10.3.0",
"storybook": "10.3.1",
"tailwindcss": "3.4.19",
"taze": "19.10.0",
"tsx": "4.21.0",
"typescript": "5.9.3",
"uglify-js": "3.19.3",
"vinext": "0.0.31",
"vite": "npm:@voidzero-dev/vite-plus-core@0.1.12",
"vinext": "0.0.33",
"vite": "npm:@voidzero-dev/vite-plus-core@0.1.13",
"vite-plugin-inspect": "11.3.3",
"vite-plus": "0.1.12",
"vitest": "npm:@voidzero-dev/vite-plus-test@0.1.12",
"vite-plus": "0.1.13",
"vitest": "npm:@voidzero-dev/vite-plus-test@0.1.13",
"vitest-canvas-mock": "1.1.3"
},
"pnpm": {
@@ -261,7 +261,7 @@
"array.prototype.tosorted": "npm:@nolyfill/array.prototype.tosorted@^1.0.44",
"assert": "npm:@nolyfill/assert@^1.0.26",
"brace-expansion@<2.0.2": "2.0.2",
"canvas": "^3.2.1",
"canvas": "^3.2.2",
"devalue@<5.3.2": "5.3.2",
"dompurify@>=3.1.3 <=3.3.1": "3.3.2",
"es-iterator-helpers": "npm:@nolyfill/es-iterator-helpers@^1.0.21",
@@ -297,8 +297,8 @@
"tar@<=7.5.10": "7.5.11",
"typed-array-buffer": "npm:@nolyfill/typed-array-buffer@^1.0.44",
"undici@>=7.0.0 <7.24.0": "7.24.0",
"vite": "npm:@voidzero-dev/vite-plus-core@0.1.12",
"vitest": "npm:@voidzero-dev/vite-plus-test@0.1.12",
"vite": "npm:@voidzero-dev/vite-plus-core@0.1.13",
"vitest": "npm:@voidzero-dev/vite-plus-test@0.1.13",
"which-typed-array": "npm:@nolyfill/which-typed-array@^1.0.44",
"yauzl@<3.2.1": "3.2.1"
},

2591
web/pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@@ -3,10 +3,8 @@ import { fileURLToPath } from 'node:url'
import { getIconCollections, iconsPlugin } from '@egoist/tailwindcss-icons'
import tailwindTypography from '@tailwindcss/typography'
import { importSvgCollections } from 'iconify-import-svg'
// @ts-expect-error workaround for turbopack issue
import { cssAsPlugin } from './tailwind-css-plugin.ts'
// @ts-expect-error workaround for turbopack issue
import tailwindThemeVarDefine from './themes/tailwind-theme-var-define.ts'
import { cssAsPlugin } from './tailwind-css-plugin'
import tailwindThemeVarDefine from './themes/tailwind-theme-var-define'
import typography from './typography.js'
const _dirname = typeof __dirname !== 'undefined'

View File

@@ -1,7 +1,7 @@
// import type { Config } from 'tailwindcss'
import commonConfig from './tailwind-common-config.ts'
import type { Config } from 'tailwindcss'
import commonConfig from './tailwind-common-config'
const config = {
const config: Config = {
content: [
'./app/**/*.{js,ts,jsx,tsx}',
'./components/**/*.{js,ts,jsx,tsx}',