update

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: hj24 <mambahj24@gmail.com>

.codex/skills/component-refactoring

@@ -1 +0,0 @@
-../../.agents/skills/component-refactoring

.codex/skills/frontend-code-review

@@ -1 +0,0 @@
-../../.agents/skills/frontend-code-review

.codex/skills/frontend-testing

@@ -1 +0,0 @@
-../../.agents/skills/frontend-testing

.codex/skills/orpc-contract-first

@@ -1 +0,0 @@
-../../.agents/skills/orpc-contract-first

.github/workflows/autofix.yml

@@ -79,29 +79,6 @@ jobs:
           find . -name "*.py" -type f -exec sed -i.bak -E 's/"([^"]+)" \| None/Optional["\1"]/g; s/'"'"'([^'"'"']+)'"'"' \| None/Optional['"'"'\1'"'"']/g' {} \;
           find . -name "*.py.bak" -type f -delete
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Install web dependencies
-        run: |
-          cd web
-          pnpm install --frozen-lockfile
-
-      - name: ESLint autofix
-        run: |
-          cd web
-          pnpm lint:fix || true
-
       # mdformat breaks YAML front matter in markdown files. Add --exclude for directories containing YAML front matter.
       - name: mdformat
         run: |

.github/workflows/deploy-hitl.yml

@@ -4,8 +4,7 @@ on:
   workflow_run:
     workflows: ["Build and Push API & Web"]
     branches:
-      - "feat/hitl-frontend"
-      - "feat/hitl-backend"
+      - "build/feat/hitl"
     types:
       - completed
 
@@ -14,10 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       github.event.workflow_run.conclusion == 'success' &&
-      (
-        github.event.workflow_run.head_branch == 'feat/hitl-frontend' ||
-        github.event.workflow_run.head_branch == 'feat/hitl-backend'
-      )
+      github.event.workflow_run.head_branch == 'build/feat/hitl'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v1

api/.importlinter

@@ -136,7 +136,6 @@ ignore_imports =
     core.workflow.nodes.llm.llm_utils -> models.provider
     core.workflow.nodes.llm.llm_utils -> services.credit_pool_service
     core.workflow.nodes.llm.node -> core.tools.signature
-    core.workflow.nodes.template_transform.template_transform_node -> configs
     core.workflow.nodes.tool.tool_node -> core.callback_handler.workflow_tool_callback_handler
     core.workflow.nodes.tool.tool_node -> core.tools.tool_engine
     core.workflow.nodes.tool.tool_node -> core.tools.tool_manager

api/README.md

@@ -122,7 +122,7 @@ These commands assume you start from the repository root.
 
    ```bash
    cd api
-   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
+   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
    ```
 
 1. Optional: start Celery Beat (scheduled tasks, in a new terminal).

api/commands.py

@@ -739,8 +739,10 @@ def upgrade_db():
 
             click.echo(click.style("Database migration successful!", fg="green"))
 
-        except Exception:
+        except Exception as e:
             logger.exception("Failed to execute database migration")
+            click.echo(click.style(f"Database migration failed: {e}", fg="red"))
+            raise SystemExit(1)
         finally:
             lock.release()
     else:

api/configs/feature/__init__.py

@@ -1155,6 +1155,16 @@ class CeleryScheduleTasksConfig(BaseSettings):
         default=0,
     )
 
+    # API token last_used_at batch update
+    ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK: bool = Field(
+        description="Enable periodic batch update of API token last_used_at timestamps",
+        default=True,
+    )
+    API_TOKEN_LAST_USED_UPDATE_INTERVAL: int = Field(
+        description="Interval in minutes for batch updating API token last_used_at (default 30)",
+        default=30,
+    )
+
     # Trigger provider refresh (simple version)
     ENABLE_TRIGGER_PROVIDER_REFRESH_TASK: bool = Field(
         description="Enable trigger provider refresh poller",

api/controllers/console/apikey.py

@@ -10,6 +10,7 @@ from libs.helper import TimestampField
 from libs.login import current_account_with_tenant, login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
+from services.api_token_service import ApiTokenCache
 
 from . import console_ns
 from .wraps import account_initialization_required, edit_permission_required, setup_required
@@ -131,6 +132,11 @@ class BaseApiKeyResource(Resource):
         if key is None:
             flask_restx.abort(HTTPStatus.NOT_FOUND, message="API key not found")
 
+        # Invalidate cache before deleting from database
+        # Type assertion: key is guaranteed to be non-None here because abort() raises
+        assert key is not None  # nosec - for type checker only
+        ApiTokenCache.delete(key.token, key.type)
+
         db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
         db.session.commit()
 

api/controllers/console/app/app.py

@@ -1,3 +1,4 @@
+import logging
 import uuid
 from datetime import datetime
 from typing import Any, Literal, TypeAlias
@@ -54,6 +55,8 @@ ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "co
 
 register_enum_models(console_ns, IconType)
 
+_logger = logging.getLogger(__name__)
+
 
 class AppListQuery(BaseModel):
     page: int = Field(default=1, ge=1, le=99999, description="Page number (1-99999)")
@@ -499,6 +502,7 @@ class AppListApi(Resource):
                     select(Workflow).where(
                         Workflow.version == Workflow.VERSION_DRAFT,
                         Workflow.app_id.in_(workflow_capable_app_ids),
+                        Workflow.tenant_id == current_tenant_id,
                     )
                 )
                 .scalars()
@@ -510,12 +514,14 @@ class AppListApi(Resource):
                 NodeType.TRIGGER_PLUGIN,
             }
             for workflow in draft_workflows:
+                node_id = None
                 try:
-                    for _, node_data in workflow.walk_nodes():
+                    for node_id, node_data in workflow.walk_nodes():
                         if node_data.get("type") in trigger_node_types:
                             draft_trigger_app_ids.add(str(workflow.app_id))
                             break
                 except Exception:
+                    _logger.exception("error while walking nodes, workflow_id=%s, node_id=%s", workflow.id, node_id)
                     continue
 
         for app in app_pagination.items:

api/controllers/console/datasets/datasets.py

@@ -55,6 +55,7 @@ from libs.login import current_account_with_tenant, login_required
 from models import ApiToken, Dataset, Document, DocumentSegment, UploadFile
 from models.dataset import DatasetPermissionEnum
 from models.provider_ids import ModelProviderID
+from services.api_token_service import ApiTokenCache
 from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
 # Register models for flask_restx to avoid dict type issues in Swagger
@@ -820,6 +821,11 @@ class DatasetApiDeleteApi(Resource):
         if key is None:
             console_ns.abort(404, message="API key not found")
 
+        # Invalidate cache before deleting from database
+        # Type assertion: key is guaranteed to be non-None here because abort() raises
+        assert key is not None  # nosec - for type checker only
+        ApiTokenCache.delete(key.token, key.type)
+
         db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
         db.session.commit()
 

api/controllers/console/tag/tags.py

@@ -120,7 +120,7 @@ class TagUpdateDeleteApi(Resource):
 
         TagService.delete_tag(tag_id)
 
-        return 204
+        return "", 204
 
 
 @console_ns.route("/tag-bindings/create")

api/controllers/service_api/dataset/dataset.py

@@ -396,7 +396,7 @@ class DatasetApi(DatasetApiResource):
         try:
             if DatasetService.delete_dataset(dataset_id_str, current_user):
                 DatasetPermissionService.clear_partial_member_list(dataset_id_str)
-                return 204
+                return "", 204
             else:
                 raise NotFound("Dataset not found.")
         except services.errors.dataset.DatasetInUseError:
@@ -557,7 +557,7 @@ class DatasetTagsApi(DatasetApiResource):
         payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
         TagService.delete_tag(payload.tag_id)
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/tags/binding")
@@ -581,7 +581,7 @@ class DatasetTagBindingApi(DatasetApiResource):
         payload = TagBindingPayload.model_validate(service_api_ns.payload or {})
         TagService.save_tag_binding({"tag_ids": payload.tag_ids, "target_id": payload.target_id, "type": "knowledge"})
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/tags/unbinding")
@@ -605,7 +605,7 @@ class DatasetTagUnbindingApi(DatasetApiResource):
         payload = TagUnbindingPayload.model_validate(service_api_ns.payload or {})
         TagService.delete_tag_binding({"tag_id": payload.tag_id, "target_id": payload.target_id, "type": "knowledge"})
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/tags")

api/controllers/service_api/dataset/document.py

@@ -746,4 +746,4 @@ class DocumentApi(DatasetApiResource):
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
-        return 204
+        return "", 204

api/controllers/service_api/dataset/metadata.py

@@ -128,7 +128,7 @@ class DatasetMetadataServiceApi(DatasetApiResource):
         DatasetService.check_dataset_permission(dataset, current_user)
 
         MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in")

api/controllers/service_api/dataset/segment.py

@@ -233,7 +233,7 @@ class DatasetSegmentApi(DatasetApiResource):
         if not segment:
             raise NotFound("Segment not found.")
         SegmentService.delete_segment(segment, document, dataset)
-        return 204
+        return "", 204
 
     @service_api_ns.expect(service_api_ns.models[SegmentUpdatePayload.__name__])
     @service_api_ns.doc("update_segment")
@@ -499,7 +499,7 @@ class DatasetChildChunkApi(DatasetApiResource):
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
 
-        return 204
+        return "", 204
 
     @service_api_ns.expect(service_api_ns.models[ChildChunkUpdatePayload.__name__])
     @service_api_ns.doc("update_child_chunk")

api/controllers/service_api/wraps.py

@@ -1,27 +1,24 @@
 import logging
 import time
 from collections.abc import Callable
-from datetime import timedelta
 from enum import StrEnum, auto
 from functools import wraps
-from typing import Concatenate, ParamSpec, TypeVar
+from typing import Concatenate, ParamSpec, TypeVar, cast
 
 from flask import current_app, request
 from flask_login import user_logged_in
 from flask_restx import Resource
 from pydantic import BaseModel
-from sqlalchemy import select, update
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from enums.cloud_plan import CloudPlan
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from libs.datetime_utils import naive_utc_now
 from libs.login import current_user
 from models import Account, Tenant, TenantAccountJoin, TenantStatus
 from models.dataset import Dataset, RateLimitLog
 from models.model import ApiToken, App
+from services.api_token_service import ApiTokenCache, fetch_token_with_single_flight, record_token_usage
 from services.end_user_service import EndUserService
 from services.feature_service import FeatureService
 
@@ -296,7 +293,14 @@ def validate_dataset_token(view: Callable[Concatenate[T, P], R] | None = None):
 
 def validate_and_get_api_token(scope: str | None = None):
     """
-    Validate and get API token.
+    Validate and get API token with Redis caching.
+
+    This function uses a two-tier approach:
+    1. First checks Redis cache for the token
+    2. If not cached, queries database and caches the result
+
+    The last_used_at field is updated asynchronously via Celery task
+    to avoid blocking the request.
     """
     auth_header = request.headers.get("Authorization")
     if auth_header is None or " " not in auth_header:
@@ -308,29 +312,18 @@ def validate_and_get_api_token(scope: str | None = None):
     if auth_scheme != "bearer":
         raise Unauthorized("Authorization scheme must be 'Bearer'")
 
-    current_time = naive_utc_now()
-    cutoff_time = current_time - timedelta(minutes=1)
-    with Session(db.engine, expire_on_commit=False) as session:
-        update_stmt = (
-            update(ApiToken)
-            .where(
-                ApiToken.token == auth_token,
-                (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < cutoff_time)),
-                ApiToken.type == scope,
-            )
-            .values(last_used_at=current_time)
-        )
-        stmt = select(ApiToken).where(ApiToken.token == auth_token, ApiToken.type == scope)
-        result = session.execute(update_stmt)
-        api_token = session.scalar(stmt)
+    # Try to get token from cache first
+    # Returns a CachedApiToken (plain Python object), not a SQLAlchemy model
+    cached_token = ApiTokenCache.get(auth_token, scope)
+    if cached_token is not None:
+        logger.debug("Token validation served from cache for scope: %s", scope)
+        # Record usage in Redis for later batch update (no Celery task per request)
+        record_token_usage(auth_token, scope)
+        return cast(ApiToken, cached_token)
 
-        if hasattr(result, "rowcount") and result.rowcount > 0:
-            session.commit()
-
-        if not api_token:
-            raise Unauthorized("Access token is invalid")
-
-    return api_token
+    # Cache miss - use Redis lock for single-flight mode
+    # This ensures only one request queries DB for the same token concurrently
+    return fetch_token_with_single_flight(auth_token, scope)
 
 
 class DatasetApiResource(Resource):

api/core/app/workflow/node_factory.py

@@ -47,6 +47,7 @@ class DifyNodeFactory(NodeFactory):
         code_providers: Sequence[type[CodeNodeProvider]] | None = None,
         code_limits: CodeNodeLimits | None = None,
         template_renderer: Jinja2TemplateRenderer | None = None,
+        template_transform_max_output_length: int | None = None,
         http_request_http_client: HttpClientProtocol | None = None,
         http_request_tool_file_manager_factory: Callable[[], ToolFileManager] = ToolFileManager,
         http_request_file_manager: FileManagerProtocol | None = None,
@@ -68,6 +69,9 @@ class DifyNodeFactory(NodeFactory):
             max_object_array_length=dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH,
         )
         self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer()
+        self._template_transform_max_output_length = (
+            template_transform_max_output_length or dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH
+        )
         self._http_request_http_client = http_request_http_client or ssrf_proxy
         self._http_request_tool_file_manager_factory = http_request_tool_file_manager_factory
         self._http_request_file_manager = http_request_file_manager or file_manager
@@ -122,6 +126,7 @@ class DifyNodeFactory(NodeFactory):
                 graph_init_params=self.graph_init_params,
                 graph_runtime_state=self.graph_runtime_state,
                 template_renderer=self._template_renderer,
+                max_output_length=self._template_transform_max_output_length,
             )
 
         if node_type == NodeType.HTTP_REQUEST:

api/core/helper/marketplace.py

@@ -6,7 +6,8 @@ from yarl import URL
 
 from configs import dify_config
 from core.helper.download import download_with_size_limit
-from core.plugin.entities.marketplace import MarketplacePluginDeclaration
+from core.plugin.entities.marketplace import MarketplacePluginDeclaration, MarketplacePluginSnapshot
+from extensions.ext_redis import redis_client
 
 marketplace_api_url = URL(str(dify_config.MARKETPLACE_API_URL))
 logger = logging.getLogger(__name__)
@@ -43,28 +44,37 @@ def batch_fetch_plugin_by_ids(plugin_ids: list[str]) -> list[dict]:
     return data.get("data", {}).get("plugins", [])
 
 
-def batch_fetch_plugin_manifests_ignore_deserialization_error(
-    plugin_ids: list[str],
-) -> Sequence[MarketplacePluginDeclaration]:
-    if len(plugin_ids) == 0:
-        return []
-
-    url = str(marketplace_api_url / "api/v1/plugins/batch")
-    response = httpx.post(url, json={"plugin_ids": plugin_ids}, headers={"X-Dify-Version": dify_config.project.version})
-    response.raise_for_status()
-    result: list[MarketplacePluginDeclaration] = []
-    for plugin in response.json()["data"]["plugins"]:
-        try:
-            result.append(MarketplacePluginDeclaration.model_validate(plugin))
-        except Exception:
-            logger.exception(
-                "Failed to deserialize marketplace plugin manifest for %s", plugin.get("plugin_id", "unknown")
-            )
-
-    return result
-
-
 def record_install_plugin_event(plugin_unique_identifier: str):
     url = str(marketplace_api_url / "api/v1/stats/plugins/install_count")
     response = httpx.post(url, json={"unique_identifier": plugin_unique_identifier})
     response.raise_for_status()
+
+
+def fetch_global_plugin_manifest(cache_key_prefix: str, cache_ttl: int) -> None:
+    """
+    Fetch all plugin manifests from marketplace and cache them in Redis.
+    This should be called once per check cycle to populate the instance-level cache.
+
+    Args:
+        cache_key_prefix: Redis key prefix for caching plugin manifests
+        cache_ttl: Cache TTL in seconds
+
+    Raises:
+        httpx.HTTPError: If the HTTP request fails
+        Exception: If any other error occurs during fetching or caching
+    """
+    url = str(marketplace_api_url / "api/v1/dist/plugins/manifest.json")
+    response = httpx.get(url, headers={"X-Dify-Version": dify_config.project.version}, timeout=30)
+    response.raise_for_status()
+
+    raw_json = response.json()
+    plugins_data = raw_json.get("plugins", [])
+
+    # Parse and cache all plugin snapshots
+    for plugin_data in plugins_data:
+        plugin_snapshot = MarketplacePluginSnapshot.model_validate(plugin_data)
+        redis_client.setex(
+            name=f"{cache_key_prefix}{plugin_snapshot.plugin_id}",
+            time=cache_ttl,
+            value=plugin_snapshot.model_dump_json(),
+        )

api/core/plugin/entities/marketplace.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, computed_field, model_validator
 
 from core.model_runtime.entities.provider_entities import ProviderEntity
 from core.plugin.entities.endpoint import EndpointProviderDeclaration
@@ -48,3 +48,15 @@ class MarketplacePluginDeclaration(BaseModel):
         if "tool" in data and not data["tool"]:
             del data["tool"]
         return data
+
+
+class MarketplacePluginSnapshot(BaseModel):
+    org: str
+    name: str
+    latest_version: str
+    latest_package_identifier: str
+    latest_package_url: str
+
+    @computed_field
+    def plugin_id(self) -> str:
+        return f"{self.org}/{self.name}"

api/core/variables/variables.py

@@ -112,7 +112,7 @@ class ArrayBooleanVariable(ArrayBooleanSegment, ArrayVariable):
 
 class RAGPipelineVariable(BaseModel):
     belong_to_node_id: str = Field(description="belong to which node id, shared means public")
-    type: str = Field(description="variable type, text-input, paragraph, select, number,  file, file-list")
+    type: str = Field(description="variable type, text-input, paragraph, select, number, file, file-list")
     label: str = Field(description="label")
     description: str | None = Field(description="description", default="")
     variable: str = Field(description="variable key", default="")

api/core/workflow/nodes/template_transform/template_transform_node.py

@@ -1,7 +1,6 @@
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
-from configs import dify_config
 from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
 from core.workflow.node_events import NodeRunResult
 from core.workflow.nodes.base.node import Node
@@ -16,12 +15,13 @@ if TYPE_CHECKING:
     from core.workflow.entities import GraphInitParams
     from core.workflow.runtime import GraphRuntimeState
 
-MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH = dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH
+DEFAULT_TEMPLATE_TRANSFORM_MAX_OUTPUT_LENGTH = 400_000
 
 
 class TemplateTransformNode(Node[TemplateTransformNodeData]):
     node_type = NodeType.TEMPLATE_TRANSFORM
     _template_renderer: Jinja2TemplateRenderer
+    _max_output_length: int
 
     def __init__(
         self,
@@ -31,6 +31,7 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         graph_runtime_state: "GraphRuntimeState",
         *,
         template_renderer: Jinja2TemplateRenderer | None = None,
+        max_output_length: int | None = None,
     ) -> None:
         super().__init__(
             id=id,
@@ -40,6 +41,10 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         )
         self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer()
 
+        if max_output_length is not None and max_output_length <= 0:
+            raise ValueError("max_output_length must be a positive integer")
+        self._max_output_length = max_output_length or DEFAULT_TEMPLATE_TRANSFORM_MAX_OUTPUT_LENGTH
+
     @classmethod
     def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]:
         """
@@ -69,11 +74,11 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         except TemplateRenderError as e:
             return NodeRunResult(inputs=variables, status=WorkflowNodeExecutionStatus.FAILED, error=str(e))
 
-        if len(rendered) > MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH:
+        if len(rendered) > self._max_output_length:
             return NodeRunResult(
                 inputs=variables,
                 status=WorkflowNodeExecutionStatus.FAILED,
-                error=f"Output length exceeds {MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH} characters",
+                error=f"Output length exceeds {self._max_output_length} characters",
             )
 
         return NodeRunResult(

api/docker/entrypoint.sh

@@ -35,10 +35,10 @@ if [[ "${MODE}" == "worker" ]]; then
   if [[ -z "${CELERY_QUEUES}" ]]; then
     if [[ "${EDITION}" == "CLOUD" ]]; then
       # Cloud edition: separate queues for dataset and trigger tasks
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
+      DEFAULT_QUEUES="api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     else
       # Community edition (SELF_HOSTED): dataset, pipeline and workflow have separate queues
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
+      DEFAULT_QUEUES="api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     fi
   else
     DEFAULT_QUEUES="${CELERY_QUEUES}"

api/extensions/ext_celery.py

@@ -184,6 +184,14 @@ def init_app(app: DifyApp) -> Celery:
             "task": "schedule.trigger_provider_refresh_task.trigger_provider_refresh",
             "schedule": timedelta(minutes=dify_config.TRIGGER_PROVIDER_REFRESH_INTERVAL),
         }
+
+    if dify_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK:
+        imports.append("schedule.update_api_token_last_used_task")
+        beat_schedule["batch_update_api_token_last_used"] = {
+            "task": "schedule.update_api_token_last_used_task.batch_update_api_token_last_used",
+            "schedule": timedelta(minutes=dify_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL),
+        }
+
     celery_app.conf.update(beat_schedule=beat_schedule, imports=imports)
 
     return celery_app

api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py

@@ -10,6 +10,10 @@ import models as models
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+
+def _is_pg(conn):
+    return conn.dialect.name == "postgresql"
+
 # revision identifiers, used by Alembic.
 revision = '7df29de0f6be'
 down_revision = '03ea244985ce'
@@ -19,16 +23,31 @@ depends_on = None
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('tenant_credit_pools',
-    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
-    sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
-    sa.Column('quota_limit', sa.BigInteger(), nullable=False),
-    sa.Column('quota_used', sa.BigInteger(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
-    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
-    sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
-    )
+    conn = op.get_bind()
+
+    if _is_pg(conn):
+        op.create_table('tenant_credit_pools',
+        sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+        sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+        sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
+        sa.Column('quota_limit', sa.BigInteger(), nullable=False),
+        sa.Column('quota_used', sa.BigInteger(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
+        )
+    else:
+        # For MySQL and other databases, UUID should be generated at application level
+        op.create_table('tenant_credit_pools',
+        sa.Column('id', models.types.StringUUID(), nullable=False),
+        sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+        sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
+        sa.Column('quota_limit', sa.BigInteger(), nullable=False),
+        sa.Column('quota_used', sa.BigInteger(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
+        sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
+        )
     with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op:
         batch_op.create_index('tenant_credit_pool_pool_type_idx', ['pool_type'], unique=False)
         batch_op.create_index('tenant_credit_pool_tenant_id_idx', ['tenant_id'], unique=False)

api/models/model.py

@@ -2166,7 +2166,9 @@ class TenantCreditPool(TypeBase):
         sa.Index("tenant_credit_pool_pool_type_idx", "pool_type"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()"), init=False)
+    id: Mapped[str] = mapped_column(
+        StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
+    )
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     pool_type: Mapped[str] = mapped_column(String(40), nullable=False, default="trial", server_default="trial")
     quota_limit: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)

api/schedule/check_upgradable_plugin_task.py

@@ -1,16 +1,24 @@
+import logging
 import math
 import time
 
 import click
 
 import app
+from core.helper.marketplace import fetch_global_plugin_manifest
 from extensions.ext_database import db
 from models.account import TenantPluginAutoUpgradeStrategy
 from tasks import process_tenant_plugin_autoupgrade_check_task as check_task
 
+logger = logging.getLogger(__name__)
+
 AUTO_UPGRADE_MINIMAL_CHECKING_INTERVAL = 15 * 60  # 15 minutes
 MAX_CONCURRENT_CHECK_TASKS = 20
 
+# Import cache constants from the task module
+CACHE_REDIS_KEY_PREFIX = check_task.CACHE_REDIS_KEY_PREFIX
+CACHE_REDIS_TTL = check_task.CACHE_REDIS_TTL
+
 
 @app.celery.task(queue="plugin")
 def check_upgradable_plugin_task():
@@ -40,6 +48,22 @@ def check_upgradable_plugin_task():
     )  # make sure all strategies are checked in this interval
     batch_interval_time = (AUTO_UPGRADE_MINIMAL_CHECKING_INTERVAL / batch_chunk_count) if batch_chunk_count > 0 else 0
 
+    if total_strategies == 0:
+        click.echo(click.style("no strategies to process, skipping plugin manifest fetch.", fg="green"))
+        return
+
+    # Fetch and cache all plugin manifests before processing tenants
+    # This reduces load on marketplace from 300k requests to 1 request per check cycle
+    logger.info("fetching global plugin manifest from marketplace")
+    try:
+        fetch_global_plugin_manifest(CACHE_REDIS_KEY_PREFIX, CACHE_REDIS_TTL)
+        logger.info("successfully fetched and cached global plugin manifest")
+    except Exception as e:
+        logger.exception("failed to fetch global plugin manifest")
+        click.echo(click.style(f"failed to fetch global plugin manifest: {e}", fg="red"))
+        click.echo(click.style("skipping plugin upgrade check for this cycle", fg="yellow"))
+        return
+
     for i in range(0, total_strategies, MAX_CONCURRENT_CHECK_TASKS):
         batch_strategies = strategies[i : i + MAX_CONCURRENT_CHECK_TASKS]
         for strategy in batch_strategies:

api/schedule/update_api_token_last_used_task.py

@@ -0,0 +1,114 @@
+"""
+Scheduled task to batch-update API token last_used_at timestamps.
+
+Instead of updating the database on every request, token usage is recorded
+in Redis as lightweight SET keys (api_token_active:{scope}:{token}).
+This task runs periodically (default every 30 minutes) to flush those
+records into the database in a single batch operation.
+"""
+
+import logging
+import time
+from datetime import datetime
+
+import click
+from sqlalchemy import update
+from sqlalchemy.orm import Session
+
+import app
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.model import ApiToken
+from services.api_token_service import ACTIVE_TOKEN_KEY_PREFIX
+
+logger = logging.getLogger(__name__)
+
+
+@app.celery.task(queue="api_token")
+def batch_update_api_token_last_used():
+    """
+    Batch update last_used_at for all recently active API tokens.
+
+    Scans Redis for api_token_active:* keys, parses the token and scope
+    from each key, and performs a batch database update.
+    """
+    click.echo(click.style("batch_update_api_token_last_used: start.", fg="green"))
+    start_at = time.perf_counter()
+
+    updated_count = 0
+    scanned_count = 0
+
+    try:
+        # Collect all active token keys and their values (the actual usage timestamps)
+        token_entries: list[tuple[str, str | None, datetime]] = []  # (token, scope, usage_time)
+        keys_to_delete: list[str | bytes] = []
+
+        for key in redis_client.scan_iter(match=f"{ACTIVE_TOKEN_KEY_PREFIX}*", count=200):
+            if isinstance(key, bytes):
+                key = key.decode("utf-8")
+            scanned_count += 1
+
+            # Read the value (ISO timestamp recorded at actual request time)
+            value = redis_client.get(key)
+            if not value:
+                keys_to_delete.append(key)
+                continue
+
+            if isinstance(value, bytes):
+                value = value.decode("utf-8")
+
+            try:
+                usage_time = datetime.fromisoformat(value)
+            except (ValueError, TypeError):
+                logger.warning("Invalid timestamp in key %s: %s", key, value)
+                keys_to_delete.append(key)
+                continue
+
+            # Parse token info from key: api_token_active:{scope}:{token}
+            suffix = key[len(ACTIVE_TOKEN_KEY_PREFIX) :]
+            parts = suffix.split(":", 1)
+            if len(parts) == 2:
+                scope_str, token = parts
+                scope = None if scope_str == "None" else scope_str
+                token_entries.append((token, scope, usage_time))
+            keys_to_delete.append(key)
+
+        if not token_entries:
+            click.echo(click.style("batch_update_api_token_last_used: no active tokens found.", fg="yellow"))
+            # Still clean up any invalid keys
+            if keys_to_delete:
+                redis_client.delete(*keys_to_delete)
+            return
+
+        # Update each token in its own short transaction to avoid long transactions
+        for token, scope, usage_time in token_entries:
+            with Session(db.engine, expire_on_commit=False) as session, session.begin():
+                stmt = (
+                    update(ApiToken)
+                    .where(
+                        ApiToken.token == token,
+                        ApiToken.type == scope,
+                        (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < usage_time)),
+                    )
+                    .values(last_used_at=usage_time)
+                )
+                result = session.execute(stmt)
+                rowcount = getattr(result, "rowcount", 0)
+                if rowcount > 0:
+                    updated_count += 1
+
+        # Delete processed keys from Redis
+        if keys_to_delete:
+            redis_client.delete(*keys_to_delete)
+
+    except Exception:
+        logger.exception("batch_update_api_token_last_used failed")
+
+    elapsed = time.perf_counter() - start_at
+    click.echo(
+        click.style(
+            f"batch_update_api_token_last_used: done. "
+            f"scanned={scanned_count}, updated={updated_count}, elapsed={elapsed:.2f}s",
+            fg="green",
+        )
+    )

api/services/api_token_service.py

@@ -0,0 +1,330 @@
+"""
+API Token Service
+
+Handles all API token caching, validation, and usage recording.
+Includes Redis cache operations, database queries, and single-flight concurrency control.
+"""
+
+import logging
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Unauthorized
+
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client, redis_fallback
+from libs.datetime_utils import naive_utc_now
+from models.model import ApiToken
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------
+# Pydantic DTO
+# ---------------------------------------------------------------------
+
+
+class CachedApiToken(BaseModel):
+    """
+    Pydantic model for cached API token data.
+
+    This is NOT a SQLAlchemy model instance, but a plain Pydantic model
+    that mimics the ApiToken model interface for read-only access.
+    """
+
+    id: str
+    app_id: str | None
+    tenant_id: str | None
+    type: str
+    token: str
+    last_used_at: datetime | None
+    created_at: datetime | None
+
+    def __repr__(self) -> str:
+        return f"<CachedApiToken id={self.id} type={self.type}>"
+
+
+# ---------------------------------------------------------------------
+# Cache configuration
+# ---------------------------------------------------------------------
+
+CACHE_KEY_PREFIX = "api_token"
+CACHE_TTL_SECONDS = 600  # 10 minutes
+CACHE_NULL_TTL_SECONDS = 60  # 1 minute for non-existent tokens
+ACTIVE_TOKEN_KEY_PREFIX = "api_token_active:"
+
+
+# ---------------------------------------------------------------------
+# Cache class
+# ---------------------------------------------------------------------
+
+
+class ApiTokenCache:
+    """
+    Redis cache wrapper for API tokens.
+    Handles serialization, deserialization, and cache invalidation.
+    """
+
+    @staticmethod
+    def make_active_key(token: str, scope: str | None = None) -> str:
+        """Generate Redis key for recording token usage."""
+        return f"{ACTIVE_TOKEN_KEY_PREFIX}{scope}:{token}"
+
+    @staticmethod
+    def _make_tenant_index_key(tenant_id: str) -> str:
+        """Generate Redis key for tenant token index."""
+        return f"tenant_tokens:{tenant_id}"
+
+    @staticmethod
+    def _make_cache_key(token: str, scope: str | None = None) -> str:
+        """Generate cache key for the given token and scope."""
+        scope_str = scope or "any"
+        return f"{CACHE_KEY_PREFIX}:{scope_str}:{token}"
+
+    @staticmethod
+    def _serialize_token(api_token: Any) -> bytes:
+        """Serialize ApiToken object to JSON bytes."""
+        if isinstance(api_token, CachedApiToken):
+            return api_token.model_dump_json().encode("utf-8")
+
+        cached = CachedApiToken(
+            id=str(api_token.id),
+            app_id=str(api_token.app_id) if api_token.app_id else None,
+            tenant_id=str(api_token.tenant_id) if api_token.tenant_id else None,
+            type=api_token.type,
+            token=api_token.token,
+            last_used_at=api_token.last_used_at,
+            created_at=api_token.created_at,
+        )
+        return cached.model_dump_json().encode("utf-8")
+
+    @staticmethod
+    def _deserialize_token(cached_data: bytes | str) -> Any:
+        """Deserialize JSON bytes/string back to a CachedApiToken Pydantic model."""
+        if cached_data in {b"null", "null"}:
+            return None
+
+        try:
+            if isinstance(cached_data, bytes):
+                cached_data = cached_data.decode("utf-8")
+            return CachedApiToken.model_validate_json(cached_data)
+        except (ValueError, Exception) as e:
+            logger.warning("Failed to deserialize token from cache: %s", e)
+            return None
+
+    @staticmethod
+    @redis_fallback(default_return=None)
+    def get(token: str, scope: str | None) -> Any | None:
+        """Get API token from cache."""
+        cache_key = ApiTokenCache._make_cache_key(token, scope)
+        cached_data = redis_client.get(cache_key)
+
+        if cached_data is None:
+            logger.debug("Cache miss for token key: %s", cache_key)
+            return None
+
+        logger.debug("Cache hit for token key: %s", cache_key)
+        return ApiTokenCache._deserialize_token(cached_data)
+
+    @staticmethod
+    def _add_to_tenant_index(tenant_id: str | None, cache_key: str) -> None:
+        """Add cache key to tenant index for efficient invalidation."""
+        if not tenant_id:
+            return
+
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            redis_client.sadd(index_key, cache_key)
+            redis_client.expire(index_key, CACHE_TTL_SECONDS + 60)
+        except Exception as e:
+            logger.warning("Failed to update tenant index: %s", e)
+
+    @staticmethod
+    def _remove_from_tenant_index(tenant_id: str | None, cache_key: str) -> None:
+        """Remove cache key from tenant index."""
+        if not tenant_id:
+            return
+
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            redis_client.srem(index_key, cache_key)
+        except Exception as e:
+            logger.warning("Failed to remove from tenant index: %s", e)
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def set(token: str, scope: str | None, api_token: Any | None, ttl: int = CACHE_TTL_SECONDS) -> bool:
+        """Set API token in cache."""
+        cache_key = ApiTokenCache._make_cache_key(token, scope)
+
+        if api_token is None:
+            cached_value = b"null"
+            ttl = CACHE_NULL_TTL_SECONDS
+        else:
+            cached_value = ApiTokenCache._serialize_token(api_token)
+
+        try:
+            redis_client.setex(cache_key, ttl, cached_value)
+
+            if api_token is not None and hasattr(api_token, "tenant_id"):
+                ApiTokenCache._add_to_tenant_index(api_token.tenant_id, cache_key)
+
+            logger.debug("Cached token with key: %s, ttl: %ss", cache_key, ttl)
+            return True
+        except Exception as e:
+            logger.warning("Failed to cache token: %s", e)
+            return False
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def delete(token: str, scope: str | None = None) -> bool:
+        """Delete API token from cache."""
+        if scope is None:
+            pattern = f"{CACHE_KEY_PREFIX}:*:{token}"
+            try:
+                keys_to_delete = list(redis_client.scan_iter(match=pattern))
+                if keys_to_delete:
+                    redis_client.delete(*keys_to_delete)
+                    logger.info("Deleted %d cache entries for token", len(keys_to_delete))
+                return True
+            except Exception as e:
+                logger.warning("Failed to delete token cache with pattern: %s", e)
+                return False
+        else:
+            cache_key = ApiTokenCache._make_cache_key(token, scope)
+            try:
+                tenant_id = None
+                try:
+                    cached_data = redis_client.get(cache_key)
+                    if cached_data and cached_data != b"null":
+                        cached_token = ApiTokenCache._deserialize_token(cached_data)
+                        if cached_token:
+                            tenant_id = cached_token.tenant_id
+                except Exception as e:
+                    logger.debug("Failed to get tenant_id for cache cleanup: %s", e)
+
+                redis_client.delete(cache_key)
+
+                if tenant_id:
+                    ApiTokenCache._remove_from_tenant_index(tenant_id, cache_key)
+
+                logger.info("Deleted cache for key: %s", cache_key)
+                return True
+            except Exception as e:
+                logger.warning("Failed to delete token cache: %s", e)
+                return False
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def invalidate_by_tenant(tenant_id: str) -> bool:
+        """Invalidate all API token caches for a specific tenant via tenant index."""
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            cache_keys = redis_client.smembers(index_key)
+
+            if cache_keys:
+                deleted_count = 0
+                for cache_key in cache_keys:
+                    if isinstance(cache_key, bytes):
+                        cache_key = cache_key.decode("utf-8")
+                    redis_client.delete(cache_key)
+                    deleted_count += 1
+
+                redis_client.delete(index_key)
+
+                logger.info(
+                    "Invalidated %d token cache entries for tenant: %s",
+                    deleted_count,
+                    tenant_id,
+                )
+            else:
+                logger.info(
+                    "No tenant index found for %s, relying on TTL expiration",
+                    tenant_id,
+                )
+
+            return True
+
+        except Exception as e:
+            logger.warning("Failed to invalidate tenant token cache: %s", e)
+            return False
+
+
+# ---------------------------------------------------------------------
+# Token usage recording (for batch update)
+# ---------------------------------------------------------------------
+
+
+def record_token_usage(auth_token: str, scope: str | None) -> None:
+    """
+    Record token usage in Redis for later batch update by a scheduled job.
+
+    Instead of dispatching a Celery task per request, we simply SET a key in Redis.
+    A Celery Beat scheduled task will periodically scan these keys and batch-update
+    last_used_at in the database.
+    """
+    try:
+        key = ApiTokenCache.make_active_key(auth_token, scope)
+        redis_client.set(key, naive_utc_now().isoformat(), ex=3600)
+    except Exception as e:
+        logger.warning("Failed to record token usage: %s", e)
+
+
+# ---------------------------------------------------------------------
+# Database query + single-flight
+# ---------------------------------------------------------------------
+
+
+def query_token_from_db(auth_token: str, scope: str | None) -> ApiToken:
+    """
+    Query API token from database and cache the result.
+
+    Raises Unauthorized if token is invalid.
+    """
+    with Session(db.engine, expire_on_commit=False) as session:
+        stmt = select(ApiToken).where(ApiToken.token == auth_token, ApiToken.type == scope)
+        api_token = session.scalar(stmt)
+
+        if not api_token:
+            ApiTokenCache.set(auth_token, scope, None)
+            raise Unauthorized("Access token is invalid")
+
+        ApiTokenCache.set(auth_token, scope, api_token)
+        record_token_usage(auth_token, scope)
+        return api_token
+
+
+def fetch_token_with_single_flight(auth_token: str, scope: str | None) -> ApiToken | Any:
+    """
+    Fetch token from DB with single-flight pattern using Redis lock.
+
+    Ensures only one concurrent request queries the database for the same token.
+    Falls back to direct query if lock acquisition fails.
+    """
+    logger.debug("Token cache miss, attempting to acquire query lock for scope: %s", scope)
+
+    lock_key = f"api_token_query_lock:{scope}:{auth_token}"
+    lock = redis_client.lock(lock_key, timeout=10, blocking_timeout=5)
+
+    try:
+        if lock.acquire(blocking=True):
+            try:
+                cached_token = ApiTokenCache.get(auth_token, scope)
+                if cached_token is not None:
+                    logger.debug("Token cached by concurrent request, using cached version")
+                    return cached_token
+
+                return query_token_from_db(auth_token, scope)
+            finally:
+                lock.release()
+        else:
+            logger.warning("Lock timeout for token: %s, proceeding with direct query", auth_token[:10])
+            return query_token_from_db(auth_token, scope)
+    except Unauthorized:
+        raise
+    except Exception as e:
+        logger.warning("Redis lock failed for token query: %s, proceeding anyway", e)
+        return query_token_from_db(auth_token, scope)

api/services/dataset_service.py

@@ -1696,13 +1696,18 @@ class DocumentService:
             for document in documents
             if document.data_source_type == "upload_file" and document.data_source_info_dict
         ]
-        if dataset.doc_form is not None:
-            batch_clean_document_task.delay(document_ids, dataset.id, dataset.doc_form, file_ids)
 
+        # Delete documents first, then dispatch cleanup task after commit
+        # to avoid deadlock between main transaction and async task
         for document in documents:
             db.session.delete(document)
         db.session.commit()
 
+        # Dispatch cleanup task after commit to avoid lock contention
+        # Task cleans up segments, files, and vector indexes
+        if dataset.doc_form is not None:
+            batch_clean_document_task.delay(document_ids, dataset.id, dataset.doc_form, file_ids)
+
     @staticmethod
     def rename_document(dataset_id: str, document_id: str, name: str) -> Document:
         assert isinstance(current_user, Account)

api/tasks/batch_clean_document_task.py

@@ -14,6 +14,9 @@ from models.model import UploadFile
 
 logger = logging.getLogger(__name__)
 
+# Batch size for database operations to keep transactions short
+BATCH_SIZE = 1000
+
 
 @shared_task(queue="dataset")
 def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form: str | None, file_ids: list[str]):
@@ -31,63 +34,179 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
     if not doc_form:
         raise ValueError("doc_form is required")
 
-    with session_factory.create_session() as session:
-        try:
-            dataset = session.query(Dataset).where(Dataset.id == dataset_id).first()
-
-            if not dataset:
-                raise Exception("Document has no dataset")
-
-            session.query(DatasetMetadataBinding).where(
-                DatasetMetadataBinding.dataset_id == dataset_id,
-                DatasetMetadataBinding.document_id.in_(document_ids),
-            ).delete(synchronize_session=False)
+    storage_keys_to_delete: list[str] = []
+    index_node_ids: list[str] = []
+    segment_ids: list[str] = []
+    total_image_upload_file_ids: list[str] = []
 
+    try:
+        # ============ Step 1: Query segment and file data (short read-only transaction) ============
+        with session_factory.create_session() as session:
+            # Get segments info
             segments = session.scalars(
                 select(DocumentSegment).where(DocumentSegment.document_id.in_(document_ids))
             ).all()
-            # check segment is exist
+
             if segments:
                 index_node_ids = [segment.index_node_id for segment in segments]
-                index_processor = IndexProcessorFactory(doc_form).init_index_processor()
-                index_processor.clean(
-                    dataset, index_node_ids, with_keywords=True, delete_child_chunks=True, delete_summaries=True
-                )
+                segment_ids = [segment.id for segment in segments]
 
+                # Collect image file IDs from segment content
                 for segment in segments:
                     image_upload_file_ids = get_image_upload_file_ids(segment.content)
-                    image_files = session.query(UploadFile).where(UploadFile.id.in_(image_upload_file_ids)).all()
-                    for image_file in image_files:
-                        try:
-                            if image_file and image_file.key:
-                                storage.delete(image_file.key)
-                        except Exception:
-                            logger.exception(
-                                "Delete image_files failed when storage deleted, \
-                                              image_upload_file_is: %s",
-                                image_file.id,
-                            )
-                    stmt = delete(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))
-                    session.execute(stmt)
-                    session.delete(segment)
+                    total_image_upload_file_ids.extend(image_upload_file_ids)
+
+            # Query storage keys for image files
+            if total_image_upload_file_ids:
+                image_files = session.scalars(
+                    select(UploadFile).where(UploadFile.id.in_(total_image_upload_file_ids))
+                ).all()
+                storage_keys_to_delete.extend([f.key for f in image_files if f and f.key])
+
+            # Query storage keys for document files
             if file_ids:
                 files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()
-                for file in files:
-                    try:
-                        storage.delete(file.key)
-                    except Exception:
-                        logger.exception("Delete file failed when document deleted, file_id: %s", file.id)
-                stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))
-                session.execute(stmt)
+                storage_keys_to_delete.extend([f.key for f in files if f and f.key])
 
-            session.commit()
-
-            end_at = time.perf_counter()
-            logger.info(
-                click.style(
-                    f"Cleaned documents when documents deleted latency: {end_at - start_at}",
-                    fg="green",
+        # ============ Step 2: Clean vector index (external service, fresh session for dataset) ============
+        if index_node_ids:
+            try:
+                # Fetch dataset in a fresh session to avoid DetachedInstanceError
+                with session_factory.create_session() as session:
+                    dataset = session.query(Dataset).where(Dataset.id == dataset_id).first()
+                    if not dataset:
+                        logger.warning("Dataset not found for vector index cleanup, dataset_id: %s", dataset_id)
+                    else:
+                        index_processor = IndexProcessorFactory(doc_form).init_index_processor()
+                        index_processor.clean(
+                            dataset, index_node_ids, with_keywords=True, delete_child_chunks=True, delete_summaries=True
+                        )
+            except Exception:
+                logger.exception(
+                    "Failed to clean vector index for dataset_id: %s, document_ids: %s, index_node_ids count: %d",
+                    dataset_id,
+                    document_ids,
+                    len(index_node_ids),
                 )
-            )
+
+        # ============ Step 3: Delete metadata binding (separate short transaction) ============
+        try:
+            with session_factory.create_session() as session:
+                deleted_count = (
+                    session.query(DatasetMetadataBinding)
+                    .where(
+                        DatasetMetadataBinding.dataset_id == dataset_id,
+                        DatasetMetadataBinding.document_id.in_(document_ids),
+                    )
+                    .delete(synchronize_session=False)
+                )
+                session.commit()
+                logger.debug("Deleted %d metadata bindings for dataset_id: %s", deleted_count, dataset_id)
         except Exception:
-            logger.exception("Cleaned documents when documents deleted failed")
+            logger.exception(
+                "Failed to delete metadata bindings for dataset_id: %s, document_ids: %s",
+                dataset_id,
+                document_ids,
+            )
+
+        # ============ Step 4: Batch delete UploadFile records (multiple short transactions) ============
+        if total_image_upload_file_ids:
+            failed_batches = 0
+            total_batches = (len(total_image_upload_file_ids) + BATCH_SIZE - 1) // BATCH_SIZE
+            for i in range(0, len(total_image_upload_file_ids), BATCH_SIZE):
+                batch = total_image_upload_file_ids[i : i + BATCH_SIZE]
+                try:
+                    with session_factory.create_session() as session:
+                        stmt = delete(UploadFile).where(UploadFile.id.in_(batch))
+                        session.execute(stmt)
+                        session.commit()
+                except Exception:
+                    failed_batches += 1
+                    logger.exception(
+                        "Failed to delete image UploadFile batch %d-%d for dataset_id: %s",
+                        i,
+                        i + len(batch),
+                        dataset_id,
+                    )
+            if failed_batches > 0:
+                logger.warning(
+                    "Image UploadFile deletion: %d/%d batches failed for dataset_id: %s",
+                    failed_batches,
+                    total_batches,
+                    dataset_id,
+                )
+
+        # ============ Step 5: Batch delete DocumentSegment records (multiple short transactions) ============
+        if segment_ids:
+            failed_batches = 0
+            total_batches = (len(segment_ids) + BATCH_SIZE - 1) // BATCH_SIZE
+            for i in range(0, len(segment_ids), BATCH_SIZE):
+                batch = segment_ids[i : i + BATCH_SIZE]
+                try:
+                    with session_factory.create_session() as session:
+                        segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(batch))
+                        session.execute(segment_delete_stmt)
+                        session.commit()
+                except Exception:
+                    failed_batches += 1
+                    logger.exception(
+                        "Failed to delete DocumentSegment batch %d-%d for dataset_id: %s, document_ids: %s",
+                        i,
+                        i + len(batch),
+                        dataset_id,
+                        document_ids,
+                    )
+            if failed_batches > 0:
+                logger.warning(
+                    "DocumentSegment deletion: %d/%d batches failed, document_ids: %s",
+                    failed_batches,
+                    total_batches,
+                    document_ids,
+                )
+
+        # ============ Step 6: Delete document-associated files (separate short transaction) ============
+        if file_ids:
+            try:
+                with session_factory.create_session() as session:
+                    stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))
+                    session.execute(stmt)
+                    session.commit()
+            except Exception:
+                logger.exception(
+                    "Failed to delete document UploadFile records for dataset_id: %s, file_ids: %s",
+                    dataset_id,
+                    file_ids,
+                )
+
+        # ============ Step 7: Delete storage files (I/O operations, no DB transaction) ============
+        storage_delete_failures = 0
+        for storage_key in storage_keys_to_delete:
+            try:
+                storage.delete(storage_key)
+            except Exception:
+                storage_delete_failures += 1
+                logger.exception("Failed to delete file from storage, key: %s", storage_key)
+        if storage_delete_failures > 0:
+            logger.warning(
+                "Storage file deletion completed with %d failures out of %d total files for dataset_id: %s",
+                storage_delete_failures,
+                len(storage_keys_to_delete),
+                dataset_id,
+            )
+
+        end_at = time.perf_counter()
+        logger.info(
+            click.style(
+                f"Cleaned documents when documents deleted latency: {end_at - start_at:.2f}s, "
+                f"dataset_id: {dataset_id}, document_ids: {document_ids}, "
+                f"segments: {len(segment_ids)}, image_files: {len(total_image_upload_file_ids)}, "
+                f"storage_files: {len(storage_keys_to_delete)}",
+                fg="green",
+            )
+        )
+    except Exception:
+        logger.exception(
+            "Batch clean documents failed for dataset_id: %s, document_ids: %s",
+            dataset_id,
+            document_ids,
+        )

api/tasks/delete_segment_from_index_task.py

@@ -3,6 +3,7 @@ import time
 
 import click
 from celery import shared_task
+from sqlalchemy import delete
 
 from core.db.session_factory import session_factory
 from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
@@ -67,8 +68,14 @@ def delete_segment_from_index_task(
                 if segment_attachment_bindings:
                     attachment_ids = [binding.attachment_id for binding in segment_attachment_bindings]
                     index_processor.clean(dataset=dataset, node_ids=attachment_ids, with_keywords=False)
-                    for binding in segment_attachment_bindings:
-                        session.delete(binding)
+                    segment_attachment_bind_ids = [i.id for i in segment_attachment_bindings]
+
+                    for i in range(0, len(segment_attachment_bind_ids), 1000):
+                        segment_attachment_bind_delete_stmt = delete(SegmentAttachmentBinding).where(
+                            SegmentAttachmentBinding.id.in_(segment_attachment_bind_ids[i : i + 1000])
+                        )
+                        session.execute(segment_attachment_bind_delete_stmt)
+
                     # delete upload file
                     session.query(UploadFile).where(UploadFile.id.in_(attachment_ids)).delete(synchronize_session=False)
                     session.commit()

api/tasks/document_indexing_sync_task.py

@@ -28,7 +28,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
     logger.info(click.style(f"Start sync document: {document_id}", fg="green"))
     start_at = time.perf_counter()
 
-    with session_factory.create_session() as session:
+    with session_factory.create_session() as session, session.begin():
         document = session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
 
         if not document:
@@ -68,7 +68,6 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
                 document.indexing_status = "error"
                 document.error = "Datasource credential not found. Please reconnect your Notion workspace."
                 document.stopped_at = naive_utc_now()
-                session.commit()
                 return
 
             loader = NotionExtractor(
@@ -85,7 +84,6 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
             if last_edited_time != page_edited_time:
                 document.indexing_status = "parsing"
                 document.processing_started_at = naive_utc_now()
-                session.commit()
 
                 # delete all document segment and index
                 try:

api/tasks/process_tenant_plugin_autoupgrade_check_task.py

@@ -6,8 +6,8 @@ import typing
 import click
 from celery import shared_task
 
-from core.helper import marketplace
-from core.helper.marketplace import MarketplacePluginDeclaration
+from core.helper.marketplace import record_install_plugin_event
+from core.plugin.entities.marketplace import MarketplacePluginSnapshot
 from core.plugin.entities.plugin import PluginInstallationSource
 from core.plugin.impl.plugin import PluginInstaller
 from extensions.ext_redis import redis_client
@@ -16,7 +16,7 @@ from models.account import TenantPluginAutoUpgradeStrategy
 logger = logging.getLogger(__name__)
 
 RETRY_TIMES_OF_ONE_PLUGIN_IN_ONE_TENANT = 3
-CACHE_REDIS_KEY_PREFIX = "plugin_autoupgrade_check_task:cached_plugin_manifests:"
+CACHE_REDIS_KEY_PREFIX = "plugin_autoupgrade_check_task:cached_plugin_snapshot:"
 CACHE_REDIS_TTL = 60 * 60  # 1 hour
 
 
@@ -25,11 +25,11 @@ def _get_redis_cache_key(plugin_id: str) -> str:
     return f"{CACHE_REDIS_KEY_PREFIX}{plugin_id}"
 
 
-def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginDeclaration, None, bool]:
+def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginSnapshot, None, bool]:
     """
     Get cached plugin manifest from Redis.
     Returns:
-        - MarketplacePluginDeclaration: if found in cache
+        - MarketplacePluginSnapshot: if found in cache
         - None: if cached as not found (marketplace returned no result)
         - False: if not in cache at all
     """
@@ -43,76 +43,31 @@ def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginDeclar
         if cached_json is None:
             return None
 
-        return MarketplacePluginDeclaration.model_validate(cached_json)
+        return MarketplacePluginSnapshot.model_validate(cached_json)
     except Exception:
         logger.exception("Failed to get cached manifest for plugin %s", plugin_id)
         return False
 
 
-def _set_cached_manifest(plugin_id: str, manifest: typing.Union[MarketplacePluginDeclaration, None]) -> None:
-    """
-    Cache plugin manifest in Redis.
-    Args:
-        plugin_id: The plugin ID
-        manifest: The manifest to cache, or None if not found in marketplace
-    """
-    try:
-        key = _get_redis_cache_key(plugin_id)
-        if manifest is None:
-            # Cache the fact that this plugin was not found
-            redis_client.setex(key, CACHE_REDIS_TTL, json.dumps(None))
-        else:
-            # Cache the manifest data
-            redis_client.setex(key, CACHE_REDIS_TTL, manifest.model_dump_json())
-    except Exception:
-        # If Redis fails, continue without caching
-        # traceback.print_exc()
-        logger.exception("Failed to set cached manifest for plugin %s", plugin_id)
-
-
 def marketplace_batch_fetch_plugin_manifests(
     plugin_ids_plain_list: list[str],
-) -> list[MarketplacePluginDeclaration]:
-    """Fetch plugin manifests with Redis caching support."""
-    cached_manifests: dict[str, typing.Union[MarketplacePluginDeclaration, None]] = {}
-    not_cached_plugin_ids: list[str] = []
+) -> list[MarketplacePluginSnapshot]:
+    """
+    Fetch plugin manifests from Redis cache only.
+    This function assumes fetch_global_plugin_manifest() has been called
+    to pre-populate the cache with all marketplace plugins.
+    """
+    result: list[MarketplacePluginSnapshot] = []
 
     # Check Redis cache for each plugin
     for plugin_id in plugin_ids_plain_list:
         cached_result = _get_cached_manifest(plugin_id)
-        if cached_result is False:
-            # Not in cache, need to fetch
-            not_cached_plugin_ids.append(plugin_id)
-        else:
-            # Either found manifest or cached as None (not found in marketplace)
-            # At this point, cached_result is either MarketplacePluginDeclaration or None
-            if isinstance(cached_result, bool):
-                # This should never happen due to the if condition above, but for type safety
-                continue
-            cached_manifests[plugin_id] = cached_result
+        if not isinstance(cached_result, MarketplacePluginSnapshot):
+            # cached_result is False (not in cache) or None (cached as not found)
+            logger.warning("plugin %s not found in cache, skipping", plugin_id)
+            continue
 
-    # Fetch uncached plugins from marketplace
-    if not_cached_plugin_ids:
-        manifests = marketplace.batch_fetch_plugin_manifests_ignore_deserialization_error(not_cached_plugin_ids)
-
-        # Cache the fetched manifests
-        for manifest in manifests:
-            cached_manifests[manifest.plugin_id] = manifest
-            _set_cached_manifest(manifest.plugin_id, manifest)
-
-        # Cache plugins that were not found in marketplace
-        fetched_plugin_ids = {manifest.plugin_id for manifest in manifests}
-        for plugin_id in not_cached_plugin_ids:
-            if plugin_id not in fetched_plugin_ids:
-                cached_manifests[plugin_id] = None
-                _set_cached_manifest(plugin_id, None)
-
-    # Build result list from cached manifests
-    result: list[MarketplacePluginDeclaration] = []
-    for plugin_id in plugin_ids_plain_list:
-        cached_manifest: typing.Union[MarketplacePluginDeclaration, None] = cached_manifests.get(plugin_id)
-        if cached_manifest is not None:
-            result.append(cached_manifest)
+        result.append(cached_result)
 
     return result
 
@@ -211,7 +166,7 @@ def process_tenant_plugin_autoupgrade_check_task(
                         # execute upgrade
                         new_unique_identifier = manifest.latest_package_identifier
 
-                        marketplace.record_install_plugin_event(new_unique_identifier)
+                        record_install_plugin_event(new_unique_identifier)
                         click.echo(
                             click.style(
                                 f"Upgrade plugin: {original_unique_identifier} -> {new_unique_identifier}",

api/tasks/remove_app_and_related_data_task.py

@@ -48,6 +48,7 @@ from models.workflow import (
     WorkflowArchiveLog,
 )
 from repositories.factory import DifyAPIRepositoryFactory
+from services.api_token_service import ApiTokenCache
 
 logger = logging.getLogger(__name__)
 
@@ -134,6 +135,12 @@ def _delete_app_mcp_servers(tenant_id: str, app_id: str):
 
 def _delete_app_api_tokens(tenant_id: str, app_id: str):
     def del_api_token(session, api_token_id: str):
+        # Fetch token details for cache invalidation
+        token_obj = session.query(ApiToken).where(ApiToken.id == api_token_id).first()
+        if token_obj:
+            # Invalidate cache before deletion
+            ApiTokenCache.delete(token_obj.token, token_obj.type)
+
         session.query(ApiToken).where(ApiToken.id == api_token_id).delete(synchronize_session=False)
 
     _delete_records(

api/tests/integration_tests/libs/test_api_token_cache_integration.py

@@ -0,0 +1,375 @@
+"""
+Integration tests for API Token Cache with Redis.
+
+These tests require:
+- Redis server running
+- Test database configured
+"""
+
+import time
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import pytest
+
+from extensions.ext_redis import redis_client
+from models.model import ApiToken
+from services.api_token_service import ApiTokenCache, CachedApiToken
+
+
+class TestApiTokenCacheRedisIntegration:
+    """Integration tests with real Redis."""
+
+    def setup_method(self):
+        """Setup test fixtures and clean Redis."""
+        self.test_token = "test-integration-token-123"
+        self.test_scope = "app"
+        self.cache_key = f"api_token:{self.test_scope}:{self.test_token}"
+
+        # Clean up any existing test data
+        self._cleanup()
+
+    def teardown_method(self):
+        """Cleanup test data from Redis."""
+        self._cleanup()
+
+    def _cleanup(self):
+        """Remove test data from Redis."""
+        try:
+            redis_client.delete(self.cache_key)
+            redis_client.delete(ApiTokenCache._make_tenant_index_key("test-tenant-id"))
+            redis_client.delete(ApiTokenCache.make_active_key(self.test_token, self.test_scope))
+        except Exception:
+            pass  # Ignore cleanup errors
+
+    def test_cache_set_and_get_with_real_redis(self):
+        """Test cache set and get operations with real Redis."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id-123"
+        mock_token.app_id = "test-app-456"
+        mock_token.tenant_id = "test-tenant-789"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = datetime.now()
+        mock_token.created_at = datetime.now() - timedelta(days=30)
+
+        # Set in cache
+        result = ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+        assert result is True
+
+        # Verify in Redis
+        cached_data = redis_client.get(self.cache_key)
+        assert cached_data is not None
+
+        # Get from cache
+        cached_token = ApiTokenCache.get(self.test_token, self.test_scope)
+        assert cached_token is not None
+        assert isinstance(cached_token, CachedApiToken)
+        assert cached_token.id == "test-id-123"
+        assert cached_token.app_id == "test-app-456"
+        assert cached_token.tenant_id == "test-tenant-789"
+        assert cached_token.type == "app"
+        assert cached_token.token == self.test_token
+
+    def test_cache_ttl_with_real_redis(self):
+        """Test cache TTL is set correctly."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        ttl = redis_client.ttl(self.cache_key)
+        assert 595 <= ttl <= 600  # Should be around 600 seconds (10 minutes)
+
+    def test_cache_null_value_for_invalid_token(self):
+        """Test caching null value for invalid tokens."""
+        result = ApiTokenCache.set(self.test_token, self.test_scope, None)
+        assert result is True
+
+        cached_data = redis_client.get(self.cache_key)
+        assert cached_data == b"null"
+
+        cached_token = ApiTokenCache.get(self.test_token, self.test_scope)
+        assert cached_token is None
+
+        ttl = redis_client.ttl(self.cache_key)
+        assert 55 <= ttl <= 60
+
+    def test_cache_delete_with_real_redis(self):
+        """Test cache deletion with real Redis."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+        assert redis_client.exists(self.cache_key) == 1
+
+        result = ApiTokenCache.delete(self.test_token, self.test_scope)
+        assert result is True
+        assert redis_client.exists(self.cache_key) == 0
+
+    def test_tenant_index_creation(self):
+        """Test tenant index is created when caching token."""
+        from unittest.mock import MagicMock
+
+        tenant_id = "test-tenant-id"
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = tenant_id
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+        assert redis_client.exists(index_key) == 1
+
+        members = redis_client.smembers(index_key)
+        cache_keys = [m.decode("utf-8") if isinstance(m, bytes) else m for m in members]
+        assert self.cache_key in cache_keys
+
+    def test_invalidate_by_tenant_via_index(self):
+        """Test tenant-wide cache invalidation using index (fast path)."""
+        from unittest.mock import MagicMock
+
+        tenant_id = "test-tenant-id"
+
+        for i in range(3):
+            token_value = f"test-token-{i}"
+            mock_token = MagicMock()
+            mock_token.id = f"test-id-{i}"
+            mock_token.app_id = "test-app"
+            mock_token.tenant_id = tenant_id
+            mock_token.type = "app"
+            mock_token.token = token_value
+            mock_token.last_used_at = None
+            mock_token.created_at = datetime.now()
+
+            ApiTokenCache.set(token_value, "app", mock_token)
+
+        for i in range(3):
+            key = f"api_token:app:test-token-{i}"
+            assert redis_client.exists(key) == 1
+
+        result = ApiTokenCache.invalidate_by_tenant(tenant_id)
+        assert result is True
+
+        for i in range(3):
+            key = f"api_token:app:test-token-{i}"
+            assert redis_client.exists(key) == 0
+
+        assert redis_client.exists(ApiTokenCache._make_tenant_index_key(tenant_id)) == 0
+
+    def test_concurrent_cache_access(self):
+        """Test concurrent cache access doesn't cause issues."""
+        import concurrent.futures
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        def get_from_cache():
+            return ApiTokenCache.get(self.test_token, self.test_scope)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            futures = [executor.submit(get_from_cache) for _ in range(50)]
+            results = [f.result() for f in concurrent.futures.as_completed(futures)]
+
+        assert len(results) == 50
+        assert all(r is not None for r in results)
+        assert all(isinstance(r, CachedApiToken) for r in results)
+
+
+class TestTokenUsageRecording:
+    """Tests for recording token usage in Redis (batch update approach)."""
+
+    def setup_method(self):
+        self.test_token = "test-usage-token"
+        self.test_scope = "app"
+        self.active_key = ApiTokenCache.make_active_key(self.test_token, self.test_scope)
+
+    def teardown_method(self):
+        try:
+            redis_client.delete(self.active_key)
+        except Exception:
+            pass
+
+    def test_record_token_usage_sets_redis_key(self):
+        """Test that record_token_usage writes an active key to Redis."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+
+        # Key should exist
+        assert redis_client.exists(self.active_key) == 1
+
+        # Value should be an ISO timestamp
+        value = redis_client.get(self.active_key)
+        if isinstance(value, bytes):
+            value = value.decode("utf-8")
+        datetime.fromisoformat(value)  # Should not raise
+
+    def test_record_token_usage_has_ttl(self):
+        """Test that active keys have a TTL as safety net."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+
+        ttl = redis_client.ttl(self.active_key)
+        assert 3595 <= ttl <= 3600  # ~1 hour
+
+    def test_record_token_usage_overwrites(self):
+        """Test that repeated calls overwrite the same key (no accumulation)."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+        first_value = redis_client.get(self.active_key)
+
+        time.sleep(0.01)  # Tiny delay so timestamp differs
+
+        record_token_usage(self.test_token, self.test_scope)
+        second_value = redis_client.get(self.active_key)
+
+        # Key count should still be 1 (overwritten, not accumulated)
+        assert redis_client.exists(self.active_key) == 1
+
+
+class TestEndToEndCacheFlow:
+    """End-to-end integration test for complete cache flow."""
+
+    @pytest.mark.usefixtures("db_session")
+    def test_complete_flow_cache_miss_then_hit(self, db_session):
+        """
+        Test complete flow:
+        1. First request (cache miss) -> query DB -> cache result
+        2. Second request (cache hit) -> return from cache
+        3. Verify Redis state
+        """
+        test_token_value = "test-e2e-token"
+        test_scope = "app"
+
+        test_token = ApiToken()
+        test_token.id = "test-e2e-id"
+        test_token.token = test_token_value
+        test_token.type = test_scope
+        test_token.app_id = "test-app"
+        test_token.tenant_id = "test-tenant"
+        test_token.last_used_at = None
+        test_token.created_at = datetime.now()
+
+        db_session.add(test_token)
+        db_session.commit()
+
+        try:
+            # Step 1: Cache miss - set token in cache
+            ApiTokenCache.set(test_token_value, test_scope, test_token)
+
+            cache_key = f"api_token:{test_scope}:{test_token_value}"
+            assert redis_client.exists(cache_key) == 1
+
+            # Step 2: Cache hit - get from cache
+            cached_token = ApiTokenCache.get(test_token_value, test_scope)
+            assert cached_token is not None
+            assert cached_token.id == test_token.id
+            assert cached_token.token == test_token_value
+
+            # Step 3: Verify tenant index
+            index_key = ApiTokenCache._make_tenant_index_key(test_token.tenant_id)
+            assert redis_client.exists(index_key) == 1
+            assert cache_key.encode() in redis_client.smembers(index_key)
+
+            # Step 4: Delete and verify cleanup
+            ApiTokenCache.delete(test_token_value, test_scope)
+            assert redis_client.exists(cache_key) == 0
+            assert cache_key.encode() not in redis_client.smembers(index_key)
+
+        finally:
+            db_session.delete(test_token)
+            db_session.commit()
+            redis_client.delete(f"api_token:{test_scope}:{test_token_value}")
+            redis_client.delete(ApiTokenCache._make_tenant_index_key(test_token.tenant_id))
+
+    def test_high_concurrency_simulation(self):
+        """Simulate high concurrency access to cache."""
+        import concurrent.futures
+        from unittest.mock import MagicMock
+
+        test_token_value = "test-concurrent-token"
+        test_scope = "app"
+
+        mock_token = MagicMock()
+        mock_token.id = "concurrent-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = test_scope
+        mock_token.token = test_token_value
+        mock_token.last_used_at = datetime.now()
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(test_token_value, test_scope, mock_token)
+
+        try:
+
+            def read_cache():
+                return ApiTokenCache.get(test_token_value, test_scope)
+
+            start_time = time.time()
+            with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
+                futures = [executor.submit(read_cache) for _ in range(100)]
+                results = [f.result() for f in concurrent.futures.as_completed(futures)]
+            elapsed = time.time() - start_time
+
+            assert len(results) == 100
+            assert all(r is not None for r in results)
+
+            assert elapsed < 1.0, f"Too slow: {elapsed}s for 100 cache reads"
+
+        finally:
+            ApiTokenCache.delete(test_token_value, test_scope)
+            redis_client.delete(ApiTokenCache._make_tenant_index_key(mock_token.tenant_id))
+
+
+class TestRedisFailover:
+    """Test behavior when Redis is unavailable."""
+
+    @patch("services.api_token_service.redis_client")
+    def test_graceful_degradation_when_redis_fails(self, mock_redis):
+        """Test system degrades gracefully when Redis is unavailable."""
+        from redis import RedisError
+
+        mock_redis.get.side_effect = RedisError("Connection failed")
+        mock_redis.setex.side_effect = RedisError("Connection failed")
+
+        result_get = ApiTokenCache.get("test-token", "app")
+        assert result_get is None
+
+        result_set = ApiTokenCache.set("test-token", "app", None)
+        assert result_set is False

api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py

@@ -217,7 +217,6 @@ class TestTemplateTransformNode:
     @patch(
         "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template"
     )
-    @patch("core.workflow.nodes.template_transform.template_transform_node.MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH", 10)
     def test_run_output_length_exceeds_limit(
         self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params
     ):
@@ -231,6 +230,7 @@ class TestTemplateTransformNode:
             graph_init_params=graph_init_params,
             graph=mock_graph,
             graph_runtime_state=mock_graph_runtime_state,
+            max_output_length=10,
         )
 
         result = node._run()

api/tests/unit_tests/extensions/test_celery_ssl.py

@@ -132,6 +132,8 @@ class TestCelerySSLConfiguration:
         mock_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK = 0
         mock_config.ENABLE_TRIGGER_PROVIDER_REFRESH_TASK = False
         mock_config.TRIGGER_PROVIDER_REFRESH_INTERVAL = 15
+        mock_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK = False
+        mock_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL = 30
 
         with patch("extensions.ext_celery.dify_config", mock_config):
             from dify_app import DifyApp

api/tests/unit_tests/libs/test_api_token_cache.py

@@ -0,0 +1,250 @@
+"""
+Unit tests for API Token Cache module.
+"""
+
+import json
+from datetime import datetime
+from unittest.mock import MagicMock, patch
+
+from services.api_token_service import (
+    CACHE_KEY_PREFIX,
+    CACHE_NULL_TTL_SECONDS,
+    CACHE_TTL_SECONDS,
+    ApiTokenCache,
+    CachedApiToken,
+)
+
+
+class TestApiTokenCache:
+    """Test cases for ApiTokenCache class."""
+
+    def setup_method(self):
+        """Setup test fixtures."""
+        self.mock_token = MagicMock()
+        self.mock_token.id = "test-token-id-123"
+        self.mock_token.app_id = "test-app-id-456"
+        self.mock_token.tenant_id = "test-tenant-id-789"
+        self.mock_token.type = "app"
+        self.mock_token.token = "test-token-value-abc"
+        self.mock_token.last_used_at = datetime(2026, 2, 3, 10, 0, 0)
+        self.mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+    def test_make_cache_key(self):
+        """Test cache key generation."""
+        # Test with scope
+        key = ApiTokenCache._make_cache_key("my-token", "app")
+        assert key == f"{CACHE_KEY_PREFIX}:app:my-token"
+
+        # Test without scope
+        key = ApiTokenCache._make_cache_key("my-token", None)
+        assert key == f"{CACHE_KEY_PREFIX}:any:my-token"
+
+    def test_serialize_token(self):
+        """Test token serialization."""
+        serialized = ApiTokenCache._serialize_token(self.mock_token)
+        data = json.loads(serialized)
+
+        assert data["id"] == "test-token-id-123"
+        assert data["app_id"] == "test-app-id-456"
+        assert data["tenant_id"] == "test-tenant-id-789"
+        assert data["type"] == "app"
+        assert data["token"] == "test-token-value-abc"
+        assert data["last_used_at"] == "2026-02-03T10:00:00"
+        assert data["created_at"] == "2026-01-01T00:00:00"
+
+    def test_serialize_token_with_nulls(self):
+        """Test token serialization with None values."""
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = None
+        mock_token.tenant_id = None
+        mock_token.type = "dataset"
+        mock_token.token = "test-token"
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+        serialized = ApiTokenCache._serialize_token(mock_token)
+        data = json.loads(serialized)
+
+        assert data["app_id"] is None
+        assert data["tenant_id"] is None
+        assert data["last_used_at"] is None
+
+    def test_deserialize_token(self):
+        """Test token deserialization."""
+        cached_data = json.dumps(
+            {
+                "id": "test-id",
+                "app_id": "test-app",
+                "tenant_id": "test-tenant",
+                "type": "app",
+                "token": "test-token",
+                "last_used_at": "2026-02-03T10:00:00",
+                "created_at": "2026-01-01T00:00:00",
+            }
+        )
+
+        result = ApiTokenCache._deserialize_token(cached_data)
+
+        assert isinstance(result, CachedApiToken)
+        assert result.id == "test-id"
+        assert result.app_id == "test-app"
+        assert result.tenant_id == "test-tenant"
+        assert result.type == "app"
+        assert result.token == "test-token"
+        assert result.last_used_at == datetime(2026, 2, 3, 10, 0, 0)
+        assert result.created_at == datetime(2026, 1, 1, 0, 0, 0)
+
+    def test_deserialize_null_token(self):
+        """Test deserialization of null token (cached miss)."""
+        result = ApiTokenCache._deserialize_token("null")
+        assert result is None
+
+    def test_deserialize_invalid_json(self):
+        """Test deserialization with invalid JSON."""
+        result = ApiTokenCache._deserialize_token("invalid-json{")
+        assert result is None
+
+    @patch("services.api_token_service.redis_client")
+    def test_get_cache_hit(self, mock_redis):
+        """Test cache hit scenario."""
+        cached_data = json.dumps(
+            {
+                "id": "test-id",
+                "app_id": "test-app",
+                "tenant_id": "test-tenant",
+                "type": "app",
+                "token": "test-token",
+                "last_used_at": "2026-02-03T10:00:00",
+                "created_at": "2026-01-01T00:00:00",
+            }
+        ).encode("utf-8")
+        mock_redis.get.return_value = cached_data
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        assert result is not None
+        assert isinstance(result, CachedApiToken)
+        assert result.app_id == "test-app"
+        mock_redis.get.assert_called_once_with(f"{CACHE_KEY_PREFIX}:app:test-token")
+
+    @patch("services.api_token_service.redis_client")
+    def test_get_cache_miss(self, mock_redis):
+        """Test cache miss scenario."""
+        mock_redis.get.return_value = None
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        assert result is None
+        mock_redis.get.assert_called_once()
+
+    @patch("services.api_token_service.redis_client")
+    def test_set_valid_token(self, mock_redis):
+        """Test setting a valid token in cache."""
+        result = ApiTokenCache.set("test-token", "app", self.mock_token)
+
+        assert result is True
+        mock_redis.setex.assert_called_once()
+        args = mock_redis.setex.call_args[0]
+        assert args[0] == f"{CACHE_KEY_PREFIX}:app:test-token"
+        assert args[1] == CACHE_TTL_SECONDS
+
+    @patch("services.api_token_service.redis_client")
+    def test_set_null_token(self, mock_redis):
+        """Test setting a null token (cache penetration prevention)."""
+        result = ApiTokenCache.set("invalid-token", "app", None)
+
+        assert result is True
+        mock_redis.setex.assert_called_once()
+        args = mock_redis.setex.call_args[0]
+        assert args[0] == f"{CACHE_KEY_PREFIX}:app:invalid-token"
+        assert args[1] == CACHE_NULL_TTL_SECONDS
+        assert args[2] == b"null"
+
+    @patch("services.api_token_service.redis_client")
+    def test_delete_with_scope(self, mock_redis):
+        """Test deleting token cache with specific scope."""
+        result = ApiTokenCache.delete("test-token", "app")
+
+        assert result is True
+        mock_redis.delete.assert_called_once_with(f"{CACHE_KEY_PREFIX}:app:test-token")
+
+    @patch("services.api_token_service.redis_client")
+    def test_delete_without_scope(self, mock_redis):
+        """Test deleting token cache without scope (delete all)."""
+        # Mock scan_iter to return an iterator of keys
+        mock_redis.scan_iter.return_value = iter(
+            [
+                b"api_token:app:test-token",
+                b"api_token:dataset:test-token",
+            ]
+        )
+
+        result = ApiTokenCache.delete("test-token", None)
+
+        assert result is True
+        # Verify scan_iter was called with the correct pattern
+        mock_redis.scan_iter.assert_called_once()
+        call_args = mock_redis.scan_iter.call_args
+        assert call_args[1]["match"] == f"{CACHE_KEY_PREFIX}:*:test-token"
+
+        # Verify delete was called with all matched keys
+        mock_redis.delete.assert_called_once_with(
+            b"api_token:app:test-token",
+            b"api_token:dataset:test-token",
+        )
+
+    @patch("services.api_token_service.redis_client")
+    def test_redis_fallback_on_exception(self, mock_redis):
+        """Test Redis fallback when Redis is unavailable."""
+        from redis import RedisError
+
+        mock_redis.get.side_effect = RedisError("Connection failed")
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        # Should return None (fallback) instead of raising exception
+        assert result is None
+
+
+class TestApiTokenCacheIntegration:
+    """Integration test scenarios."""
+
+    @patch("services.api_token_service.redis_client")
+    def test_full_cache_lifecycle(self, mock_redis):
+        """Test complete cache lifecycle: set -> get -> delete."""
+        # Setup mock token
+        mock_token = MagicMock()
+        mock_token.id = "id-123"
+        mock_token.app_id = "app-456"
+        mock_token.tenant_id = "tenant-789"
+        mock_token.type = "app"
+        mock_token.token = "token-abc"
+        mock_token.last_used_at = datetime(2026, 2, 3, 10, 0, 0)
+        mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+        # 1. Set token in cache
+        ApiTokenCache.set("token-abc", "app", mock_token)
+        assert mock_redis.setex.called
+
+        # 2. Simulate cache hit
+        cached_data = ApiTokenCache._serialize_token(mock_token)
+        mock_redis.get.return_value = cached_data  # bytes from model_dump_json().encode()
+
+        retrieved = ApiTokenCache.get("token-abc", "app")
+        assert retrieved is not None
+        assert isinstance(retrieved, CachedApiToken)
+
+        # 3. Delete from cache
+        ApiTokenCache.delete("token-abc", "app")
+        assert mock_redis.delete.called
+
+    @patch("services.api_token_service.redis_client")
+    def test_cache_penetration_prevention(self, mock_redis):
+        """Test that non-existent tokens are cached as null."""
+        # Set null token (cache miss)
+        ApiTokenCache.set("non-existent-token", "app", None)
+
+        args = mock_redis.setex.call_args[0]
+        assert args[2] == b"null"
+        assert args[1] == CACHE_NULL_TTL_SECONDS  # Shorter TTL for null values

api/tests/unit_tests/tasks/test_document_indexing_sync_task.py

@@ -114,6 +114,21 @@ def mock_db_session():
         session = MagicMock()
         # Ensure tests can observe session.close() via context manager teardown
         session.close = MagicMock()
+        session.commit = MagicMock()
+
+        # Mock session.begin() context manager to auto-commit on exit
+        begin_cm = MagicMock()
+        begin_cm.__enter__.return_value = session
+
+        def _begin_exit_side_effect(*args, **kwargs):
+            # session.begin().__exit__() should commit if no exception
+            if args[0] is None:  # No exception
+                session.commit()
+
+        begin_cm.__exit__.side_effect = _begin_exit_side_effect
+        session.begin.return_value = begin_cm
+
+        # Mock create_session() context manager
         cm = MagicMock()
         cm.__enter__.return_value = session
 

web/app/components/app/configuration/config/agent/agent-tools/index.tsx

@@ -109,6 +109,7 @@ const AgentTools: FC = () => {
       tool_parameters: paramsWithDefaultValue,
       notAuthor: !tool.is_team_authorization,
       enabled: true,
+      type: tool.provider_type as CollectionType,
     }
   }
   const handleSelectTool = (tool: ToolDefaultValue) => {

web/app/components/base/effect/index.stories.tsx

@@ -1,4 +1,3 @@
-/* eslint-disable tailwindcss/classnames-order */
 import type { Meta, StoryObj } from '@storybook/nextjs-vite'
 import Effect from '.'
 
@@ -29,8 +28,8 @@ type Story = StoryObj<typeof meta>
 export const Playground: Story = {
   render: () => (
     <div className="relative h-40 w-72 overflow-hidden rounded-2xl border border-divider-subtle bg-background-default-subtle">
-      <Effect className="top-6 left-8" />
-      <Effect className="top-14 right-10 bg-util-colors-purple-brand-purple-brand-500" />
+      <Effect className="left-8 top-6" />
+      <Effect className="bg-util-colors-purple-brand-purple-brand-500 right-10 top-14" />
       <div className="absolute inset-x-0 bottom-4 flex justify-center text-xs text-text-secondary">
         Accent glow
       </div>

web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/error-message.tsx

@@ -14,7 +14,6 @@ const ErrorMessage = ({
   errorMsg,
 }: ErrorMessageProps) => {
   return (
-    // eslint-disable-next-line tailwindcss/migration-from-tailwind-2
     <div className={cn(
       'flex gap-x-0.5 rounded-xl border-[0.5px] border-components-panel-border bg-opacity-40 bg-toast-error-bg p-2 shadow-xs shadow-shadow-shadow-3',
       className,

web/app/components/plugins/plugin-detail-panel/tool-selector/hooks/use-tool-selector-state.ts

@@ -129,6 +129,7 @@ export const useToolSelectorState = ({
       extra: {
         description: tool.tool_description,
       },
+      type: tool.provider_type,
     }
   }, [])
 

web/app/components/workflow/block-selector/types.ts

@@ -87,6 +87,7 @@ export type ToolValue = {
   enabled?: boolean
   extra?: { description?: string } & Record<string, unknown>
   credential_id?: string
+  type?: string
 }
 
 export type DataSourceItem = {

web/app/serwist/[path]/route.ts

@@ -1,14 +1,11 @@
-import { spawnSync } from 'node:child_process'
-import { randomUUID } from 'node:crypto'
 import { createSerwistRoute } from '@serwist/turbopack'
 
 const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''
-const revision = spawnSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8' }).stdout?.trim() || randomUUID()
 
 export const { dynamic, dynamicParams, revalidate, generateStaticParams, GET } = createSerwistRoute({
-  additionalPrecacheEntries: [{ url: `${basePath}/_offline.html`, revision }],
   swSrc: 'app/sw.ts',
   nextConfig: {
     basePath,
   },
+  useNativeEsbuild: true,
 })

web/app/sw.ts

@@ -3,7 +3,9 @@
 /// <reference lib="webworker" />
 
 import type { PrecacheEntry, SerwistGlobalConfig } from 'serwist'
-import { CacheableResponsePlugin, CacheFirst, ExpirationPlugin, NetworkFirst, Serwist, StaleWhileRevalidate } from 'serwist'
+import { defaultCache } from '@serwist/turbopack/worker'
+import { Serwist } from 'serwist'
+import { withLeadingSlash } from 'ufo'
 
 declare global {
   // eslint-disable-next-line ts/consistent-type-definitions
@@ -18,78 +20,30 @@ const scopePathname = new URL(self.registration.scope).pathname
 const basePath = scopePathname.replace(/\/serwist\/$/, '').replace(/\/$/, '')
 const offlineUrl = `${basePath}/_offline.html`
 
+const normalizeManifestUrl = (url: string): string => {
+  if (url.startsWith('/serwist/'))
+    return url.replace(/^\/serwist\//, '/')
+
+  return withLeadingSlash(url)
+}
+
+const manifest = self.__SW_MANIFEST?.map((entry) => {
+  if (typeof entry === 'string')
+    return normalizeManifestUrl(entry)
+
+  return {
+    ...entry,
+    url: normalizeManifestUrl(entry.url),
+  }
+})
+
 const serwist = new Serwist({
-  precacheEntries: self.__SW_MANIFEST,
+  precacheEntries: manifest,
   skipWaiting: true,
   disableDevLogs: true,
   clientsClaim: true,
   navigationPreload: true,
-  runtimeCaching: [
-    {
-      matcher: ({ url }) => url.origin === 'https://fonts.googleapis.com',
-      handler: new CacheFirst({
-        cacheName: 'google-fonts',
-        plugins: [
-          new CacheableResponsePlugin({ statuses: [0, 200] }),
-          new ExpirationPlugin({
-            maxEntries: 4,
-            maxAgeSeconds: 365 * 24 * 60 * 60,
-          }),
-        ],
-      }),
-    },
-    {
-      matcher: ({ url }) => url.origin === 'https://fonts.gstatic.com',
-      handler: new CacheFirst({
-        cacheName: 'google-fonts-webfonts',
-        plugins: [
-          new CacheableResponsePlugin({ statuses: [0, 200] }),
-          new ExpirationPlugin({
-            maxEntries: 4,
-            maxAgeSeconds: 365 * 24 * 60 * 60,
-          }),
-        ],
-      }),
-    },
-    {
-      matcher: ({ request }) => request.destination === 'image',
-      handler: new CacheFirst({
-        cacheName: 'images',
-        plugins: [
-          new CacheableResponsePlugin({ statuses: [0, 200] }),
-          new ExpirationPlugin({
-            maxEntries: 64,
-            maxAgeSeconds: 30 * 24 * 60 * 60,
-          }),
-        ],
-      }),
-    },
-    {
-      matcher: ({ request }) => request.destination === 'script' || request.destination === 'style',
-      handler: new StaleWhileRevalidate({
-        cacheName: 'static-resources',
-        plugins: [
-          new ExpirationPlugin({
-            maxEntries: 32,
-            maxAgeSeconds: 24 * 60 * 60,
-          }),
-        ],
-      }),
-    },
-    {
-      matcher: ({ url, sameOrigin }) => sameOrigin && url.pathname.startsWith('/api/'),
-      handler: new NetworkFirst({
-        cacheName: 'api-cache',
-        networkTimeoutSeconds: 10,
-        plugins: [
-          new ExpirationPlugin({
-            maxEntries: 16,
-            maxAgeSeconds: 60 * 60,
-          }),
-        ],
-      }),
-    },
-  ],
+  runtimeCaching: defaultCache,
   fallbacks: {
     entries: [
       {

web/docs/lint.md

@@ -38,6 +38,11 @@ pnpm lint:tss
 
 This command lints the entire project and is intended for final verification before committing or pushing changes.
 
+### Introducing New Plugins or Rules
+
+If a new rule causes many existing code errors or automatic fixes generate too many diffs, do not use the `--fix` option for automatic fixes.
+You can introduce the rule first, then use the `--suppress-all` option to temporarily suppress these errors, and gradually fix them in subsequent changes.
+
 ## Type Check
 
 You should be able to see suggestions from TypeScript in your editor for all open files.

web/eslint-rules/index.js

@@ -2,9 +2,7 @@ import consistentPlaceholders from './rules/consistent-placeholders.js'
 import noAsAnyInT from './rules/no-as-any-in-t.js'
 import noExtraKeys from './rules/no-extra-keys.js'
 import noLegacyNamespacePrefix from './rules/no-legacy-namespace-prefix.js'
-import noVersionPrefix from './rules/no-version-prefix.js'
 import requireNsOption from './rules/require-ns-option.js'
-import validI18nKeys from './rules/valid-i18n-keys.js'
 
 /** @type {import('eslint').ESLint.Plugin} */
 const plugin = {
@@ -17,9 +15,7 @@ const plugin = {
     'no-as-any-in-t': noAsAnyInT,
     'no-extra-keys': noExtraKeys,
     'no-legacy-namespace-prefix': noLegacyNamespacePrefix,
-    'no-version-prefix': noVersionPrefix,
     'require-ns-option': requireNsOption,
-    'valid-i18n-keys': validI18nKeys,
   },
 }
 

web/eslint-rules/rules/no-version-prefix.js

@@ -1,45 +0,0 @@
-const DEPENDENCY_KEYS = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']
-const VERSION_PREFIXES = ['^', '~']
-
-/** @type {import('eslint').Rule.RuleModule} */
-export default {
-  meta: {
-    type: 'problem',
-    docs: {
-      description: `Ensure package.json dependencies do not use version prefixes (${VERSION_PREFIXES.join(' or ')})`,
-    },
-    fixable: 'code',
-  },
-  create(context) {
-    const { filename } = context
-
-    if (!filename.endsWith('package.json'))
-      return {}
-
-    const selector = `JSONProperty:matches(${DEPENDENCY_KEYS.map(k => `[key.value="${k}"]`).join(', ')}) > JSONObjectExpression > JSONProperty`
-
-    return {
-      [selector](node) {
-        const versionNode = node.value
-
-        if (versionNode && versionNode.type === 'JSONLiteral' && typeof versionNode.value === 'string') {
-          const version = versionNode.value
-          const foundPrefix = VERSION_PREFIXES.find(prefix => version.startsWith(prefix))
-
-          if (foundPrefix) {
-            const packageName = node.key.value || node.key.name
-            const cleanVersion = version.substring(1)
-            const canAutoFix = /^\d+\.\d+\.\d+$/.test(cleanVersion)
-            context.report({
-              node: versionNode,
-              message: `Dependency "${packageName}" has version prefix "${foundPrefix}" that should be removed (found: "${version}", expected: "${cleanVersion}")`,
-              fix: canAutoFix
-                ? fixer => fixer.replaceText(versionNode, `"${cleanVersion}"`)
-                : undefined,
-            })
-          }
-        }
-      },
-    }
-  },
-}

web/eslint-rules/rules/valid-i18n-keys.js

@@ -1,61 +0,0 @@
-import { cleanJsonText } from '../utils.js'
-
-/** @type {import('eslint').Rule.RuleModule} */
-export default {
-  meta: {
-    type: 'problem',
-    docs: {
-      description: 'Ensure i18n JSON keys are flat and valid as object paths',
-    },
-  },
-  create(context) {
-    return {
-      Program(node) {
-        const { filename, sourceCode } = context
-
-        if (!filename.endsWith('.json'))
-          return
-
-        let json
-        try {
-          json = JSON.parse(cleanJsonText(sourceCode.text))
-        }
-        catch {
-          context.report({
-            node,
-            message: 'Invalid JSON format',
-          })
-          return
-        }
-
-        const keys = Object.keys(json)
-        const keyPrefixes = new Set()
-
-        for (const key of keys) {
-          if (key.includes('.')) {
-            const parts = key.split('.')
-            for (let i = 1; i < parts.length; i++) {
-              const prefix = parts.slice(0, i).join('.')
-              if (keys.includes(prefix)) {
-                context.report({
-                  node,
-                  message: `Invalid key structure: '${key}' conflicts with '${prefix}'`,
-                })
-              }
-              keyPrefixes.add(prefix)
-            }
-          }
-        }
-
-        for (const key of keys) {
-          if (keyPrefixes.has(key)) {
-            context.report({
-              node,
-              message: `Invalid key structure: '${key}' is a prefix of another key`,
-            })
-          }
-        }
-      },
-    }
-  },
-}

web/eslint.config.mjs

@@ -1,9 +1,10 @@
 // @ts-check
 import antfu from '@antfu/eslint-config'
 import pluginQuery from '@tanstack/eslint-plugin-query'
+import tailwindcss from 'eslint-plugin-better-tailwindcss'
+import hyoban from 'eslint-plugin-hyoban'
 import sonar from 'eslint-plugin-sonarjs'
 import storybook from 'eslint-plugin-storybook'
-import tailwind from 'eslint-plugin-tailwindcss'
 import dify from './eslint-rules/index.js'
 
 export default antfu(
@@ -23,7 +24,7 @@ export default antfu(
       },
     },
     nextjs: true,
-    ignores: ['public', 'types/doc-paths.ts'],
+    ignores: ['public', 'types/doc-paths.ts', 'eslint-suppressions.json'],
     typescript: {
       overrides: {
         'ts/consistent-type-definitions': ['error', 'type'],
@@ -66,46 +67,60 @@ export default antfu(
       sonarjs: sonar,
     },
   },
-  tailwind.configs['flat/recommended'],
   {
-    settings: {
-      tailwindcss: {
-        // These are the default values but feel free to customize
-        callees: ['classnames', 'clsx', 'ctl', 'cn', 'classNames'],
-        config: 'tailwind.config.js', // returned from `loadConfig()` utility if not provided
-        cssFiles: [
-          '**/*.css',
-          '!**/node_modules',
-          '!**/.*',
-          '!**/dist',
-          '!**/build',
-          '!**/.storybook',
-          '!**/.next',
-          '!**/.public',
-        ],
-        cssFilesRefreshRate: 5_000,
-        removeDuplicates: true,
-        skipClassAttribute: false,
-        whitelist: [],
-        tags: [], // can be set to e.g. ['tw'] for use in tw`bg-blue`
-        classRegex: '^class(Name)?$', // can be modified to support custom attributes. E.g. "^tw$" for `twin.macro`
-      },
+    files: ['**/*.{ts,tsx}'],
+    plugins: {
+      tailwindcss,
     },
     rules: {
-      // due to 1k lines of tailwind config, these rule have performance issue
-      'tailwindcss/no-contradicting-classname': 'off',
-      'tailwindcss/enforces-shorthand': 'off',
-      'tailwindcss/no-custom-classname': 'off',
-      'tailwindcss/no-unnecessary-arbitrary-value': 'off',
-
-      'tailwindcss/no-arbitrary-value': 'off',
-      'tailwindcss/classnames-order': 'warn',
-      'tailwindcss/enforces-negative-arbitrary-values': 'warn',
-      'tailwindcss/migration-from-tailwind-2': 'warn',
+      'tailwindcss/enforce-consistent-class-order': 'error',
+      'tailwindcss/no-duplicate-classes': 'error',
+      'tailwindcss/no-unnecessary-whitespace': 'error',
+      'tailwindcss/no-unknown-classes': 'warn',
     },
   },
   {
-    plugins: { dify },
+    name: 'dify/custom/setup',
+    plugins: {
+      dify,
+      hyoban,
+    },
+  },
+  {
+    files: ['**/*.tsx'],
+    rules: {
+      'hyoban/prefer-tailwind-icons': ['warn', {
+        prefix: 'i-',
+        propMappings: {
+          size: 'size',
+          width: 'w',
+          height: 'h',
+        },
+        libraries: [
+          {
+            prefix: 'i-custom-',
+            source: '^@/app/components/base/icons/src/(?<set>(?:public|vender)(?:/.*)?)$',
+            name: '^(?<name>.*)$',
+          },
+          {
+            source: '^@remixicon/react$',
+            name: '^(?<set>Ri)(?<name>.+)$',
+          },
+          {
+            source: '^@(?<set>heroicons)/react/24/outline$',
+            name: '^(?<name>.*)Icon$',
+          },
+          {
+            source: '^@(?<set>heroicons)/react/24/(?<variant>solid)$',
+            name: '^(?<name>.*)Icon$',
+          },
+          {
+            source: '^@(?<set>heroicons)/react/(?<variant>\\d+/(?:solid|outline))$',
+            name: '^(?<name>.*)Icon$',
+          },
+        ],
+      }],
+    },
   },
   {
     files: ['i18n/**/*.json'],
@@ -114,7 +129,7 @@ export default antfu(
       'max-lines': 'off',
       'jsonc/sort-keys': 'error',
 
-      'dify/valid-i18n-keys': 'error',
+      'hyoban/i18n-flat-key': 'error',
       'dify/no-extra-keys': 'error',
       'dify/consistent-placeholders': 'error',
     },
@@ -122,7 +137,7 @@ export default antfu(
   {
     files: ['**/package.json'],
     rules: {
-      'dify/no-version-prefix': 'error',
+      'hyoban/no-dependency-version-prefix': 'error',
     },
   },
 )

web/next.config.ts

@@ -29,7 +29,7 @@ const remoteImageURLs = ([hasSetWebPrefix ? new URL(`${process.env.NEXT_PUBLIC_W
 
 const nextConfig: NextConfig = {
   basePath: process.env.NEXT_PUBLIC_BASE_PATH || '',
-  serverExternalPackages: ['esbuild-wasm'],
+  serverExternalPackages: ['esbuild'],
   transpilePackages: ['echarts', 'zrender'],
   turbopack: {
     rules: codeInspectorPlugin({

web/package.json

@@ -31,8 +31,8 @@
     "build": "next build",
     "build:docker": "next build && node scripts/optimize-standalone.js",
     "start": "node ./scripts/copy-and-start.mjs",
-    "lint": "eslint --cache --concurrency=\"auto\"",
-    "lint:ci": "eslint --cache --concurrency 3",
+    "lint": "eslint --cache --concurrency=auto",
+    "lint:ci": "eslint --cache --concurrency 2",
     "lint:fix": "pnpm lint --fix",
     "lint:quiet": "pnpm lint --quiet",
     "lint:complexity": "pnpm lint --rule 'complexity: [error, {max: 15}]' --quiet",
@@ -47,7 +47,7 @@
     "i18n:check": "tsx ./scripts/check-i18n.js",
     "test": "vitest run",
     "test:coverage": "vitest run --coverage",
-    "test:ci": "vitest run --coverage --reporter vitest-tiny-reporter --silent=passed-only",
+    "test:ci": "vitest run --coverage --silent=passed-only",
     "test:watch": "vitest --watch",
     "analyze-component": "node ./scripts/analyze-component.js",
     "refactor-component": "node ./scripts/refactor-component.js",
@@ -154,8 +154,9 @@
     "sharp": "0.33.5",
     "sortablejs": "1.15.6",
     "string-ts": "2.3.1",
-    "tailwind-merge": "2.6.0",
+    "tailwind-merge": "2.6.1",
     "tldts": "7.0.17",
+    "ufo": "1.6.3",
     "use-context-selector": "2.0.0",
     "uuid": "10.0.0",
     "zod": "3.25.76",
@@ -165,21 +166,24 @@
   "devDependencies": {
     "@antfu/eslint-config": "7.2.0",
     "@chromatic-com/storybook": "5.0.0",
-    "@eslint-react/eslint-plugin": "2.8.1",
+    "@egoist/tailwindcss-icons": "1.9.2",
+    "@eslint-react/eslint-plugin": "2.9.4",
+    "@iconify-json/heroicons": "1.2.3",
+    "@iconify-json/ri": "1.2.7",
     "@mdx-js/loader": "3.1.1",
     "@mdx-js/react": "3.1.1",
     "@next/bundle-analyzer": "16.1.5",
     "@next/eslint-plugin-next": "16.1.6",
     "@next/mdx": "16.1.5",
     "@rgrove/parse-xml": "4.2.0",
-    "@serwist/turbopack": "9.5.0",
+    "@serwist/turbopack": "9.5.4",
     "@storybook/addon-docs": "10.2.0",
     "@storybook/addon-links": "10.2.0",
     "@storybook/addon-onboarding": "10.2.0",
     "@storybook/addon-themes": "10.2.0",
     "@storybook/nextjs-vite": "10.2.0",
     "@storybook/react": "10.2.0",
-    "@tanstack/eslint-plugin-query": "5.91.3",
+    "@tanstack/eslint-plugin-query": "5.91.4",
     "@tanstack/react-devtools": "0.9.2",
     "@tanstack/react-form-devtools": "0.2.12",
     "@tanstack/react-query-devtools": "5.90.2",
@@ -210,14 +214,16 @@
     "autoprefixer": "10.4.21",
     "code-inspector-plugin": "1.3.6",
     "cross-env": "10.1.0",
-    "esbuild-wasm": "0.27.2",
+    "esbuild": "0.27.2",
     "eslint": "9.39.2",
+    "eslint-plugin-better-tailwindcss": "https://pkg.pr.new/hyoban/eslint-plugin-better-tailwindcss@c0161c7",
+    "eslint-plugin-hyoban": "0.10.1",
     "eslint-plugin-react-hooks": "7.0.1",
-    "eslint-plugin-react-refresh": "0.4.26",
+    "eslint-plugin-react-refresh": "0.5.0",
     "eslint-plugin-sonarjs": "3.0.6",
-    "eslint-plugin-storybook": "10.2.1",
-    "eslint-plugin-tailwindcss": "3.18.2",
+    "eslint-plugin-storybook": "10.2.6",
     "husky": "9.1.7",
+    "iconify-import-svg": "0.1.1",
     "jsdom": "27.3.0",
     "jsdom-testing-mocks": "1.16.0",
     "knip": "5.78.0",
@@ -226,17 +232,16 @@
     "postcss": "8.5.6",
     "react-scan": "0.4.3",
     "sass": "1.93.2",
-    "serwist": "9.5.0",
+    "serwist": "9.5.4",
     "storybook": "10.2.0",
-    "tailwindcss": "3.4.18",
+    "tailwindcss": "3.4.19",
     "tsx": "4.21.0",
     "typescript": "5.9.3",
     "uglify-js": "3.19.3",
     "vite": "7.3.1",
     "vite-tsconfig-paths": "6.0.4",
     "vitest": "4.0.17",
-    "vitest-canvas-mock": "1.1.3",
-    "vitest-tiny-reporter": "1.3.1"
+    "vitest-canvas-mock": "1.1.3"
   },
   "pnpm": {
     "overrides": {

web/tailwind-common-config.ts

@@ -1,8 +1,18 @@
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { getIconCollections, iconsPlugin } from '@egoist/tailwindcss-icons'
 import tailwindTypography from '@tailwindcss/typography'
+import { importSvgCollections } from 'iconify-import-svg'
 // @ts-expect-error workaround for turbopack issue
 import tailwindThemeVarDefine from './themes/tailwind-theme-var-define.ts'
 import typography from './typography.js'
 
+const _dirname = typeof __dirname !== 'undefined'
+  ? __dirname
+  : path.dirname(fileURLToPath(import.meta.url))
+
+const isProduction = process.env.NODE_ENV === 'production'
+
 const config = {
   theme: {
     typography,
@@ -146,7 +156,31 @@ const config = {
       },
     },
   },
-  plugins: [tailwindTypography],
+  plugins: [
+    tailwindTypography,
+    iconsPlugin({
+      collections: {
+        ...getIconCollections(['heroicons', 'ri']),
+        ...importSvgCollections({
+          source: path.resolve(_dirname, 'app/components/base/icons/assets/public'),
+          prefix: 'custom-public',
+          ignoreImportErrors: true,
+          runSVGO: isProduction,
+        }),
+        ...importSvgCollections({
+          source: path.resolve(_dirname, 'app/components/base/icons/assets/vender'),
+          prefix: 'custom-vender',
+          ignoreImportErrors: true,
+          runSVGO: isProduction,
+        }),
+      },
+      extraProperties: {
+        width: '1rem',
+        height: '1rem',
+        display: 'block',
+      },
+    }),
+  ],
   // https://github.com/tailwindlabs/tailwindcss/discussions/5969
   corePlugins: {
     preflight: false,

web/utils/classnames.spec.ts

@@ -1,157 +0,0 @@
-/**
- * Test suite for the classnames utility function
- * This utility combines the classnames library with tailwind-merge
- * to handle conditional CSS classes and merge conflicting Tailwind classes
- */
-import { cn } from './classnames'
-
-describe('classnames', () => {
-  /**
-   * Tests basic classnames library features:
-   * - String concatenation
-   * - Array handling
-   * - Falsy value filtering
-   * - Object-based conditional classes
-   */
-  it('classnames libs feature', () => {
-    expect(cn('foo')).toBe('foo')
-    expect(cn('foo', 'bar')).toBe('foo bar')
-    expect(cn(['foo', 'bar'])).toBe('foo bar')
-
-    expect(cn(undefined)).toBe('')
-    expect(cn(null)).toBe('')
-    expect(cn(false)).toBe('')
-
-    expect(cn({
-      foo: true,
-      bar: false,
-      baz: true,
-    })).toBe('foo baz')
-  })
-
-  /**
-   * Tests tailwind-merge functionality:
-   * - Conflicting class resolution (last one wins)
-   * - Modifier handling (hover, focus, etc.)
-   * - Important prefix (!)
-   * - Custom color classes
-   * - Arbitrary values
-   */
-  it('tailwind-merge', () => {
-    /* eslint-disable tailwindcss/classnames-order */
-    expect(cn('p-0')).toBe('p-0')
-    expect(cn('text-right text-center text-left')).toBe('text-left')
-    expect(cn('pl-4 p-8')).toBe('p-8')
-    expect(cn('m-[2px] m-[4px]')).toBe('m-[4px]')
-    expect(cn('m-1 m-[4px]')).toBe('m-[4px]')
-    expect(cn('overflow-x-auto hover:overflow-x-hidden overflow-x-scroll')).toBe(
-      'hover:overflow-x-hidden overflow-x-scroll',
-    )
-    expect(cn('h-10 h-min')).toBe('h-min')
-    expect(cn('bg-grey-5 bg-hotpink')).toBe('bg-hotpink')
-
-    expect(cn('hover:block hover:inline')).toBe('hover:inline')
-
-    expect(cn('font-medium !font-bold')).toBe('font-medium !font-bold')
-    expect(cn('!font-medium !font-bold')).toBe('!font-bold')
-
-    expect(cn('text-gray-100 text-primary-200')).toBe('text-primary-200')
-    expect(cn('text-some-unknown-color text-components-input-bg-disabled text-primary-200')).toBe('text-primary-200')
-    expect(cn('bg-some-unknown-color bg-components-input-bg-disabled bg-primary-200')).toBe('bg-primary-200')
-
-    expect(cn('border-t border-white/10')).toBe('border-t border-white/10')
-    expect(cn('border-t border-white')).toBe('border-t border-white')
-    expect(cn('text-3.5xl text-black')).toBe('text-3.5xl text-black')
-  })
-
-  /**
-   * Tests the integration of classnames and tailwind-merge:
-   * - Object-based conditional classes with Tailwind conflict resolution
-   */
-  it('classnames combined with tailwind-merge', () => {
-    expect(cn('text-right', {
-      'text-center': true,
-    })).toBe('text-center')
-
-    expect(cn('text-right', {
-      'text-center': false,
-    })).toBe('text-right')
-  })
-
-  /**
-   * Tests handling of multiple mixed argument types:
-   * - Strings, arrays, and objects in a single call
-   * - Tailwind merge working across different argument types
-   */
-  it('multiple mixed argument types', () => {
-    expect(cn('foo', ['bar', 'baz'], { qux: true, quux: false })).toBe('foo bar baz qux')
-    expect(cn('p-4', ['p-2', 'm-4'], { 'text-left': true, 'text-right': true })).toBe('p-2 m-4 text-right')
-  })
-
-  /**
-   * Tests nested array handling:
-   * - Deep array flattening
-   * - Tailwind merge with nested structures
-   */
-  it('nested arrays', () => {
-    expect(cn(['foo', ['bar', 'baz']])).toBe('foo bar baz')
-    expect(cn(['p-4', ['p-2', 'text-center']])).toBe('p-2 text-center')
-  })
-
-  /**
-   * Tests empty input handling:
-   * - Empty strings, arrays, and objects
-   * - Mixed empty and non-empty values
-   */
-  it('empty inputs', () => {
-    expect(cn('')).toBe('')
-    expect(cn([])).toBe('')
-    expect(cn({})).toBe('')
-    expect(cn('', [], {})).toBe('')
-    expect(cn('foo', '', 'bar')).toBe('foo bar')
-  })
-
-  /**
-   * Tests number input handling:
-   * - Truthy numbers converted to strings
-   * - Zero treated as falsy
-   */
-  it('numbers as inputs', () => {
-    expect(cn(1)).toBe('1')
-    expect(cn(0)).toBe('')
-    expect(cn('foo', 1, 'bar')).toBe('foo 1 bar')
-  })
-
-  /**
-   * Tests multiple object arguments:
-   * - Object merging
-   * - Tailwind conflict resolution across objects
-   */
-  it('multiple objects', () => {
-    expect(cn({ foo: true }, { bar: true })).toBe('foo bar')
-    expect(cn({ foo: true, bar: false }, { bar: true, baz: true })).toBe('foo bar baz')
-    expect(cn({ 'p-4': true }, { 'p-2': true })).toBe('p-2')
-  })
-
-  /**
-   * Tests complex edge cases:
-   * - Mixed falsy values
-   * - Nested arrays with falsy values
-   * - Multiple conflicting Tailwind classes
-   */
-  it('complex edge cases', () => {
-    expect(cn('foo', null, undefined, false, 'bar', 0, 1, '')).toBe('foo bar 1')
-    expect(cn(['foo', null, ['bar', undefined, 'baz']])).toBe('foo bar baz')
-    expect(cn('text-sm', { 'text-lg': false, 'text-xl': true }, 'text-2xl')).toBe('text-2xl')
-  })
-
-  /**
-   * Tests important (!) modifier behavior:
-   * - Important modifiers in objects
-   * - Conflict resolution with important prefix
-   */
-  it('important modifier with objects', () => {
-    expect(cn({ '!font-medium': true }, { '!font-bold': true })).toBe('!font-bold')
-    expect(cn('font-normal', { '!font-bold': true })).toBe('font-normal !font-bold')
-  })
-})

web/vitest.config.ts

@@ -1,6 +1,8 @@
 import { defineConfig, mergeConfig } from 'vitest/config'
 import viteConfig from './vite.config'
 
+const isCI = !!process.env.CI
+
 export default mergeConfig(viteConfig, defineConfig({
   test: {
     environment: 'jsdom',
@@ -8,7 +10,7 @@ export default mergeConfig(viteConfig, defineConfig({
     setupFiles: ['./vitest.setup.ts'],
     coverage: {
       provider: 'v8',
-      reporter: ['json', 'json-summary'],
+      reporter: isCI ? ['json', 'json-summary'] : ['text', 'json', 'json-summary'],
     },
   },
 }))