Complete RegExp.exec refactor for performance optimization

Co-authored-by: asukaminato0721 <30024051+asukaminato0721@users.noreply.github.com>

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.12
+FROM mcr.microsoft.com/devcontainers/python:3.12-bullseye
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
      && apt-get -y install libgmp-dev libmpfr-dev libmpc-dev

.devcontainer/post_create_command.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
+WORKSPACE_ROOT=$(pwd)
 
 corepack enable
 cd web && pnpm install
 pipx install uv
 
-echo 'alias start-api="cd /workspaces/dify/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
-echo 'alias start-worker="cd /workspaces/dify/api && uv run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage"' >> ~/.bashrc
-echo 'alias start-web="cd /workspaces/dify/web && pnpm dev"' >> ~/.bashrc
-echo 'alias start-web-prod="cd /workspaces/dify/web && pnpm build && pnpm start"' >> ~/.bashrc
-echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d"' >> ~/.bashrc
-echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down"' >> ~/.bashrc
+echo "alias start-api=\"cd $WORKSPACE_ROOT/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug\"" >> ~/.bashrc
+echo "alias start-worker=\"cd $WORKSPACE_ROOT/api && uv run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage\"" >> ~/.bashrc
+echo "alias start-web=\"cd $WORKSPACE_ROOT/web && pnpm dev\"" >> ~/.bashrc
+echo "alias start-web-prod=\"cd $WORKSPACE_ROOT/web && pnpm build && pnpm start\"" >> ~/.bashrc
+echo "alias start-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d\"" >> ~/.bashrc
+echo "alias stop-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down\"" >> ~/.bashrc
 
 source /home/vscode/.bashrc
 

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/web"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2
+  - package-ecosystem: "uv"
+    directory: "/api"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2

.github/workflows/autofix.yml

@@ -22,12 +22,58 @@ jobs:
           # Fix lint errors
           uv run ruff check --fix .
           # Format code
-          uv run ruff format .
+          uv run ruff format ..
+
       - name: ast-grep
         run: |
           uvx --from ast-grep-cli sg --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all
           uvx --from ast-grep-cli sg --pattern 'session.query($WHATEVER).filter($HERE)' --rewrite 'session.query($WHATEVER).where($HERE)' -l py --update-all
+          # Convert Optional[T] to T | None (ignoring quoted types)
+          cat > /tmp/optional-rule.yml << 'EOF'
+          id: convert-optional-to-union
+          language: python
+          rule:
+            kind: generic_type
+            all:
+              - has:
+                  kind: identifier
+                  pattern: Optional
+              - has:
+                  kind: type_parameter
+                  has:
+                    kind: type
+                    pattern: $T
+          fix: $T | None
+          EOF
+          uvx --from ast-grep-cli sg scan --inline-rules "$(cat /tmp/optional-rule.yml)" --update-all
+          # Fix forward references that were incorrectly converted (Python doesn't support "Type" | None syntax)
+          find . -name "*.py" -type f -exec sed -i.bak -E 's/"([^"]+)" \| None/Optional["\1"]/g; s/'"'"'([^'"'"']+)'"'"' \| None/Optional['"'"'\1'"'"']/g' {} \;
+          find . -name "*.py.bak" -type f -delete
+
       - name: mdformat
         run: |
           uvx mdformat .
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: web/package.json
+          run_install: false
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: ./web/package.json
+
+      - name: Web dependencies
+        working-directory: ./web
+        run: pnpm install --frozen-lockfile
+
+      - name: oxlint
+        working-directory: ./web
+        run: |
+          pnpx oxlint --fix
+
       - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27

.github/workflows/build-push.yml

@@ -8,6 +8,7 @@ on:
       - "deploy/enterprise"
       - "build/**"
       - "release/e-*"
+      - "hotfix/**"
     tags:
       - "*"
 

.github/workflows/deploy-dev.yml

@@ -12,12 +12,13 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     if: |
-      github.event.workflow_run.conclusion == 'success'
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_branch == 'deploy/dev'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v0.1.8
         with:
-          host: ${{ secrets.SSH_HOST }}
+          host: ${{ secrets.RAG_SSH_HOST }}
           username: ${{ secrets.SSH_USER }}
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           script: |

.github/workflows/style.yml

@@ -12,7 +12,6 @@ permissions:
   statuses: write
   contents: read
 
-
 jobs:
   python-style:
     name: Python Style
@@ -44,6 +43,10 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: uv sync --project api --dev
 
+      - name: Run Import Linter
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: uv run --directory api --dev lint-imports
+
       - name: Run Basedpyright Checks
         if: steps.changed-files.outputs.any_changed == 'true'
         run: dev/basedpyright-check
@@ -99,7 +102,6 @@ jobs:
         working-directory: ./web
         run: |
           pnpm run lint
-          pnpm run eslint
 
   docker-compose-template:
     name: Docker Compose Template

.gitignore

@@ -227,3 +227,11 @@ web/public/fallback-*.js
 .roo/
 api/.env.backup
 /clickzetta
+
+# Benchmark
+scripts/stress-test/setup/config/
+scripts/stress-test/reports/
+
+# mcp
+.playwright-mcp/
+.serena/

AGENTS.md

@@ -1 +0,0 @@
-CLAUDE.md

AGENTS.md

@@ -0,0 +1,87 @@
+# AGENTS.md
+
+## Project Overview
+
+Dify is an open-source platform for developing LLM applications with an intuitive interface combining agentic AI workflows, RAG pipelines, agent capabilities, and model management.
+
+The codebase consists of:
+
+- **Backend API** (`/api`): Python Flask application with Domain-Driven Design architecture
+- **Frontend Web** (`/web`): Next.js 15 application with TypeScript and React 19
+- **Docker deployment** (`/docker`): Containerized deployment configurations
+
+## Development Commands
+
+### Backend (API)
+
+All Python commands must be prefixed with `uv run --project api`:
+
+```bash
+# Start development servers
+./dev/start-api                   # Start API server
+./dev/start-worker                # Start Celery worker
+
+# Run tests
+uv run --project api pytest      # Run all tests
+uv run --project api pytest tests/unit_tests/     # Unit tests only
+uv run --project api pytest tests/integration_tests/  # Integration tests
+
+# Code quality
+./dev/reformat                    # Run all formatters and linters
+uv run --project api ruff check --fix ./    # Fix linting issues
+uv run --project api ruff format ./         # Format code
+uv run --directory api basedpyright         # Type checking
+```
+
+### Frontend (Web)
+
+```bash
+cd web
+pnpm lint                         # Run ESLint
+pnpm eslint-fix                   # Fix ESLint issues
+pnpm test                         # Run Jest tests
+```
+
+## Testing Guidelines
+
+### Backend Testing
+
+- Use `pytest` for all backend tests
+- Write tests first (TDD approach)
+- Test structure: Arrange-Act-Assert
+
+## Code Style Requirements
+
+### Python
+
+- Use type hints for all functions and class attributes
+- No `Any` types unless absolutely necessary
+- Implement special methods (`__repr__`, `__str__`) appropriately
+
+### TypeScript/JavaScript
+
+- Strict TypeScript configuration
+- ESLint with Prettier integration
+- Avoid `any` type
+
+## Important Notes
+
+- **Environment Variables**: Always use UV for Python commands: `uv run --project api <command>`
+- **Comments**: Only write meaningful comments that explain "why", not "what"
+- **File Creation**: Always prefer editing existing files over creating new ones
+- **Documentation**: Don't create documentation files unless explicitly requested
+- **Code Quality**: Always run `./dev/reformat` before committing backend changes
+
+## Common Development Tasks
+
+### Adding a New API Endpoint
+
+1. Create controller in `/api/controllers/`
+1. Add service logic in `/api/services/`
+1. Update routes in controller's `__init__.py`
+1. Write tests in `/api/tests/`
+
+## Project-Specific Conventions
+
+- All async tasks use Celery with Redis as broker
+- **Internationalization**: Frontend supports multiple languages with English (`web/i18n/en-US/`) as the source. All user-facing text must use i18n keys, no hardcoded strings. Edit corresponding module files in `en-US/` directory for translations.

CLAUDE.md

@@ -1,89 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-Dify is an open-source platform for developing LLM applications with an intuitive interface combining agentic AI workflows, RAG pipelines, agent capabilities, and model management.
-
-The codebase consists of:
-
-- **Backend API** (`/api`): Python Flask application with Domain-Driven Design architecture
-- **Frontend Web** (`/web`): Next.js 15 application with TypeScript and React 19
-- **Docker deployment** (`/docker`): Containerized deployment configurations
-
-## Development Commands
-
-### Backend (API)
-
-All Python commands must be prefixed with `uv run --project api`:
-
-```bash
-# Start development servers
-./dev/start-api                   # Start API server
-./dev/start-worker                # Start Celery worker
-
-# Run tests
-uv run --project api pytest      # Run all tests
-uv run --project api pytest tests/unit_tests/     # Unit tests only
-uv run --project api pytest tests/integration_tests/  # Integration tests
-
-# Code quality
-./dev/reformat                    # Run all formatters and linters
-uv run --project api ruff check --fix ./    # Fix linting issues
-uv run --project api ruff format ./         # Format code
-uv run --directory api basedpyright         # Type checking
-```
-
-### Frontend (Web)
-
-```bash
-cd web
-pnpm lint                         # Run ESLint
-pnpm eslint-fix                   # Fix ESLint issues
-pnpm test                         # Run Jest tests
-```
-
-## Testing Guidelines
-
-### Backend Testing
-
-- Use `pytest` for all backend tests
-- Write tests first (TDD approach)
-- Test structure: Arrange-Act-Assert
-
-## Code Style Requirements
-
-### Python
-
-- Use type hints for all functions and class attributes
-- No `Any` types unless absolutely necessary
-- Implement special methods (`__repr__`, `__str__`) appropriately
-
-### TypeScript/JavaScript
-
-- Strict TypeScript configuration
-- ESLint with Prettier integration
-- Avoid `any` type
-
-## Important Notes
-
-- **Environment Variables**: Always use UV for Python commands: `uv run --project api <command>`
-- **Comments**: Only write meaningful comments that explain "why", not "what"
-- **File Creation**: Always prefer editing existing files over creating new ones
-- **Documentation**: Don't create documentation files unless explicitly requested
-- **Code Quality**: Always run `./dev/reformat` before committing backend changes
-
-## Common Development Tasks
-
-### Adding a New API Endpoint
-
-1. Create controller in `/api/controllers/`
-1. Add service logic in `/api/services/`
-1. Update routes in controller's `__init__.py`
-1. Write tests in `/api/tests/`
-
-## Project-Specific Conventions
-
-- All async tasks use Celery with Redis as broker
-- **Internationalization**: Frontend supports multiple languages with English (`web/i18n/en-US/`) as the source. All user-facing text must use i18n keys, no hardcoded strings. Edit corresponding module files in `en-US/` directory for translations.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Makefile

@@ -4,10 +4,13 @@ WEB_IMAGE=$(DOCKER_REGISTRY)/dify-web
 API_IMAGE=$(DOCKER_REGISTRY)/dify-api
 VERSION=latest
 
+# Default target - show help
+.DEFAULT_GOAL := help
+
 # Backend Development Environment Setup
 .PHONY: dev-setup prepare-docker prepare-web prepare-api
 
-# Default dev setup target
+# Dev setup target
 dev-setup: prepare-docker prepare-web prepare-api
 	@echo "✅ Backend development environment setup complete!"
 
@@ -46,6 +49,28 @@ dev-clean:
 	@rm -rf api/storage
 	@echo "✅ Cleanup complete"
 
+# Backend Code Quality Commands
+format:
+	@echo "🎨 Running ruff format..."
+	@uv run --project api --dev ruff format ./api
+	@echo "✅ Code formatting complete"
+
+check:
+	@echo "🔍 Running ruff check..."
+	@uv run --project api --dev ruff check ./api
+	@echo "✅ Code check complete"
+
+lint:
+	@echo "🔧 Running ruff format, check with fixes, and import linter..."
+	@uv run --project api --dev sh -c 'ruff format ./api && ruff check --fix ./api'
+	@uv run --directory api --dev lint-imports
+	@echo "✅ Linting complete"
+
+type-check:
+	@echo "📝 Running type check with basedpyright..."
+	@uv run --directory api --dev basedpyright
+	@echo "✅ Type check complete"
+
 # Build Docker images
 build-web:
 	@echo "Building web Docker image: $(WEB_IMAGE):$(VERSION)..."
@@ -90,6 +115,12 @@ help:
 	@echo "  make prepare-api    - Set up API environment"
 	@echo "  make dev-clean      - Stop Docker middleware containers"
 	@echo ""
+	@echo "Backend Code Quality:"
+	@echo "  make format         - Format code with ruff"
+	@echo "  make check          - Check code with ruff"
+	@echo "  make lint           - Format and fix code with ruff"
+	@echo "  make type-check     - Run type checking with basedpyright"
+	@echo ""
 	@echo "Docker Build Targets:"
 	@echo "  make build-web      - Build web Docker image"
 	@echo "  make build-api      - Build API Docker image"
@@ -98,4 +129,4 @@ help:
 	@echo "  make build-push-all - Build and push all Docker images"
 
 # Phony targets
-.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help
+.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help format check lint type-check

api/.env.example

@@ -76,6 +76,7 @@ DB_HOST=localhost
 DB_PORT=5432
 DB_DATABASE=dify
 SQLALCHEMY_POOL_PRE_PING=true
+SQLALCHEMY_POOL_TIMEOUT=30
 
 # Storage configuration
 # use for store upload files, private keys...
@@ -303,6 +304,8 @@ BAIDU_VECTOR_DB_API_KEY=dify
 BAIDU_VECTOR_DB_DATABASE=dify
 BAIDU_VECTOR_DB_SHARD=1
 BAIDU_VECTOR_DB_REPLICAS=3
+BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER=DEFAULT_ANALYZER
+BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE=COARSE_MODE
 
 # Upstash configuration
 UPSTASH_VECTOR_URL=your-server-url
@@ -328,7 +331,7 @@ MATRIXONE_DATABASE=dify
 LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
 LINDORM_USERNAME=admin
 LINDORM_PASSWORD=admin
-USING_UGC_INDEX=False
+LINDORM_USING_UGC=True
 LINDORM_QUERY_TIMEOUT=1
 
 # OceanBase Vector configuration
@@ -405,6 +408,9 @@ SSRF_DEFAULT_TIME_OUT=5
 SSRF_DEFAULT_CONNECT_TIME_OUT=5
 SSRF_DEFAULT_READ_TIME_OUT=5
 SSRF_DEFAULT_WRITE_TIME_OUT=5
+SSRF_POOL_MAX_CONNECTIONS=100
+SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+SSRF_POOL_KEEPALIVE_EXPIRY=5.0
 
 BATCH_UPLOAD_LIMIT=10
 KEYWORD_DATA_SOURCE_TYPE=database
@@ -415,6 +421,10 @@ WORKFLOW_FILE_UPLOAD_LIMIT=10
 # CODE EXECUTION CONFIGURATION
 CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
 CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_EXECUTION_SSL_VERIFY=True
+CODE_EXECUTION_POOL_MAX_CONNECTIONS=100
+CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY=5.0
 CODE_MAX_NUMBER=9223372036854775807
 CODE_MIN_NUMBER=-9223372036854775808
 CODE_MAX_STRING_LENGTH=80000
@@ -458,9 +468,18 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
 WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
-WORKFLOW_PARALLEL_DEPTH_LIMIT=3
 MAX_VARIABLE_SIZE=204800
 
+# GraphEngine Worker Pool Configuration
+# Minimum number of workers per GraphEngine instance (default: 1)
+GRAPH_ENGINE_MIN_WORKERS=1
+# Maximum number of workers per GraphEngine instance (default: 10)
+GRAPH_ENGINE_MAX_WORKERS=10
+# Queue depth threshold that triggers worker scale up (default: 3)
+GRAPH_ENGINE_SCALE_UP_THRESHOLD=3
+# Seconds of idle time before scaling down workers (default: 5.0)
+GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0
+
 # Workflow storage configuration
 # Options: rdbms, hybrid
 # rdbms: Use only the relational database (default)
@@ -530,6 +549,7 @@ ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5
 CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
 OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
 

api/.importlinter

@@ -0,0 +1,105 @@
+[importlinter]
+root_packages =
+    core
+    configs
+    controllers
+    models
+    tasks
+    services
+
+[importlinter:contract:workflow]
+name = Workflow
+type=layers
+layers =
+    graph_engine
+    graph_events
+    graph
+    nodes
+    node_events
+    entities
+containers =
+    core.workflow
+ignore_imports =
+    core.workflow.nodes.base.node -> core.workflow.graph_events
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_events
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_events
+
+    core.workflow.nodes.node_factory -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine.command_channels
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels
+
+[importlinter:contract:rsc]
+name = RSC
+type = layers
+layers =
+    graph_engine
+    response_coordinator
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:worker]
+name = Worker
+type = layers
+layers =
+    graph_engine
+    worker
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:graph-engine-architecture]
+name = Graph Engine Architecture
+type = layers
+layers =
+    graph_engine
+    orchestration
+    command_processing
+    event_management
+    error_handler
+    graph_traversal
+    graph_state_manager
+    worker_management
+    domain
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:domain-isolation]
+name = Domain Model Isolation
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.domain
+forbidden_modules =
+    core.workflow.graph_engine.worker_management
+    core.workflow.graph_engine.command_channels
+    core.workflow.graph_engine.layers
+    core.workflow.graph_engine.protocols
+
+[importlinter:contract:worker-management]
+name = Worker Management
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.worker_management
+forbidden_modules =
+    core.workflow.graph_engine.orchestration
+    core.workflow.graph_engine.command_processing
+    core.workflow.graph_engine.event_management
+
+
+[importlinter:contract:graph-traversal-components]
+name = Graph Traversal Components
+type = layers
+layers =
+    edge_processor
+    skip_propagator
+containers =
+    core.workflow.graph_engine.graph_traversal
+
+[importlinter:contract:command-channels]
+name = Command Channels Independence
+type = independence
+modules =
+    core.workflow.graph_engine.command_channels.in_memory_channel
+    core.workflow.graph_engine.command_channels.redis_channel

api/.ruff.toml

@@ -5,7 +5,7 @@ line-length = 120
 quote-style = "double"
 
 [lint]
-preview = false
+preview = true
 select = [
     "B",       # flake8-bugbear rules
     "C4",      # flake8-comprehensions
@@ -30,6 +30,7 @@ select = [
     "RUF022",  # unsorted-dunder-all
     "S506",    # unsafe-yaml-load
     "SIM",     # flake8-simplify rules
+    "T201",    # print-found
     "TRY400",  # error-instead-of-exception
     "TRY401",  # verbose-log-message
     "UP",      # pyupgrade rules
@@ -65,6 +66,7 @@ ignore = [
     "B006",    # mutable-argument-default
     "B007",    # unused-loop-control-variable
     "B026",    # star-arg-unpacking-after-keyword-arg
+    "B901",    # allow return in yield
     "B903",    # class-as-data-structure
     "B904",    # raise-without-from-inside-except
     "B905",    # zip-without-explicit-strict
@@ -90,11 +92,18 @@ ignore = [
 "configs/*" = [
     "N802", # invalid-function-name
 ]
+"core/model_runtime/callbacks/base_callback.py" = [
+    "T201",
+]
+"core/workflow/callbacks/workflow_logging_callback.py" = [
+    "T201",
+]
 "libs/gmpy2_pkcs10aep_cipher.py" = [
     "N803", # invalid-argument-name
 ]
 "tests/*" = [
     "F811", # redefined-while-unused
+    "T201", # allow print in tests
 ]
 
 [lint.pyflakes]

api/app.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
 
@@ -17,20 +16,20 @@ else:
     # It seems that JetBrains Python debugger does not work well with gevent,
     # so we need to disable gevent in debug mode.
     # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
-    if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
-        from gevent import monkey
+    # if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
+    # from gevent import monkey
+    #
+    # # gevent
+    # monkey.patch_all()
+    #
+    # from grpc.experimental import gevent as grpc_gevent  # type: ignore
+    #
+    # # grpc gevent
+    # grpc_gevent.init_gevent()
 
-        # gevent
-        monkey.patch_all()
-
-        from grpc.experimental import gevent as grpc_gevent  # type: ignore
-
-        # grpc gevent
-        grpc_gevent.init_gevent()
-
-        import psycogreen.gevent  # type: ignore
-
-        psycogreen.gevent.patch_psycopg()
+    # import psycogreen.gevent  # type: ignore
+    #
+    # psycogreen.gevent.patch_psycopg()
 
     from app_factory import create_app
 

api/celery_entrypoint.py

@@ -0,0 +1,13 @@
+import psycogreen.gevent as pscycogreen_gevent  # type: ignore
+from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+# grpc gevent
+grpc_gevent.init_gevent()
+print("gRPC patched with gevent.", flush=True)  # noqa: T201
+pscycogreen_gevent.patch_psycopg()
+print("psycopg2 patched with gevent.", flush=True)  # noqa: T201
+
+
+from app import app, celery
+
+__all__ = ["app", "celery"]

api/commands.py

@@ -2,7 +2,7 @@ import base64
 import json
 import logging
 import secrets
-from typing import Any, Optional
+from typing import Any
 
 import click
 import sqlalchemy as sa
@@ -10,32 +10,41 @@ from flask import current_app
 from pydantic import TypeAdapter
 from sqlalchemy import select
 from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.orm import sessionmaker
 
 from configs import dify_config
 from constants.languages import languages
-from core.plugin.entities.plugin import ToolProviderID
+from core.helper import encrypter
+from core.plugin.impl.plugin import PluginInstaller
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.rag.models.document import Document
+from core.tools.entities.tool_entities import CredentialType
 from core.tools.utils.system_oauth_encryption import encrypt_system_oauth_params
 from events.app_event import app_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from extensions.ext_storage import storage
+from extensions.storage.opendal_storage import OpenDALStorage
+from extensions.storage.storage_type import StorageType
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
 from models import Tenant
 from models.dataset import Dataset, DatasetCollectionBinding, DatasetMetadata, DatasetMetadataBinding, DocumentSegment
 from models.dataset import Document as DatasetDocument
-from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
+from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation, UploadFile
+from models.oauth import DatasourceOauthParamConfig, DatasourceProvider
 from models.provider import Provider, ProviderModel
+from models.provider_ids import DatasourceProviderID, ToolProviderID
+from models.source import DataSourceApiKeyAuthBinding, DataSourceOauthBinding
 from models.tools import ToolOAuthSystemClient
 from services.account_service import AccountService, RegisterService, TenantService
 from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs
 from services.plugin.data_migration import PluginDataMigration
 from services.plugin.plugin_migration import PluginMigration
+from services.plugin.plugin_service import PluginService
 from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
 
 logger = logging.getLogger(__name__)
@@ -53,31 +62,30 @@ def reset_password(email, new_password, password_confirm):
     if str(new_password).strip() != str(password_confirm).strip():
         click.echo(click.style("Passwords do not match.", fg="red"))
         return
+    with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
+        account = session.query(Account).where(Account.email == email).one_or_none()
 
-    account = db.session.query(Account).where(Account.email == email).one_or_none()
+        if not account:
+            click.echo(click.style(f"Account not found for email: {email}", fg="red"))
+            return
 
-    if not account:
-        click.echo(click.style(f"Account not found for email: {email}", fg="red"))
-        return
+        try:
+            valid_password(new_password)
+        except:
+            click.echo(click.style(f"Invalid password. Must match {password_pattern}", fg="red"))
+            return
 
-    try:
-        valid_password(new_password)
-    except:
-        click.echo(click.style(f"Invalid password. Must match {password_pattern}", fg="red"))
-        return
+        # generate password salt
+        salt = secrets.token_bytes(16)
+        base64_salt = base64.b64encode(salt).decode()
 
-    # generate password salt
-    salt = secrets.token_bytes(16)
-    base64_salt = base64.b64encode(salt).decode()
-
-    # encrypt password with salt
-    password_hashed = hash_password(new_password, salt)
-    base64_password_hashed = base64.b64encode(password_hashed).decode()
-    account.password = base64_password_hashed
-    account.password_salt = base64_salt
-    db.session.commit()
-    AccountService.reset_login_error_rate_limit(email)
-    click.echo(click.style("Password reset successfully.", fg="green"))
+        # encrypt password with salt
+        password_hashed = hash_password(new_password, salt)
+        base64_password_hashed = base64.b64encode(password_hashed).decode()
+        account.password = base64_password_hashed
+        account.password_salt = base64_salt
+        AccountService.reset_login_error_rate_limit(email)
+        click.echo(click.style("Password reset successfully.", fg="green"))
 
 
 @click.command("reset-email", help="Reset the account email.")
@@ -92,22 +100,21 @@ def reset_email(email, new_email, email_confirm):
     if str(new_email).strip() != str(email_confirm).strip():
         click.echo(click.style("New emails do not match.", fg="red"))
         return
+    with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
+        account = session.query(Account).where(Account.email == email).one_or_none()
 
-    account = db.session.query(Account).where(Account.email == email).one_or_none()
+        if not account:
+            click.echo(click.style(f"Account not found for email: {email}", fg="red"))
+            return
 
-    if not account:
-        click.echo(click.style(f"Account not found for email: {email}", fg="red"))
-        return
+        try:
+            email_validate(new_email)
+        except:
+            click.echo(click.style(f"Invalid email: {new_email}", fg="red"))
+            return
 
-    try:
-        email_validate(new_email)
-    except:
-        click.echo(click.style(f"Invalid email: {new_email}", fg="red"))
-        return
-
-    account.email = new_email
-    db.session.commit()
-    click.echo(click.style("Email updated successfully.", fg="green"))
+        account.email = new_email
+        click.echo(click.style("Email updated successfully.", fg="green"))
 
 
 @click.command(
@@ -131,25 +138,24 @@ def reset_encrypt_key_pair():
     if dify_config.EDITION != "SELF_HOSTED":
         click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red"))
         return
+    with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
+        tenants = session.query(Tenant).all()
+        for tenant in tenants:
+            if not tenant:
+                click.echo(click.style("No workspaces found. Run /install first.", fg="red"))
+                return
 
-    tenants = db.session.query(Tenant).all()
-    for tenant in tenants:
-        if not tenant:
-            click.echo(click.style("No workspaces found. Run /install first.", fg="red"))
-            return
+            tenant.encrypt_public_key = generate_key_pair(tenant.id)
 
-        tenant.encrypt_public_key = generate_key_pair(tenant.id)
+            session.query(Provider).where(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete()
+            session.query(ProviderModel).where(ProviderModel.tenant_id == tenant.id).delete()
 
-        db.session.query(Provider).where(Provider.provider_type == "custom", Provider.tenant_id == tenant.id).delete()
-        db.session.query(ProviderModel).where(ProviderModel.tenant_id == tenant.id).delete()
-        db.session.commit()
-
-        click.echo(
-            click.style(
-                f"Congratulations! The asymmetric key pair of workspace {tenant.id} has been reset.",
-                fg="green",
+            click.echo(
+                click.style(
+                    f"Congratulations! The asymmetric key pair of workspace {tenant.id} has been reset.",
+                    fg="green",
+                )
             )
-        )
 
 
 @click.command("vdb-migrate", help="Migrate vector db.")
@@ -174,14 +180,15 @@ def migrate_annotation_vector_database():
         try:
             # get apps info
             per_page = 50
-            apps = (
-                db.session.query(App)
-                .where(App.status == "normal")
-                .order_by(App.created_at.desc())
-                .limit(per_page)
-                .offset((page - 1) * per_page)
-                .all()
-            )
+            with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
+                apps = (
+                    session.query(App)
+                    .where(App.status == "normal")
+                    .order_by(App.created_at.desc())
+                    .limit(per_page)
+                    .offset((page - 1) * per_page)
+                    .all()
+                )
             if not apps:
                 break
         except SQLAlchemyError:
@@ -195,26 +202,27 @@ def migrate_annotation_vector_database():
             )
             try:
                 click.echo(f"Creating app annotation index: {app.id}")
-                app_annotation_setting = (
-                    db.session.query(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app.id).first()
-                )
+                with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
+                    app_annotation_setting = (
+                        session.query(AppAnnotationSetting).where(AppAnnotationSetting.app_id == app.id).first()
+                    )
 
-                if not app_annotation_setting:
-                    skipped_count = skipped_count + 1
-                    click.echo(f"App annotation setting disabled: {app.id}")
-                    continue
-                # get dataset_collection_binding info
-                dataset_collection_binding = (
-                    db.session.query(DatasetCollectionBinding)
-                    .where(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id)
-                    .first()
-                )
-                if not dataset_collection_binding:
-                    click.echo(f"App annotation collection binding not found: {app.id}")
-                    continue
-                annotations = db.session.scalars(
-                    select(MessageAnnotation).where(MessageAnnotation.app_id == app.id)
-                ).all()
+                    if not app_annotation_setting:
+                        skipped_count = skipped_count + 1
+                        click.echo(f"App annotation setting disabled: {app.id}")
+                        continue
+                    # get dataset_collection_binding info
+                    dataset_collection_binding = (
+                        session.query(DatasetCollectionBinding)
+                        .where(DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id)
+                        .first()
+                    )
+                    if not dataset_collection_binding:
+                        click.echo(f"App annotation collection binding not found: {app.id}")
+                        continue
+                    annotations = session.scalars(
+                        select(MessageAnnotation).where(MessageAnnotation.app_id == app.id)
+                    ).all()
                 dataset = Dataset(
                     id=app.id,
                     tenant_id=app.tenant_id,
@@ -477,12 +485,12 @@ def convert_to_agent_apps():
             click.echo(f"Converting app: {app.id}")
 
             try:
-                app.mode = AppMode.AGENT_CHAT.value
+                app.mode = AppMode.AGENT_CHAT
                 db.session.commit()
 
                 # update conversation mode to agent
                 db.session.query(Conversation).where(Conversation.app_id == app.id).update(
-                    {Conversation.mode: AppMode.AGENT_CHAT.value}
+                    {Conversation.mode: AppMode.AGENT_CHAT}
                 )
 
                 db.session.commit()
@@ -639,7 +647,7 @@ def old_metadata_migration():
 @click.option("--email", prompt=True, help="Tenant account email.")
 @click.option("--name", prompt=True, help="Workspace name.")
 @click.option("--language", prompt=True, help="Account language, default: en-US.")
-def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None):
+def create_tenant(email: str, language: str | None = None, name: str | None = None):
     """
     Create tenant account
     """
@@ -731,18 +739,18 @@ where sites.id is null limit 1000"""
                 try:
                     app = db.session.query(App).where(App.id == app_id).first()
                     if not app:
-                        print(f"App {app_id} not found")
+                        logger.info("App %s not found", app_id)
                         continue
 
                     tenant = app.tenant
                     if tenant:
                         accounts = tenant.get_accounts()
                         if not accounts:
-                            print(f"Fix failed for app {app.id}")
+                            logger.info("Fix failed for app %s", app.id)
                             continue
 
                         account = accounts[0]
-                        print(f"Fixing missing site for app {app.id}")
+                        logger.info("Fixing missing site for app %s", app.id)
                         app_was_created.send(app, account=account)
                 except Exception:
                     failed_app_ids.append(app_id)
@@ -953,7 +961,7 @@ def clear_orphaned_file_records(force: bool):
             click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
             query = "DELETE FROM message_files WHERE id IN :ids"
             with db.engine.begin() as conn:
-                conn.execute(sa.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+                conn.execute(sa.text(query), {"ids": tuple(record["id"] for record in orphaned_message_files)})
             click.echo(
                 click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
             )
@@ -1245,15 +1253,17 @@ def _find_orphaned_draft_variables(batch_size: int = 1000) -> list[str]:
 
 def _count_orphaned_draft_variables() -> dict[str, Any]:
     """
-    Count orphaned draft variables by app.
+    Count orphaned draft variables by app, including associated file counts.
 
     Returns:
-        Dictionary with statistics about orphaned variables
+        Dictionary with statistics about orphaned variables and files
     """
-    query = """
+    # Count orphaned variables by app
+    variables_query = """
         SELECT
             wdv.app_id,
-            COUNT(*) as variable_count
+            COUNT(*) as variable_count,
+            COUNT(wdv.file_id) as file_count
         FROM workflow_draft_variables AS wdv
         WHERE NOT EXISTS(
             SELECT 1 FROM apps WHERE apps.id = wdv.app_id
@@ -1263,14 +1273,21 @@ def _count_orphaned_draft_variables() -> dict[str, Any]:
     """
 
     with db.engine.connect() as conn:
-        result = conn.execute(sa.text(query))
-        orphaned_by_app = {row[0]: row[1] for row in result}
+        result = conn.execute(sa.text(variables_query))
+        orphaned_by_app = {}
+        total_files = 0
 
-        total_orphaned = sum(orphaned_by_app.values())
+        for row in result:
+            app_id, variable_count, file_count = row
+            orphaned_by_app[app_id] = {"variables": variable_count, "files": file_count}
+            total_files += file_count
+
+        total_orphaned = sum(app_data["variables"] for app_data in orphaned_by_app.values())
         app_count = len(orphaned_by_app)
 
         return {
             "total_orphaned_variables": total_orphaned,
+            "total_orphaned_files": total_files,
             "orphaned_app_count": app_count,
             "orphaned_by_app": orphaned_by_app,
         }
@@ -1299,6 +1316,7 @@ def cleanup_orphaned_draft_variables(
     stats = _count_orphaned_draft_variables()
 
     logger.info("Found %s orphaned draft variables", stats["total_orphaned_variables"])
+    logger.info("Found %s associated offload files", stats["total_orphaned_files"])
     logger.info("Across %s non-existent apps", stats["orphaned_app_count"])
 
     if stats["total_orphaned_variables"] == 0:
@@ -1307,10 +1325,10 @@ def cleanup_orphaned_draft_variables(
 
     if dry_run:
         logger.info("DRY RUN: Would delete the following:")
-        for app_id, count in sorted(stats["orphaned_by_app"].items(), key=lambda x: x[1], reverse=True)[
+        for app_id, data in sorted(stats["orphaned_by_app"].items(), key=lambda x: x[1]["variables"], reverse=True)[
             :10
         ]:  # Show top 10
-            logger.info("  App %s: %s variables", app_id, count)
+            logger.info("  App %s: %s variables, %s files", app_id, data["variables"], data["files"])
         if len(stats["orphaned_by_app"]) > 10:
             logger.info("  ... and %s more apps", len(stats["orphaned_by_app"]) - 10)
         return
@@ -1319,7 +1337,8 @@ def cleanup_orphaned_draft_variables(
     if not force:
         click.confirm(
             f"Are you sure you want to delete {stats['total_orphaned_variables']} "
-            f"orphaned draft variables from {stats['orphaned_app_count']} apps?",
+            f"orphaned draft variables and {stats['total_orphaned_files']} associated files "
+            f"from {stats['orphaned_app_count']} apps?",
             abort=True,
         )
 
@@ -1352,3 +1371,456 @@ def cleanup_orphaned_draft_variables(
                 continue
 
     logger.info("Cleanup completed. Total deleted: %s variables across %s apps", total_deleted, processed_apps)
+
+
+@click.command("setup-datasource-oauth-client", help="Setup datasource oauth client.")
+@click.option("--provider", prompt=True, help="Provider name")
+@click.option("--client-params", prompt=True, help="Client Params")
+def setup_datasource_oauth_client(provider, client_params):
+    """
+    Setup datasource oauth client
+    """
+    provider_id = DatasourceProviderID(provider)
+    provider_name = provider_id.provider_name
+    plugin_id = provider_id.plugin_id
+
+    try:
+        # json validate
+        click.echo(click.style(f"Validating client params: {client_params}", fg="yellow"))
+        client_params_dict = TypeAdapter(dict[str, Any]).validate_json(client_params)
+        click.echo(click.style("Client params validated successfully.", fg="green"))
+    except Exception as e:
+        click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red"))
+        return
+
+    click.echo(click.style(f"Ready to delete existing oauth client params: {provider_name}", fg="yellow"))
+    deleted_count = (
+        db.session.query(DatasourceOauthParamConfig)
+        .filter_by(
+            provider=provider_name,
+            plugin_id=plugin_id,
+        )
+        .delete()
+    )
+    if deleted_count > 0:
+        click.echo(click.style(f"Deleted {deleted_count} existing oauth client params.", fg="yellow"))
+
+    click.echo(click.style(f"Ready to setup datasource oauth client: {provider_name}", fg="yellow"))
+    oauth_client = DatasourceOauthParamConfig(
+        provider=provider_name,
+        plugin_id=plugin_id,
+        system_credentials=client_params_dict,
+    )
+    db.session.add(oauth_client)
+    db.session.commit()
+    click.echo(click.style(f"provider: {provider_name}", fg="green"))
+    click.echo(click.style(f"plugin_id: {plugin_id}", fg="green"))
+    click.echo(click.style(f"params: {json.dumps(client_params_dict, indent=2, ensure_ascii=False)}", fg="green"))
+    click.echo(click.style(f"Datasource oauth client setup successfully. id: {oauth_client.id}", fg="green"))
+
+
+@click.command("transform-datasource-credentials", help="Transform datasource credentials.")
+def transform_datasource_credentials():
+    """
+    Transform datasource credentials
+    """
+    try:
+        installer_manager = PluginInstaller()
+        plugin_migration = PluginMigration()
+
+        notion_plugin_id = "langgenius/notion_datasource"
+        firecrawl_plugin_id = "langgenius/firecrawl_datasource"
+        jina_plugin_id = "langgenius/jina_datasource"
+        notion_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(notion_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        firecrawl_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(firecrawl_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        jina_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(jina_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        oauth_credential_type = CredentialType.OAUTH2
+        api_key_credential_type = CredentialType.API_KEY
+
+        # deal notion credentials
+        deal_notion_count = 0
+        notion_credentials = db.session.query(DataSourceOauthBinding).filter_by(provider="notion").all()
+        if notion_credentials:
+            notion_credentials_tenant_mapping: dict[str, list[DataSourceOauthBinding]] = {}
+            for notion_credential in notion_credentials:
+                tenant_id = notion_credential.tenant_id
+                if tenant_id not in notion_credentials_tenant_mapping:
+                    notion_credentials_tenant_mapping[tenant_id] = []
+                notion_credentials_tenant_mapping[tenant_id].append(notion_credential)
+            for tenant_id, notion_tenant_credentials in notion_credentials_tenant_mapping.items():
+                tenant = db.session.query(Tenant).filter_by(id=tenant_id).first()
+                if not tenant:
+                    continue
+                try:
+                    # check notion plugin is installed
+                    installed_plugins = installer_manager.list_plugins(tenant_id)
+                    installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                    if notion_plugin_id not in installed_plugins_ids:
+                        if notion_plugin_unique_identifier:
+                            # install notion plugin
+                            PluginService.install_from_marketplace_pkg(tenant_id, [notion_plugin_unique_identifier])
+                    auth_count = 0
+                    for notion_tenant_credential in notion_tenant_credentials:
+                        auth_count += 1
+                        # get credential oauth params
+                        access_token = notion_tenant_credential.access_token
+                        # notion info
+                        notion_info = notion_tenant_credential.source_info
+                        workspace_id = notion_info.get("workspace_id")
+                        workspace_name = notion_info.get("workspace_name")
+                        workspace_icon = notion_info.get("workspace_icon")
+                        new_credentials = {
+                            "integration_secret": encrypter.encrypt_token(tenant_id, access_token),
+                            "workspace_id": workspace_id,
+                            "workspace_name": workspace_name,
+                            "workspace_icon": workspace_icon,
+                        }
+                        datasource_provider = DatasourceProvider(
+                            provider="notion_datasource",
+                            tenant_id=tenant_id,
+                            plugin_id=notion_plugin_id,
+                            auth_type=oauth_credential_type.value,
+                            encrypted_credentials=new_credentials,
+                            name=f"Auth {auth_count}",
+                            avatar_url=workspace_icon or "default",
+                            is_default=False,
+                        )
+                        db.session.add(datasource_provider)
+                        deal_notion_count += 1
+                except Exception as e:
+                    click.echo(
+                        click.style(
+                            f"Error transforming notion credentials: {str(e)}, tenant_id: {tenant_id}", fg="red"
+                        )
+                    )
+                    continue
+                db.session.commit()
+        # deal firecrawl credentials
+        deal_firecrawl_count = 0
+        firecrawl_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="firecrawl").all()
+        if firecrawl_credentials:
+            firecrawl_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {}
+            for firecrawl_credential in firecrawl_credentials:
+                tenant_id = firecrawl_credential.tenant_id
+                if tenant_id not in firecrawl_credentials_tenant_mapping:
+                    firecrawl_credentials_tenant_mapping[tenant_id] = []
+                firecrawl_credentials_tenant_mapping[tenant_id].append(firecrawl_credential)
+            for tenant_id, firecrawl_tenant_credentials in firecrawl_credentials_tenant_mapping.items():
+                tenant = db.session.query(Tenant).filter_by(id=tenant_id).first()
+                if not tenant:
+                    continue
+                try:
+                    # check firecrawl plugin is installed
+                    installed_plugins = installer_manager.list_plugins(tenant_id)
+                    installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                    if firecrawl_plugin_id not in installed_plugins_ids:
+                        if firecrawl_plugin_unique_identifier:
+                            # install firecrawl plugin
+                            PluginService.install_from_marketplace_pkg(tenant_id, [firecrawl_plugin_unique_identifier])
+
+                    auth_count = 0
+                    for firecrawl_tenant_credential in firecrawl_tenant_credentials:
+                        auth_count += 1
+                        # get credential api key
+                        credentials_json = json.loads(firecrawl_tenant_credential.credentials)
+                        api_key = credentials_json.get("config", {}).get("api_key")
+                        base_url = credentials_json.get("config", {}).get("base_url")
+                        new_credentials = {
+                            "firecrawl_api_key": api_key,
+                            "base_url": base_url,
+                        }
+                        datasource_provider = DatasourceProvider(
+                            provider="firecrawl",
+                            tenant_id=tenant_id,
+                            plugin_id=firecrawl_plugin_id,
+                            auth_type=api_key_credential_type.value,
+                            encrypted_credentials=new_credentials,
+                            name=f"Auth {auth_count}",
+                            avatar_url="default",
+                            is_default=False,
+                        )
+                        db.session.add(datasource_provider)
+                        deal_firecrawl_count += 1
+                except Exception as e:
+                    click.echo(
+                        click.style(
+                            f"Error transforming firecrawl credentials: {str(e)}, tenant_id: {tenant_id}", fg="red"
+                        )
+                    )
+                    continue
+                db.session.commit()
+        # deal jina credentials
+        deal_jina_count = 0
+        jina_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="jinareader").all()
+        if jina_credentials:
+            jina_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {}
+            for jina_credential in jina_credentials:
+                tenant_id = jina_credential.tenant_id
+                if tenant_id not in jina_credentials_tenant_mapping:
+                    jina_credentials_tenant_mapping[tenant_id] = []
+                jina_credentials_tenant_mapping[tenant_id].append(jina_credential)
+            for tenant_id, jina_tenant_credentials in jina_credentials_tenant_mapping.items():
+                tenant = db.session.query(Tenant).filter_by(id=tenant_id).first()
+                if not tenant:
+                    continue
+                try:
+                    # check jina plugin is installed
+                    installed_plugins = installer_manager.list_plugins(tenant_id)
+                    installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                    if jina_plugin_id not in installed_plugins_ids:
+                        if jina_plugin_unique_identifier:
+                            # install jina plugin
+                            logger.debug("Installing Jina plugin %s", jina_plugin_unique_identifier)
+                            PluginService.install_from_marketplace_pkg(tenant_id, [jina_plugin_unique_identifier])
+
+                    auth_count = 0
+                    for jina_tenant_credential in jina_tenant_credentials:
+                        auth_count += 1
+                        # get credential api key
+                        credentials_json = json.loads(jina_tenant_credential.credentials)
+                        api_key = credentials_json.get("config", {}).get("api_key")
+                        new_credentials = {
+                            "integration_secret": api_key,
+                        }
+                        datasource_provider = DatasourceProvider(
+                            provider="jina",
+                            tenant_id=tenant_id,
+                            plugin_id=jina_plugin_id,
+                            auth_type=api_key_credential_type.value,
+                            encrypted_credentials=new_credentials,
+                            name=f"Auth {auth_count}",
+                            avatar_url="default",
+                            is_default=False,
+                        )
+                        db.session.add(datasource_provider)
+                        deal_jina_count += 1
+                except Exception as e:
+                    click.echo(
+                        click.style(f"Error transforming jina credentials: {str(e)}, tenant_id: {tenant_id}", fg="red")
+                    )
+                    continue
+                db.session.commit()
+    except Exception as e:
+        click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red"))
+        return
+    click.echo(click.style(f"Transforming notion successfully. deal_notion_count: {deal_notion_count}", fg="green"))
+    click.echo(
+        click.style(f"Transforming firecrawl successfully. deal_firecrawl_count: {deal_firecrawl_count}", fg="green")
+    )
+    click.echo(click.style(f"Transforming jina successfully. deal_jina_count: {deal_jina_count}", fg="green"))
+
+
+@click.command("install-rag-pipeline-plugins", help="Install rag pipeline plugins.")
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+@click.option(
+    "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
+)
+@click.option("--workers", prompt=True, help="The number of workers to install plugins.", default=100)
+def install_rag_pipeline_plugins(input_file, output_file, workers):
+    """
+    Install rag pipeline plugins
+    """
+    click.echo(click.style("Installing rag pipeline plugins", fg="yellow"))
+    plugin_migration = PluginMigration()
+    plugin_migration.install_rag_pipeline_plugins(
+        input_file,
+        output_file,
+        workers,
+    )
+    click.echo(click.style("Installing rag pipeline plugins successfully", fg="green"))
+
+
+@click.command(
+    "migrate-oss",
+    help="Migrate files from Local or OpenDAL source to a cloud OSS storage (destination must NOT be local/opendal).",
+)
+@click.option(
+    "--path",
+    "paths",
+    multiple=True,
+    help="Storage path prefixes to migrate (repeatable). Defaults: privkeys, upload_files, image_files,"
+    " tools, website_files, keyword_files, ops_trace",
+)
+@click.option(
+    "--source",
+    type=click.Choice(["local", "opendal"], case_sensitive=False),
+    default="opendal",
+    show_default=True,
+    help="Source storage type to read from",
+)
+@click.option("--overwrite", is_flag=True, default=False, help="Overwrite destination if file already exists")
+@click.option("--dry-run", is_flag=True, default=False, help="Show what would be migrated without uploading")
+@click.option("-f", "--force", is_flag=True, help="Skip confirmation and run without prompts")
+@click.option(
+    "--update-db/--no-update-db",
+    default=True,
+    help="Update upload_files.storage_type from source type to current storage after migration",
+)
+def migrate_oss(
+    paths: tuple[str, ...],
+    source: str,
+    overwrite: bool,
+    dry_run: bool,
+    force: bool,
+    update_db: bool,
+):
+    """
+    Copy all files under selected prefixes from a source storage
+    (Local filesystem or OpenDAL-backed) into the currently configured
+    destination storage backend, then optionally update DB records.
+
+    Expected usage: set STORAGE_TYPE (and its credentials) to your target backend.
+    """
+    # Ensure target storage is not local/opendal
+    if dify_config.STORAGE_TYPE in (StorageType.LOCAL, StorageType.OPENDAL):
+        click.echo(
+            click.style(
+                "Target STORAGE_TYPE must be a cloud OSS (not 'local' or 'opendal').\n"
+                "Please set STORAGE_TYPE to one of: s3, aliyun-oss, azure-blob, google-storage, tencent-cos, \n"
+                "volcengine-tos, supabase, oci-storage, huawei-obs, baidu-obs, clickzetta-volume.",
+                fg="red",
+            )
+        )
+        return
+
+    # Default paths if none specified
+    default_paths = ("privkeys", "upload_files", "image_files", "tools", "website_files", "keyword_files", "ops_trace")
+    path_list = list(paths) if paths else list(default_paths)
+    is_source_local = source.lower() == "local"
+
+    click.echo(click.style("Preparing migration to target storage.", fg="yellow"))
+    click.echo(click.style(f"Target storage type: {dify_config.STORAGE_TYPE}", fg="white"))
+    if is_source_local:
+        src_root = dify_config.STORAGE_LOCAL_PATH
+        click.echo(click.style(f"Source: local fs, root: {src_root}", fg="white"))
+    else:
+        click.echo(click.style(f"Source: opendal scheme={dify_config.OPENDAL_SCHEME}", fg="white"))
+    click.echo(click.style(f"Paths to migrate: {', '.join(path_list)}", fg="white"))
+    click.echo("")
+
+    if not force:
+        click.confirm("Proceed with migration?", abort=True)
+
+    # Instantiate source storage
+    try:
+        if is_source_local:
+            src_root = dify_config.STORAGE_LOCAL_PATH
+            source_storage = OpenDALStorage(scheme="fs", root=src_root)
+        else:
+            source_storage = OpenDALStorage(scheme=dify_config.OPENDAL_SCHEME)
+    except Exception as e:
+        click.echo(click.style(f"Failed to initialize source storage: {str(e)}", fg="red"))
+        return
+
+    total_files = 0
+    copied_files = 0
+    skipped_files = 0
+    errored_files = 0
+    copied_upload_file_keys: list[str] = []
+
+    for prefix in path_list:
+        click.echo(click.style(f"Scanning source path: {prefix}", fg="white"))
+        try:
+            keys = source_storage.scan(path=prefix, files=True, directories=False)
+        except FileNotFoundError:
+            click.echo(click.style(f"  -> Skipping missing path: {prefix}", fg="yellow"))
+            continue
+        except NotImplementedError:
+            click.echo(click.style("  -> Source storage does not support scanning.", fg="red"))
+            return
+        except Exception as e:
+            click.echo(click.style(f"  -> Error scanning '{prefix}': {str(e)}", fg="red"))
+            continue
+
+        click.echo(click.style(f"Found {len(keys)} files under {prefix}", fg="white"))
+
+        for key in keys:
+            total_files += 1
+
+            # check destination existence
+            if not overwrite:
+                try:
+                    if storage.exists(key):
+                        skipped_files += 1
+                        continue
+                except Exception as e:
+                    # existence check failures should not block migration attempt
+                    # but should be surfaced to user as a warning for visibility
+                    click.echo(
+                        click.style(
+                            f"  -> Warning: failed target existence check for {key}: {str(e)}",
+                            fg="yellow",
+                        )
+                    )
+
+            if dry_run:
+                copied_files += 1
+                continue
+
+            # read from source and write to destination
+            try:
+                data = source_storage.load_once(key)
+            except FileNotFoundError:
+                errored_files += 1
+                click.echo(click.style(f"  -> Missing on source: {key}", fg="yellow"))
+                continue
+            except Exception as e:
+                errored_files += 1
+                click.echo(click.style(f"  -> Error reading {key}: {str(e)}", fg="red"))
+                continue
+
+            try:
+                storage.save(key, data)
+                copied_files += 1
+                if prefix == "upload_files":
+                    copied_upload_file_keys.append(key)
+            except Exception as e:
+                errored_files += 1
+                click.echo(click.style(f"  -> Error writing {key} to target: {str(e)}", fg="red"))
+                continue
+
+    click.echo("")
+    click.echo(click.style("Migration summary:", fg="yellow"))
+    click.echo(click.style(f"  Total:   {total_files}", fg="white"))
+    click.echo(click.style(f"  Copied:  {copied_files}", fg="green"))
+    click.echo(click.style(f"  Skipped: {skipped_files}", fg="white"))
+    if errored_files:
+        click.echo(click.style(f"  Errors:  {errored_files}", fg="red"))
+
+    if dry_run:
+        click.echo(click.style("Dry-run complete. No changes were made.", fg="green"))
+        return
+
+    if errored_files:
+        click.echo(
+            click.style(
+                "Some files failed to migrate. Review errors above before updating DB records.",
+                fg="yellow",
+            )
+        )
+        if update_db and not force:
+            if not click.confirm("Proceed to update DB storage_type despite errors?", default=False):
+                update_db = False
+
+    # Optionally update DB records for upload_files.storage_type (only for successfully copied upload_files)
+    if update_db:
+        if not copied_upload_file_keys:
+            click.echo(click.style("No upload_files copied. Skipping DB storage_type update.", fg="yellow"))
+        else:
+            try:
+                source_storage_type = StorageType.LOCAL if is_source_local else StorageType.OPENDAL
+                updated = (
+                    db.session.query(UploadFile)
+                    .where(
+                        UploadFile.storage_type == source_storage_type,
+                        UploadFile.key.in_(copied_upload_file_keys),
+                    )
+                    .update({UploadFile.storage_type: dify_config.STORAGE_TYPE}, synchronize_session=False)
+                )
+                db.session.commit()
+                click.echo(click.style(f"Updated storage_type for {updated} upload_files records.", fg="green"))
+            except Exception as e:
+                db.session.rollback()
+                click.echo(click.style(f"Failed to update DB storage_type: {str(e)}", fg="red"))

api/configs/__init__.py

@@ -1,3 +1,3 @@
 from .app_config import DifyConfig
 
-dify_config = DifyConfig()
+dify_config = DifyConfig()  # type: ignore

api/configs/extra/notion_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,28 +7,28 @@ class NotionConfig(BaseSettings):
     Configuration settings for Notion integration
     """
 
-    NOTION_CLIENT_ID: Optional[str] = Field(
+    NOTION_CLIENT_ID: str | None = Field(
         description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_CLIENT_SECRET: Optional[str] = Field(
+    NOTION_CLIENT_SECRET: str | None = Field(
         description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+    NOTION_INTEGRATION_TYPE: str | None = Field(
         description="Type of Notion integration."
         " Set to 'internal' for internal integrations, or None for public integrations.",
         default=None,
     )
 
-    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+    NOTION_INTERNAL_SECRET: str | None = Field(
         description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+    NOTION_INTEGRATION_TOKEN: str | None = Field(
         description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
         default=None,
     )

api/configs/extra/sentry_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeFloat
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class SentryConfig(BaseSettings):
     Configuration settings for Sentry error tracking and performance monitoring
     """
 
-    SENTRY_DSN: Optional[str] = Field(
+    SENTRY_DSN: str | None = Field(
         description="Sentry Data Source Name (DSN)."
         " This is the unique identifier of your Sentry project, used to send events to the correct project.",
         default=None,

api/configs/feature/__init__.py

@@ -1,4 +1,5 @@
-from typing import Literal, Optional
+from enum import StrEnum
+from typing import Literal
 
 from pydantic import (
     AliasChoices,
@@ -31,6 +32,12 @@ class SecurityConfig(BaseSettings):
         description="Duration in minutes for which a password reset token remains valid",
         default=5,
     )
+
+    EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
+        description="Duration in minutes for which a email register token remains valid",
+        default=5,
+    )
+
     CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
         description="Duration in minutes for which a change email token remains valid",
         default=5,
@@ -51,7 +58,7 @@ class SecurityConfig(BaseSettings):
         default=False,
     )
 
-    ADMIN_API_KEY: Optional[str] = Field(
+    ADMIN_API_KEY: str | None = Field(
         description="admin api key for authentication",
         default=None,
     )
@@ -91,21 +98,36 @@ class CodeExecutionSandboxConfig(BaseSettings):
         default="dify-sandbox",
     )
 
-    CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_CONNECT_TIMEOUT: float | None = Field(
         description="Connection timeout in seconds for code execution requests",
         default=10.0,
     )
 
-    CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_READ_TIMEOUT: float | None = Field(
         description="Read timeout in seconds for code execution requests",
         default=60.0,
     )
 
-    CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_WRITE_TIMEOUT: float | None = Field(
         description="Write timeout in seconds for code execution request",
         default=10.0,
     )
 
+    CODE_EXECUTION_POOL_MAX_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of concurrent connections for the code execution HTTP client",
+        default=100,
+    )
+
+    CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of persistent keep-alive connections for the code execution HTTP client",
+        default=20,
+    )
+
+    CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY: PositiveFloat | None = Field(
+        description="Keep-alive expiry in seconds for idle connections (set to None to disable)",
+        default=5.0,
+    )
+
     CODE_MAX_NUMBER: PositiveInt = Field(
         description="Maximum allowed numeric value in code execution",
         default=9223372036854775807,
@@ -146,6 +168,11 @@ class CodeExecutionSandboxConfig(BaseSettings):
         default=1000,
     )
 
+    CODE_EXECUTION_SSL_VERIFY: bool = Field(
+        description="Enable or disable SSL verification for code execution requests",
+        default=True,
+    )
+
 
 class PluginConfig(BaseSettings):
     """
@@ -362,17 +389,17 @@ class HttpConfig(BaseSettings):
         default=3,
     )
 
-    SSRF_PROXY_ALL_URL: Optional[str] = Field(
+    SSRF_PROXY_ALL_URL: str | None = Field(
         description="Proxy URL for HTTP or HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTP_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTP_URL: str | None = Field(
         description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTPS_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTPS_URL: str | None = Field(
         description="Proxy URL for HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
@@ -397,6 +424,21 @@ class HttpConfig(BaseSettings):
         default=5,
     )
 
+    SSRF_POOL_MAX_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of concurrent connections for the SSRF HTTP client",
+        default=100,
+    )
+
+    SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of persistent keep-alive connections for the SSRF HTTP client",
+        default=20,
+    )
+
+    SSRF_POOL_KEEPALIVE_EXPIRY: PositiveFloat | None = Field(
+        description="Keep-alive expiry in seconds for idle SSRF connections (set to None to disable)",
+        default=5.0,
+    )
+
     RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field(
         description="Enable handling of X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers"
         " when the app is behind a single trusted reverse proxy.",
@@ -414,7 +456,7 @@ class InnerAPIConfig(BaseSettings):
         default=False,
     )
 
-    INNER_API_KEY: Optional[str] = Field(
+    INNER_API_KEY: str | None = Field(
         description="API key for accessing the internal API",
         default=None,
     )
@@ -430,7 +472,7 @@ class LoggingConfig(BaseSettings):
         default="INFO",
     )
 
-    LOG_FILE: Optional[str] = Field(
+    LOG_FILE: str | None = Field(
         description="File path for log output.",
         default=None,
     )
@@ -450,12 +492,12 @@ class LoggingConfig(BaseSettings):
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
     )
 
-    LOG_DATEFORMAT: Optional[str] = Field(
+    LOG_DATEFORMAT: str | None = Field(
         description="Date format string for log timestamps",
         default=None,
     )
 
-    LOG_TZ: Optional[str] = Field(
+    LOG_TZ: str | None = Field(
         description="Timezone for log timestamps (e.g., 'America/New_York')",
         default="UTC",
     )
@@ -499,6 +541,22 @@ class UpdateConfig(BaseSettings):
     )
 
 
+class WorkflowVariableTruncationConfig(BaseSettings):
+    WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE: PositiveInt = Field(
+        # 100KB
+        1024_000,
+        description="Maximum size for variable to trigger final truncation.",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH: PositiveInt = Field(
+        100000,
+        description="maximum length for string to trigger tuncation, measure in number of characters",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_ARRAY_LENGTH: PositiveInt = Field(
+        1000,
+        description="maximum length for array to trigger truncation.",
+    )
+
+
 class WorkflowConfig(BaseSettings):
     """
     Configuration for workflow execution
@@ -519,16 +577,33 @@ class WorkflowConfig(BaseSettings):
         default=5,
     )
 
-    WORKFLOW_PARALLEL_DEPTH_LIMIT: PositiveInt = Field(
-        description="Maximum allowed depth for nested parallel executions",
-        default=3,
-    )
-
     MAX_VARIABLE_SIZE: PositiveInt = Field(
         description="Maximum size in bytes for a single variable in workflows. Default to 200 KB.",
         default=200 * 1024,
     )
 
+    # GraphEngine Worker Pool Configuration
+    GRAPH_ENGINE_MIN_WORKERS: PositiveInt = Field(
+        description="Minimum number of workers per GraphEngine instance",
+        default=1,
+    )
+
+    GRAPH_ENGINE_MAX_WORKERS: PositiveInt = Field(
+        description="Maximum number of workers per GraphEngine instance",
+        default=10,
+    )
+
+    GRAPH_ENGINE_SCALE_UP_THRESHOLD: PositiveInt = Field(
+        description="Queue depth threshold that triggers worker scale up",
+        default=3,
+    )
+
+    GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME: float = Field(
+        description="Seconds of idle time before scaling down workers",
+        default=5.0,
+        ge=0.1,
+    )
+
 
 class WorkflowNodeExecutionConfig(BaseSettings):
     """
@@ -589,22 +664,22 @@ class AuthConfig(BaseSettings):
         default="/console/api/oauth/authorize",
     )
 
-    GITHUB_CLIENT_ID: Optional[str] = Field(
+    GITHUB_CLIENT_ID: str | None = Field(
         description="GitHub OAuth client ID",
         default=None,
     )
 
-    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+    GITHUB_CLIENT_SECRET: str | None = Field(
         description="GitHub OAuth client secret",
         default=None,
     )
 
-    GOOGLE_CLIENT_ID: Optional[str] = Field(
+    GOOGLE_CLIENT_ID: str | None = Field(
         description="Google OAuth client ID",
         default=None,
     )
 
-    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+    GOOGLE_CLIENT_SECRET: str | None = Field(
         description="Google OAuth client secret",
         default=None,
     )
@@ -639,6 +714,11 @@ class AuthConfig(BaseSettings):
         default=86400,
     )
 
+    EMAIL_REGISTER_LOCKOUT_DURATION: PositiveInt = Field(
+        description="Time (in seconds) a user must wait before retrying email register after exceeding the rate limit.",
+        default=86400,
+    )
+
 
 class ModerationConfig(BaseSettings):
     """
@@ -662,47 +742,71 @@ class ToolConfig(BaseSettings):
     )
 
 
+class TemplateMode(StrEnum):
+    # unsafe mode allows flexible operations in templates, but may cause security vulnerabilities
+    UNSAFE = "unsafe"
+
+    # sandbox mode restricts some unsafe operations like accessing __class__.
+    # however, it is still not 100% safe, for example, cpu exploitation can happen.
+    SANDBOX = "sandbox"
+
+    # templating is disabled
+    DISABLED = "disabled"
+
+
 class MailConfig(BaseSettings):
     """
     Configuration for email services
     """
 
-    MAIL_TYPE: Optional[str] = Field(
+    MAIL_TEMPLATING_MODE: TemplateMode = Field(
+        description="Template mode for email services",
+        default=TemplateMode.SANDBOX,
+    )
+
+    MAIL_TEMPLATING_TIMEOUT: int = Field(
+        description="""
+        Timeout for email templating in seconds. Used to prevent infinite loops in malicious templates. 
+        Only available in sandbox mode.""",
+        default=3,
+    )
+
+    MAIL_TYPE: str | None = Field(
         description="Email service provider type ('smtp' or 'resend' or 'sendGrid), default to None.",
         default=None,
     )
 
-    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+    MAIL_DEFAULT_SEND_FROM: str | None = Field(
         description="Default email address to use as the sender",
         default=None,
     )
 
-    RESEND_API_KEY: Optional[str] = Field(
+    RESEND_API_KEY: str | None = Field(
         description="API key for Resend email service",
         default=None,
     )
 
-    RESEND_API_URL: Optional[str] = Field(
+    RESEND_API_URL: str | None = Field(
         description="API URL for Resend email service",
         default=None,
     )
 
-    SMTP_SERVER: Optional[str] = Field(
+    SMTP_SERVER: str | None = Field(
         description="SMTP server hostname",
         default=None,
     )
 
-    SMTP_PORT: Optional[int] = Field(
+    SMTP_PORT: int | None = Field(
         description="SMTP server port number",
         default=465,
     )
 
-    SMTP_USERNAME: Optional[str] = Field(
+    SMTP_USERNAME: str | None = Field(
         description="Username for SMTP authentication",
         default=None,
     )
 
-    SMTP_PASSWORD: Optional[str] = Field(
+    SMTP_PASSWORD: str | None = Field(
         description="Password for SMTP authentication",
         default=None,
     )
@@ -722,7 +826,7 @@ class MailConfig(BaseSettings):
         default=50,
     )
 
-    SENDGRID_API_KEY: Optional[str] = Field(
+    SENDGRID_API_KEY: str | None = Field(
         description="API key for SendGrid service",
         default=None,
     )
@@ -745,17 +849,17 @@ class RagEtlConfig(BaseSettings):
         default="database",
     )
 
-    UNSTRUCTURED_API_URL: Optional[str] = Field(
+    UNSTRUCTURED_API_URL: str | None = Field(
         description="API URL for Unstructured.io service",
         default=None,
     )
 
-    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+    UNSTRUCTURED_API_KEY: str | None = Field(
         description="API key for Unstructured.io service",
         default="",
     )
 
-    SCARF_NO_ANALYTICS: Optional[str] = Field(
+    SCARF_NO_ANALYTICS: str | None = Field(
         description="This is about whether to disable Scarf analytics in Unstructured library.",
         default="false",
     )
@@ -1030,5 +1134,6 @@ class FeatureConfig(
     CeleryBeatConfig,
     CeleryScheduleTasksConfig,
     WorkflowLogConfig,
+    WorkflowVariableTruncationConfig,
 ):
     pass

api/configs/feature/hosted_service/__init__.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt
 from pydantic_settings import BaseSettings
 
@@ -40,17 +38,17 @@ class HostedOpenAiConfig(BaseSettings):
     Configuration for hosted OpenAI service
     """
 
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted OpenAI service",
         default=None,
     )
 
-    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted OpenAI API",
         default=None,
     )
 
-    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
+    HOSTED_OPENAI_API_ORGANIZATION: str | None = Field(
         description="Organization ID for hosted OpenAI service",
         default=None,
     )
@@ -110,12 +108,12 @@ class HostedAzureOpenAiConfig(BaseSettings):
         default=False,
     )
 
-    HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted Azure OpenAI service",
         default=None,
     )
 
-    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted Azure OpenAI API",
         default=None,
     )
@@ -131,12 +129,12 @@ class HostedAnthropicConfig(BaseSettings):
     Configuration for hosted Anthropic service
     """
 
-    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_BASE: str | None = Field(
         description="Base URL for hosted Anthropic API",
         default=None,
     )
 
-    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_KEY: str | None = Field(
         description="API key for hosted Anthropic service",
         default=None,
     )
@@ -222,11 +220,28 @@ class HostedFetchAppTemplateConfig(BaseSettings):
     )
 
 
+class HostedFetchPipelineTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching pipeline templates
+    """
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching pipeline templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote pipeline templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
 class HostedServiceConfig(
     # place the configs in alphabet order
     HostedAnthropicConfig,
     HostedAzureOpenAiConfig,
     HostedFetchAppTemplateConfig,
+    HostedFetchPipelineTemplateConfig,
     HostedMinmaxConfig,
     HostedOpenAiConfig,
     HostedSparkConfig,

api/configs/middleware/__init__.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 from urllib.parse import parse_qsl, quote_plus
 
 from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
@@ -78,18 +78,18 @@ class StorageConfig(BaseSettings):
 
 
 class VectorStoreConfig(BaseSettings):
-    VECTOR_STORE: Optional[str] = Field(
+    VECTOR_STORE: str | None = Field(
         description="Type of vector store to use for efficient similarity search."
         " Set to None if not using a vector store.",
         default=None,
     )
 
-    VECTOR_STORE_WHITELIST_ENABLE: Optional[bool] = Field(
+    VECTOR_STORE_WHITELIST_ENABLE: bool | None = Field(
         description="Enable whitelist for vector store.",
         default=False,
     )
 
-    VECTOR_INDEX_NAME_PREFIX: Optional[str] = Field(
+    VECTOR_INDEX_NAME_PREFIX: str | None = Field(
         description="Prefix used to create collection name in vector database",
         default="Vector_index",
     )
@@ -187,6 +187,11 @@ class DatabaseConfig(BaseSettings):
         default=False,
     )
 
+    SQLALCHEMY_POOL_TIMEOUT: NonNegativeInt = Field(
+        description="Number of seconds to wait for a connection from the pool before raising a timeout error.",
+        default=30,
+    )
+
     RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
         description="Number of processes for the retrieval service, default to CPU cores.",
         default=os.cpu_count() or 1,
@@ -216,6 +221,7 @@ class DatabaseConfig(BaseSettings):
             "connect_args": connect_args,
             "pool_use_lifo": self.SQLALCHEMY_POOL_USE_LIFO,
             "pool_reset_on_return": None,
+            "pool_timeout": self.SQLALCHEMY_POOL_TIMEOUT,
         }
 
 
@@ -225,26 +231,26 @@ class CeleryConfig(DatabaseConfig):
         default="redis",
     )
 
-    CELERY_BROKER_URL: Optional[str] = Field(
+    CELERY_BROKER_URL: str | None = Field(
         description="URL of the message broker for Celery tasks.",
         default=None,
     )
 
-    CELERY_USE_SENTINEL: Optional[bool] = Field(
+    CELERY_USE_SENTINEL: bool | None = Field(
         description="Whether to use Redis Sentinel for high availability.",
         default=False,
     )
 
-    CELERY_SENTINEL_MASTER_NAME: Optional[str] = Field(
+    CELERY_SENTINEL_MASTER_NAME: str | None = Field(
         description="Name of the Redis Sentinel master.",
         default=None,
     )
 
-    CELERY_SENTINEL_PASSWORD: Optional[str] = Field(
+    CELERY_SENTINEL_PASSWORD: str | None = Field(
         description="Password of the Redis Sentinel master.",
         default=None,
     )
-    CELERY_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    CELERY_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Timeout for Redis Sentinel socket operations in seconds.",
         default=0.1,
     )
@@ -268,12 +274,12 @@ class InternalTestConfig(BaseSettings):
     Configuration settings for Internal Test
     """
 
-    AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
+    AWS_SECRET_ACCESS_KEY: str | None = Field(
         description="Internal test AWS secret access key",
         default=None,
     )
 
-    AWS_ACCESS_KEY_ID: Optional[str] = Field(
+    AWS_ACCESS_KEY_ID: str | None = Field(
         description="Internal test AWS access key ID",
         default=None,
     )
@@ -284,15 +290,15 @@ class DatasetQueueMonitorConfig(BaseSettings):
     Configuration settings for Dataset Queue Monitor
     """
 
-    QUEUE_MONITOR_THRESHOLD: Optional[NonNegativeInt] = Field(
+    QUEUE_MONITOR_THRESHOLD: NonNegativeInt | None = Field(
         description="Threshold for dataset queue monitor",
         default=200,
     )
-    QUEUE_MONITOR_ALERT_EMAILS: Optional[str] = Field(
+    QUEUE_MONITOR_ALERT_EMAILS: str | None = Field(
         description="Emails for dataset queue monitor alert, separated by commas",
         default=None,
     )
-    QUEUE_MONITOR_INTERVAL: Optional[NonNegativeFloat] = Field(
+    QUEUE_MONITOR_INTERVAL: NonNegativeFloat | None = Field(
         description="Interval for dataset queue monitor in minutes",
         default=30,
     )

api/configs/middleware/cache/redis_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -19,12 +17,12 @@ class RedisConfig(BaseSettings):
         default=6379,
     )
 
-    REDIS_USERNAME: Optional[str] = Field(
+    REDIS_USERNAME: str | None = Field(
         description="Username for Redis authentication (if required)",
         default=None,
     )
 
-    REDIS_PASSWORD: Optional[str] = Field(
+    REDIS_PASSWORD: str | None = Field(
         description="Password for Redis authentication (if required)",
         default=None,
     )
@@ -44,47 +42,47 @@ class RedisConfig(BaseSettings):
         default="CERT_NONE",
     )
 
-    REDIS_SSL_CA_CERTS: Optional[str] = Field(
+    REDIS_SSL_CA_CERTS: str | None = Field(
         description="Path to the CA certificate file for SSL verification",
         default=None,
     )
 
-    REDIS_SSL_CERTFILE: Optional[str] = Field(
+    REDIS_SSL_CERTFILE: str | None = Field(
         description="Path to the client certificate file for SSL authentication",
         default=None,
     )
 
-    REDIS_SSL_KEYFILE: Optional[str] = Field(
+    REDIS_SSL_KEYFILE: str | None = Field(
         description="Path to the client private key file for SSL authentication",
         default=None,
     )
 
-    REDIS_USE_SENTINEL: Optional[bool] = Field(
+    REDIS_USE_SENTINEL: bool | None = Field(
         description="Enable Redis Sentinel mode for high availability",
         default=False,
     )
 
-    REDIS_SENTINELS: Optional[str] = Field(
+    REDIS_SENTINELS: str | None = Field(
         description="Comma-separated list of Redis Sentinel nodes (host:port)",
         default=None,
     )
 
-    REDIS_SENTINEL_SERVICE_NAME: Optional[str] = Field(
+    REDIS_SENTINEL_SERVICE_NAME: str | None = Field(
         description="Name of the Redis Sentinel service to monitor",
         default=None,
     )
 
-    REDIS_SENTINEL_USERNAME: Optional[str] = Field(
+    REDIS_SENTINEL_USERNAME: str | None = Field(
         description="Username for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_PASSWORD: Optional[str] = Field(
+    REDIS_SENTINEL_PASSWORD: str | None = Field(
         description="Password for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    REDIS_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )
@@ -94,12 +92,12 @@ class RedisConfig(BaseSettings):
         default=False,
     )
 
-    REDIS_CLUSTERS: Optional[str] = Field(
+    REDIS_CLUSTERS: str | None = Field(
         description="Comma-separated list of Redis Clusters nodes (host:port)",
         default=None,
     )
 
-    REDIS_CLUSTERS_PASSWORD: Optional[str] = Field(
+    REDIS_CLUSTERS_PASSWORD: str | None = Field(
         description="Password for Redis Clusters authentication (if required)",
         default=None,
     )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,37 +7,37 @@ class AliyunOSSStorageConfig(BaseSettings):
     Configuration settings for Aliyun Object Storage Service (OSS)
     """
 
-    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+    ALIYUN_OSS_BUCKET_NAME: str | None = Field(
         description="Name of the Aliyun OSS bucket to store and retrieve objects",
         default=None,
     )
 
-    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+    ALIYUN_OSS_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+    ALIYUN_OSS_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+    ALIYUN_OSS_ENDPOINT: str | None = Field(
         description="URL of the Aliyun OSS endpoint for your chosen region",
         default=None,
     )
 
-    ALIYUN_OSS_REGION: Optional[str] = Field(
+    ALIYUN_OSS_REGION: str | None = Field(
         description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
         default=None,
     )
 
-    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+    ALIYUN_OSS_AUTH_VERSION: str | None = Field(
         description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
         default=None,
     )
 
-    ALIYUN_OSS_PATH: Optional[str] = Field(
+    ALIYUN_OSS_PATH: str | None = Field(
         description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
         default=None,
     )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import Field
 from pydantic_settings import BaseSettings
@@ -9,27 +9,27 @@ class S3StorageConfig(BaseSettings):
     Configuration settings for S3-compatible object storage
     """
 
-    S3_ENDPOINT: Optional[str] = Field(
+    S3_ENDPOINT: str | None = Field(
         description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
         default=None,
     )
 
-    S3_REGION: Optional[str] = Field(
+    S3_REGION: str | None = Field(
         description="Region where the S3 bucket is located (e.g., 'us-east-1')",
         default=None,
     )
 
-    S3_BUCKET_NAME: Optional[str] = Field(
+    S3_BUCKET_NAME: str | None = Field(
         description="Name of the S3 bucket to store and retrieve objects",
         default=None,
     )
 
-    S3_ACCESS_KEY: Optional[str] = Field(
+    S3_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with the S3 service",
         default=None,
     )
 
-    S3_SECRET_KEY: Optional[str] = Field(
+    S3_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with the S3 service",
         default=None,
     )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class AzureBlobStorageConfig(BaseSettings):
     Configuration settings for Azure Blob Storage
     """
 
-    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_NAME: str | None = Field(
         description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_KEY: str | None = Field(
         description="Access key for authenticating with the Azure Storage account",
         default=None,
     )
 
-    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+    AZURE_BLOB_CONTAINER_NAME: str | None = Field(
         description="Name of the Azure Blob container to store and retrieve objects",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_URL: str | None = Field(
         description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
         default=None,
     )

api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class BaiduOBSStorageConfig(BaseSettings):
     Configuration settings for Baidu Object Storage Service (OBS)
     """
 
-    BAIDU_OBS_BUCKET_NAME: Optional[str] = Field(
+    BAIDU_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    BAIDU_OBS_ACCESS_KEY: Optional[str] = Field(
+    BAIDU_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_SECRET_KEY: Optional[str] = Field(
+    BAIDU_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_ENDPOINT: Optional[str] = Field(
+    BAIDU_OBS_ENDPOINT: str | None = Field(
         description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
         default=None,
     )

api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -1,7 +1,5 @@
 """ClickZetta Volume Storage Configuration"""
 
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ from pydantic_settings import BaseSettings
 class ClickZettaVolumeStorageConfig(BaseSettings):
     """Configuration for ClickZetta Volume storage."""
 
-    CLICKZETTA_VOLUME_USERNAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_USERNAME: str | None = Field(
         description="Username for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_VOLUME_PASSWORD: str | None = Field(
         description="Password for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_VOLUME_INSTANCE: str | None = Field(
         description="ClickZetta instance identifier",
         default=None,
     )
@@ -49,7 +47,7 @@ class ClickZettaVolumeStorageConfig(BaseSettings):
         default="user",
     )
 
-    CLICKZETTA_VOLUME_NAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_NAME: str | None = Field(
         description="ClickZetta volume name for external volumes",
         default=None,
     )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class GoogleCloudStorageConfig(BaseSettings):
     Configuration settings for Google Cloud Storage
     """
 
-    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+    GOOGLE_STORAGE_BUCKET_NAME: str | None = Field(
         description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
         default=None,
     )
 
-    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: str | None = Field(
         description="Base64-encoded JSON key file for Google Cloud service account authentication",
         default=None,
     )

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class HuaweiCloudOBSStorageConfig(BaseSettings):
     Configuration settings for Huawei Cloud Object Storage Service (OBS)
     """
 
-    HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
+    HUAWEI_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
+    HUAWEI_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
+    HUAWEI_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SERVER: Optional[str] = Field(
+    HUAWEI_OBS_SERVER: str | None = Field(
         description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
         default=None,
     )

api/configs/middleware/storage/oci_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OCIStorageConfig(BaseSettings):
     Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
     """
 
-    OCI_ENDPOINT: Optional[str] = Field(
+    OCI_ENDPOINT: str | None = Field(
         description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
         default=None,
     )
 
-    OCI_REGION: Optional[str] = Field(
+    OCI_REGION: str | None = Field(
         description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
         default=None,
     )
 
-    OCI_BUCKET_NAME: Optional[str] = Field(
+    OCI_BUCKET_NAME: str | None = Field(
         description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
         default=None,
     )
 
-    OCI_ACCESS_KEY: Optional[str] = Field(
+    OCI_ACCESS_KEY: str | None = Field(
         description="Access key (also known as API key) for authenticating with OCI Object Storage",
         default=None,
     )
 
-    OCI_SECRET_KEY: Optional[str] = Field(
+    OCI_SECRET_KEY: str | None = Field(
         description="Secret key associated with the access key for authenticating with OCI Object Storage",
         default=None,
     )

api/configs/middleware/storage/supabase_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class SupabaseStorageConfig(BaseSettings):
     Configuration settings for Supabase Object Storage Service
     """
 
-    SUPABASE_BUCKET_NAME: Optional[str] = Field(
+    SUPABASE_BUCKET_NAME: str | None = Field(
         description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
         default=None,
     )
 
-    SUPABASE_API_KEY: Optional[str] = Field(
+    SUPABASE_API_KEY: str | None = Field(
         description="API KEY for authenticating with Supabase",
         default=None,
     )
 
-    SUPABASE_URL: Optional[str] = Field(
+    SUPABASE_URL: str | None = Field(
         description="URL of the Supabase",
         default=None,
     )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TencentCloudCOSStorageConfig(BaseSettings):
     Configuration settings for Tencent Cloud Object Storage (COS)
     """
 
-    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+    TENCENT_COS_BUCKET_NAME: str | None = Field(
         description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
         default=None,
     )
 
-    TENCENT_COS_REGION: Optional[str] = Field(
+    TENCENT_COS_REGION: str | None = Field(
         description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
         default=None,
     )
 
-    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+    TENCENT_COS_SECRET_ID: str | None = Field(
         description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+    TENCENT_COS_SECRET_KEY: str | None = Field(
         description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SCHEME: Optional[str] = Field(
+    TENCENT_COS_SCHEME: str | None = Field(
         description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class VolcengineTOSStorageConfig(BaseSettings):
     Configuration settings for Volcengine Tinder Object Storage (TOS)
     """
 
-    VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
+    VOLCENGINE_TOS_BUCKET_NAME: str | None = Field(
         description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
         default=None,
     )
 
-    VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
+    VOLCENGINE_TOS_ENDPOINT: str | None = Field(
         description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
         default=None,
     )
 
-    VOLCENGINE_TOS_REGION: Optional[str] = Field(
+    VOLCENGINE_TOS_REGION: str | None = Field(
         description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
         default=None,
     )

api/configs/middleware/vdb/analyticdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -11,37 +9,37 @@ class AnalyticdbConfig(BaseSettings):
     https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
     """
 
-    ANALYTICDB_KEY_ID: Optional[str] = Field(
+    ANALYTICDB_KEY_ID: str | None = Field(
         default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
     )
-    ANALYTICDB_KEY_SECRET: Optional[str] = Field(
+    ANALYTICDB_KEY_SECRET: str | None = Field(
         default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
     )
-    ANALYTICDB_REGION_ID: Optional[str] = Field(
+    ANALYTICDB_REGION_ID: str | None = Field(
         default=None,
         description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
     )
-    ANALYTICDB_INSTANCE_ID: Optional[str] = Field(
+    ANALYTICDB_INSTANCE_ID: str | None = Field(
         default=None,
         description="The unique identifier of the AnalyticDB instance you want to connect to.",
     )
-    ANALYTICDB_ACCOUNT: Optional[str] = Field(
+    ANALYTICDB_ACCOUNT: str | None = Field(
         default=None,
         description="The account name used to log in to the AnalyticDB instance"
         " (usually the initial account created with the instance).",
     )
-    ANALYTICDB_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_PASSWORD: str | None = Field(
         default=None, description="The password associated with the AnalyticDB account for database authentication."
     )
-    ANALYTICDB_NAMESPACE: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE: str | None = Field(
         default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
     )
-    ANALYTICDB_NAMESPACE_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE_PASSWORD: str | None = Field(
         default=None,
         description="The password for accessing the specified namespace within the AnalyticDB instance"
         " (if namespace feature is enabled).",
     )
-    ANALYTICDB_HOST: Optional[str] = Field(
+    ANALYTICDB_HOST: str | None = Field(
         default=None, description="The host of the AnalyticDB instance you want to connect to."
     )
     ANALYTICDB_PORT: PositiveInt = Field(

api/configs/middleware/vdb/baidu_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class BaiduVectorDBConfig(BaseSettings):
     Configuration settings for Baidu Vector Database
     """
 
-    BAIDU_VECTOR_DB_ENDPOINT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ENDPOINT: str | None = Field(
         description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class BaiduVectorDBConfig(BaseSettings):
         default=30000,
     )
 
-    BAIDU_VECTOR_DB_ACCOUNT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ACCOUNT: str | None = Field(
         description="Account for authenticating with the Baidu Vector Database",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_API_KEY: Optional[str] = Field(
+    BAIDU_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Baidu Vector Database service",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_DATABASE: Optional[str] = Field(
+    BAIDU_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Baidu Vector Database to connect to",
         default=None,
     )
@@ -43,3 +41,13 @@ class BaiduVectorDBConfig(BaseSettings):
         description="Number of replicas for the Baidu Vector Database (default is 3)",
         default=3,
     )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER: str = Field(
+        description="Analyzer type for inverted index in Baidu Vector Database (default is DEFAULT_ANALYZER)",
+        default="DEFAULT_ANALYZER",
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE: str = Field(
+        description="Parser mode for inverted index in Baidu Vector Database (default is COARSE_MODE)",
+        default="COARSE_MODE",
+    )

api/configs/middleware/vdb/chroma_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class ChromaConfig(BaseSettings):
     Configuration settings for Chroma vector database
     """
 
-    CHROMA_HOST: Optional[str] = Field(
+    CHROMA_HOST: str | None = Field(
         description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
         default=None,
     )
@@ -19,22 +17,22 @@ class ChromaConfig(BaseSettings):
         default=8000,
     )
 
-    CHROMA_TENANT: Optional[str] = Field(
+    CHROMA_TENANT: str | None = Field(
         description="Tenant identifier for multi-tenancy support in Chroma",
         default=None,
     )
 
-    CHROMA_DATABASE: Optional[str] = Field(
+    CHROMA_DATABASE: str | None = Field(
         description="Name of the Chroma database to connect to",
         default=None,
     )
 
-    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+    CHROMA_AUTH_PROVIDER: str | None = Field(
         description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
         default=None,
     )
 
-    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+    CHROMA_AUTH_CREDENTIALS: str | None = Field(
         description="Authentication credentials for Chroma (format depends on the auth provider)",
         default=None,
     )

api/configs/middleware/vdb/clickzetta_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,62 +7,62 @@ class ClickzettaConfig(BaseSettings):
     Clickzetta Lakehouse vector database configuration
     """
 
-    CLICKZETTA_USERNAME: Optional[str] = Field(
+    CLICKZETTA_USERNAME: str | None = Field(
         description="Username for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_PASSWORD: str | None = Field(
         description="Password for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_INSTANCE: str | None = Field(
         description="Clickzetta Lakehouse instance ID",
         default=None,
     )
 
-    CLICKZETTA_SERVICE: Optional[str] = Field(
+    CLICKZETTA_SERVICE: str | None = Field(
         description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
         default="api.clickzetta.com",
     )
 
-    CLICKZETTA_WORKSPACE: Optional[str] = Field(
+    CLICKZETTA_WORKSPACE: str | None = Field(
         description="Clickzetta workspace name",
         default="default",
     )
 
-    CLICKZETTA_VCLUSTER: Optional[str] = Field(
+    CLICKZETTA_VCLUSTER: str | None = Field(
         description="Clickzetta virtual cluster name",
         default="default_ap",
     )
 
-    CLICKZETTA_SCHEMA: Optional[str] = Field(
+    CLICKZETTA_SCHEMA: str | None = Field(
         description="Database schema name in Clickzetta",
         default="public",
     )
 
-    CLICKZETTA_BATCH_SIZE: Optional[int] = Field(
+    CLICKZETTA_BATCH_SIZE: int | None = Field(
         description="Batch size for bulk insert operations",
         default=100,
     )
 
-    CLICKZETTA_ENABLE_INVERTED_INDEX: Optional[bool] = Field(
+    CLICKZETTA_ENABLE_INVERTED_INDEX: bool | None = Field(
         description="Enable inverted index for full-text search capabilities",
         default=True,
     )
 
-    CLICKZETTA_ANALYZER_TYPE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_TYPE: str | None = Field(
         description="Analyzer type for full-text search: keyword, english, chinese, unicode",
         default="chinese",
     )
 
-    CLICKZETTA_ANALYZER_MODE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_MODE: str | None = Field(
         description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
         default="smart",
     )
 
-    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: Optional[str] = Field(
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: str | None = Field(
         description="Distance function for vector similarity: l2_distance or cosine_distance",
         default="cosine_distance",
     )

api/configs/middleware/vdb/couchbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class CouchbaseConfig(BaseSettings):
     Couchbase configs
     """
 
-    COUCHBASE_CONNECTION_STRING: Optional[str] = Field(
+    COUCHBASE_CONNECTION_STRING: str | None = Field(
         description="COUCHBASE connection string",
         default=None,
     )
 
-    COUCHBASE_USER: Optional[str] = Field(
+    COUCHBASE_USER: str | None = Field(
         description="COUCHBASE user",
         default=None,
     )
 
-    COUCHBASE_PASSWORD: Optional[str] = Field(
+    COUCHBASE_PASSWORD: str | None = Field(
         description="COUCHBASE password",
         default=None,
     )
 
-    COUCHBASE_BUCKET_NAME: Optional[str] = Field(
+    COUCHBASE_BUCKET_NAME: str | None = Field(
         description="COUCHBASE bucket name",
         default=None,
     )
 
-    COUCHBASE_SCOPE_NAME: Optional[str] = Field(
+    COUCHBASE_SCOPE_NAME: str | None = Field(
         description="COUCHBASE scope name",
         default=None,
     )

api/configs/middleware/vdb/elasticsearch_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt, model_validator
 from pydantic_settings import BaseSettings
 
@@ -10,7 +8,7 @@ class ElasticsearchConfig(BaseSettings):
     Can load from environment variables or .env files.
     """
 
-    ELASTICSEARCH_HOST: Optional[str] = Field(
+    ELASTICSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
         default="127.0.0.1",
     )
@@ -20,30 +18,28 @@ class ElasticsearchConfig(BaseSettings):
         default=9200,
     )
 
-    ELASTICSEARCH_USERNAME: Optional[str] = Field(
+    ELASTICSEARCH_USERNAME: str | None = Field(
         description="Username for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
-    ELASTICSEARCH_PASSWORD: Optional[str] = Field(
+    ELASTICSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
     # Elastic Cloud (optional)
-    ELASTICSEARCH_USE_CLOUD: Optional[bool] = Field(
+    ELASTICSEARCH_USE_CLOUD: bool | None = Field(
         description="Set to True to use Elastic Cloud instead of self-hosted Elasticsearch", default=False
     )
-    ELASTICSEARCH_CLOUD_URL: Optional[str] = Field(
+    ELASTICSEARCH_CLOUD_URL: str | None = Field(
         description="Full URL for Elastic Cloud deployment (e.g., 'https://example.es.region.aws.found.io:443')",
         default=None,
     )
-    ELASTICSEARCH_API_KEY: Optional[str] = Field(
-        description="API key for authenticating with Elastic Cloud", default=None
-    )
+    ELASTICSEARCH_API_KEY: str | None = Field(description="API key for authenticating with Elastic Cloud", default=None)
 
     # Common options
-    ELASTICSEARCH_CA_CERTS: Optional[str] = Field(
+    ELASTICSEARCH_CA_CERTS: str | None = Field(
         description="Path to CA certificate file for SSL verification", default=None
     )
     ELASTICSEARCH_VERIFY_CERTS: bool = Field(

api/configs/middleware/vdb/huawei_cloud_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class HuaweiCloudConfig(BaseSettings):
     Configuration settings for Huawei cloud search service
     """
 
-    HUAWEI_CLOUD_HOSTS: Optional[str] = Field(
+    HUAWEI_CLOUD_HOSTS: str | None = Field(
         description="Hostname or IP address of the Huawei cloud search service instance",
         default=None,
     )
 
-    HUAWEI_CLOUD_USER: Optional[str] = Field(
+    HUAWEI_CLOUD_USER: str | None = Field(
         description="Username for authenticating with Huawei cloud search service",
         default=None,
     )
 
-    HUAWEI_CLOUD_PASSWORD: Optional[str] = Field(
+    HUAWEI_CLOUD_PASSWORD: str | None = Field(
         description="Password for authenticating with Huawei cloud search service",
         default=None,
     )

api/configs/middleware/vdb/lindorm_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class LindormConfig(BaseSettings):
     Lindorm configs
     """
 
-    LINDORM_URL: Optional[str] = Field(
+    LINDORM_URL: str | None = Field(
         description="Lindorm url",
         default=None,
     )
-    LINDORM_USERNAME: Optional[str] = Field(
+    LINDORM_USERNAME: str | None = Field(
         description="Lindorm user",
         default=None,
     )
-    LINDORM_PASSWORD: Optional[str] = Field(
+    LINDORM_PASSWORD: str | None = Field(
         description="Lindorm password",
         default=None,
     )
-    DEFAULT_INDEX_TYPE: Optional[str] = Field(
+    LINDORM_INDEX_TYPE: str | None = Field(
         description="Lindorm Vector Index Type, hnsw or flat is available in dify",
         default="hnsw",
     )
-    DEFAULT_DISTANCE_TYPE: Optional[str] = Field(
+    LINDORM_DISTANCE_TYPE: str | None = Field(
         description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
     )
-    USING_UGC_INDEX: Optional[bool] = Field(
-        description="Using UGC index will store the same type of Index in a single index but can retrieve separately.",
-        default=False,
+    LINDORM_USING_UGC: bool | None = Field(
+        description="Using UGC index will store indexes with the same IndexType/Dimension in a single big index.",
+        default=True,
     )
-    LINDORM_QUERY_TIMEOUT: Optional[float] = Field(description="The lindorm search request timeout (s)", default=2.0)
+    LINDORM_QUERY_TIMEOUT: float | None = Field(description="The lindorm search request timeout (s)", default=2.0)

api/configs/middleware/vdb/milvus_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class MilvusConfig(BaseSettings):
     Configuration settings for Milvus vector database
     """
 
-    MILVUS_URI: Optional[str] = Field(
+    MILVUS_URI: str | None = Field(
         description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
         default="http://127.0.0.1:19530",
     )
 
-    MILVUS_TOKEN: Optional[str] = Field(
+    MILVUS_TOKEN: str | None = Field(
         description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
 
-    MILVUS_USER: Optional[str] = Field(
+    MILVUS_USER: str | None = Field(
         description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
-    MILVUS_PASSWORD: Optional[str] = Field(
+    MILVUS_PASSWORD: str | None = Field(
         description="Password for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
@@ -40,7 +38,7 @@ class MilvusConfig(BaseSettings):
         default=True,
     )
 
-    MILVUS_ANALYZER_PARAMS: Optional[str] = Field(
+    MILVUS_ANALYZER_PARAMS: str | None = Field(
         description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
         default=None,
     )

api/configs/middleware/vdb/oceanbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OceanBaseVectorConfig(BaseSettings):
     Configuration settings for OceanBase Vector database
     """
 
-    OCEANBASE_VECTOR_HOST: Optional[str] = Field(
+    OCEANBASE_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PORT: Optional[PositiveInt] = Field(
+    OCEANBASE_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the OceanBase Vector server is listening (default is 2881)",
         default=2881,
     )
 
-    OCEANBASE_VECTOR_USER: Optional[str] = Field(
+    OCEANBASE_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PASSWORD: Optional[str] = Field(
+    OCEANBASE_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_DATABASE: Optional[str] = Field(
+    OCEANBASE_VECTOR_DATABASE: str | None = Field(
         description="Name of the OceanBase Vector database to connect to",
         default=None,
     )
@@ -39,3 +37,15 @@ class OceanBaseVectorConfig(BaseSettings):
         "with older versions",
         default=False,
     )
+
+    OCEANBASE_FULLTEXT_PARSER: str | None = Field(
+        description=(
+            "Fulltext parser to use for text indexing. "
+            "Built-in options: 'ngram' (N-gram tokenizer for English/numbers), "
+            "'beng' (Basic English tokenizer), 'space' (Space-based tokenizer), "
+            "'ngram2' (Improved N-gram tokenizer), 'ik' (Chinese tokenizer). "
+            "External plugins (require installation): 'japanese_ftparser' (Japanese tokenizer), "
+            "'thai_ftparser' (Thai tokenizer). Default is 'ik'"
+        ),
+        default="ik",
+    )

api/configs/middleware/vdb/opengauss_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class OpenGaussConfig(BaseSettings):
     Configuration settings for OpenGauss
     """
 
-    OPENGAUSS_HOST: Optional[str] = Field(
+    OPENGAUSS_HOST: str | None = Field(
         description="Hostname or IP address of the OpenGauss server(e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class OpenGaussConfig(BaseSettings):
         default=6600,
     )
 
-    OPENGAUSS_USER: Optional[str] = Field(
+    OPENGAUSS_USER: str | None = Field(
         description="Username for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_PASSWORD: Optional[str] = Field(
+    OPENGAUSS_PASSWORD: str | None = Field(
         description="Password for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_DATABASE: Optional[str] = Field(
+    OPENGAUSS_DATABASE: str | None = Field(
         description="Name of the OpenGauss database to connect to",
         default=None,
     )

api/configs/middleware/vdb/opensearch_config.py

@@ -1,5 +1,5 @@
-import enum
-from typing import Literal, Optional
+from enum import Enum
+from typing import Literal
 
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
@@ -10,7 +10,7 @@ class OpenSearchConfig(BaseSettings):
     Configuration settings for OpenSearch
     """
 
-    class AuthMethod(enum.StrEnum):
+    class AuthMethod(Enum):
         """
         Authentication method for OpenSearch
         """
@@ -18,7 +18,7 @@ class OpenSearchConfig(BaseSettings):
         BASIC = "basic"
         AWS_MANAGED_IAM = "aws_managed_iam"
 
-    OPENSEARCH_HOST: Optional[str] = Field(
+    OPENSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
     )
@@ -43,21 +43,21 @@ class OpenSearchConfig(BaseSettings):
         default=AuthMethod.BASIC,
     )
 
-    OPENSEARCH_USER: Optional[str] = Field(
+    OPENSEARCH_USER: str | None = Field(
         description="Username for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_PASSWORD: Optional[str] = Field(
+    OPENSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_AWS_REGION: Optional[str] = Field(
+    OPENSEARCH_AWS_REGION: str | None = Field(
         description="AWS region for OpenSearch (e.g. 'us-west-2')",
         default=None,
     )
 
-    OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+    OPENSEARCH_AWS_SERVICE: Literal["es", "aoss"] | None = Field(
         description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
     )

api/configs/middleware/vdb/oracle_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,33 +7,33 @@ class OracleConfig(BaseSettings):
     Configuration settings for Oracle database
     """
 
-    ORACLE_USER: Optional[str] = Field(
+    ORACLE_USER: str | None = Field(
         description="Username for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_PASSWORD: Optional[str] = Field(
+    ORACLE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_DSN: Optional[str] = Field(
+    ORACLE_DSN: str | None = Field(
         description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
         "For autonomous database, use the service name from tnsnames.ora in the wallet",
         default=None,
     )
 
-    ORACLE_CONFIG_DIR: Optional[str] = Field(
+    ORACLE_CONFIG_DIR: str | None = Field(
         description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
         default=None,
     )
 
-    ORACLE_WALLET_LOCATION: Optional[str] = Field(
+    ORACLE_WALLET_LOCATION: str | None = Field(
         description="Oracle wallet directory path containing the wallet files for secure connection",
         default=None,
     )
 
-    ORACLE_WALLET_PASSWORD: Optional[str] = Field(
+    ORACLE_WALLET_PASSWORD: str | None = Field(
         description="Password to decrypt the Oracle wallet, if it is encrypted",
         default=None,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectorConfig(BaseSettings):
     Configuration settings for PGVector (PostgreSQL with vector extension)
     """
 
-    PGVECTOR_HOST: Optional[str] = Field(
+    PGVECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectorConfig(BaseSettings):
         default=5433,
     )
 
-    PGVECTOR_USER: Optional[str] = Field(
+    PGVECTOR_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_PASSWORD: Optional[str] = Field(
+    PGVECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_DATABASE: Optional[str] = Field(
+    PGVECTOR_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database to connect to",
         default=None,
     )

api/configs/middleware/vdb/pgvectors_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectoRSConfig(BaseSettings):
     Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
     """
 
-    PGVECTO_RS_HOST: Optional[str] = Field(
+    PGVECTO_RS_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectoRSConfig(BaseSettings):
         default=5431,
     )
 
-    PGVECTO_RS_USER: Optional[str] = Field(
+    PGVECTO_RS_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+    PGVECTO_RS_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_DATABASE: Optional[str] = Field(
+    PGVECTO_RS_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
         default=None,
     )

api/configs/middleware/vdb/qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class QdrantConfig(BaseSettings):
     Configuration settings for Qdrant vector database
     """
 
-    QDRANT_URL: Optional[str] = Field(
+    QDRANT_URL: str | None = Field(
         description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
         default=None,
     )
 
-    QDRANT_API_KEY: Optional[str] = Field(
+    QDRANT_API_KEY: str | None = Field(
         description="API key for authenticating with the Qdrant server",
         default=None,
     )

api/configs/middleware/vdb/relyt_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class RelytConfig(BaseSettings):
     Configuration settings for Relyt database
     """
 
-    RELYT_HOST: Optional[str] = Field(
+    RELYT_HOST: str | None = Field(
         description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class RelytConfig(BaseSettings):
         default=9200,
     )
 
-    RELYT_USER: Optional[str] = Field(
+    RELYT_USER: str | None = Field(
         description="Username for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_PASSWORD: Optional[str] = Field(
+    RELYT_PASSWORD: str | None = Field(
         description="Password for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_DATABASE: Optional[str] = Field(
+    RELYT_DATABASE: str | None = Field(
         description="Name of the Relyt database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/tablestore_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class TableStoreConfig(BaseSettings):
     Configuration settings for TableStore.
     """
 
-    TABLESTORE_ENDPOINT: Optional[str] = Field(
+    TABLESTORE_ENDPOINT: str | None = Field(
         description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')",
         default=None,
     )
 
-    TABLESTORE_INSTANCE_NAME: Optional[str] = Field(
+    TABLESTORE_INSTANCE_NAME: str | None = Field(
         description="Instance name to access TableStore server (eg. 'instance-name')",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_ID: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_ID: str | None = Field(
         description="AccessKey id for the instance name",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_SECRET: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_SECRET: str | None = Field(
         description="AccessKey secret for the instance name",
         default=None,
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TencentVectorDBConfig(BaseSettings):
     Configuration settings for Tencent Vector Database
     """
 
-    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+    TENCENT_VECTOR_DB_URL: str | None = Field(
         description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+    TENCENT_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Tencent Vector Database service",
         default=None,
     )
@@ -24,12 +22,12 @@ class TencentVectorDBConfig(BaseSettings):
         default=30,
     )
 
-    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+    TENCENT_VECTOR_DB_USERNAME: str | None = Field(
         description="Username for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+    TENCENT_VECTOR_DB_PASSWORD: str | None = Field(
         description="Password for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
@@ -44,7 +42,7 @@ class TencentVectorDBConfig(BaseSettings):
         default=2,
     )
 
-    TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
+    TENCENT_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Tencent Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TidbOnQdrantConfig(BaseSettings):
     Tidb on Qdrant configs
     """
 
-    TIDB_ON_QDRANT_URL: Optional[str] = Field(
+    TIDB_ON_QDRANT_URL: str | None = Field(
         description="Tidb on Qdrant url",
         default=None,
     )
 
-    TIDB_ON_QDRANT_API_KEY: Optional[str] = Field(
+    TIDB_ON_QDRANT_API_KEY: str | None = Field(
         description="Tidb on Qdrant api key",
         default=None,
     )
@@ -34,37 +32,37 @@ class TidbOnQdrantConfig(BaseSettings):
         default=6334,
     )
 
-    TIDB_PUBLIC_KEY: Optional[str] = Field(
+    TIDB_PUBLIC_KEY: str | None = Field(
         description="Tidb account public key",
         default=None,
     )
 
-    TIDB_PRIVATE_KEY: Optional[str] = Field(
+    TIDB_PRIVATE_KEY: str | None = Field(
         description="Tidb account private key",
         default=None,
     )
 
-    TIDB_API_URL: Optional[str] = Field(
+    TIDB_API_URL: str | None = Field(
         description="Tidb API url",
         default=None,
     )
 
-    TIDB_IAM_API_URL: Optional[str] = Field(
+    TIDB_IAM_API_URL: str | None = Field(
         description="Tidb IAM API url",
         default=None,
     )
 
-    TIDB_REGION: Optional[str] = Field(
+    TIDB_REGION: str | None = Field(
         description="Tidb serverless region",
         default="regions/aws-us-east-1",
     )
 
-    TIDB_PROJECT_ID: Optional[str] = Field(
+    TIDB_PROJECT_ID: str | None = Field(
         description="Tidb project id",
         default=None,
     )
 
-    TIDB_SPEND_LIMIT: Optional[int] = Field(
+    TIDB_SPEND_LIMIT: int | None = Field(
         description="Tidb spend limit",
         default=100,
     )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TiDBVectorConfig(BaseSettings):
     Configuration settings for TiDB Vector database
     """
 
-    TIDB_VECTOR_HOST: Optional[str] = Field(
+    TIDB_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
         default=None,
     )
 
-    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+    TIDB_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the TiDB Vector server is listening (default is 4000)",
         default=4000,
     )
 
-    TIDB_VECTOR_USER: Optional[str] = Field(
+    TIDB_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+    TIDB_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+    TIDB_VECTOR_DATABASE: str | None = Field(
         description="Name of the TiDB Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/upstash_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class UpstashConfig(BaseSettings):
     Configuration settings for Upstash vector database
     """
 
-    UPSTASH_VECTOR_URL: Optional[str] = Field(
+    UPSTASH_VECTOR_URL: str | None = Field(
         description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
         default=None,
     )
 
-    UPSTASH_VECTOR_TOKEN: Optional[str] = Field(
+    UPSTASH_VECTOR_TOKEN: str | None = Field(
         description="Token for authenticating with the upstash server",
         default=None,
     )

api/configs/middleware/vdb/vastbase_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class VastbaseVectorConfig(BaseSettings):
     Configuration settings for Vector (Vastbase with vector extension)
     """
 
-    VASTBASE_HOST: Optional[str] = Field(
+    VASTBASE_HOST: str | None = Field(
         description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class VastbaseVectorConfig(BaseSettings):
         default=5432,
     )
 
-    VASTBASE_USER: Optional[str] = Field(
+    VASTBASE_USER: str | None = Field(
         description="Username for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_PASSWORD: Optional[str] = Field(
+    VASTBASE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_DATABASE: Optional[str] = Field(
+    VASTBASE_DATABASE: str | None = Field(
         description="Name of the Vastbase database to connect to",
         default=None,
     )

api/configs/middleware/vdb/vikingdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -11,14 +9,14 @@ class VikingDBConfig(BaseSettings):
     https://www.volcengine.com/docs/6291/65568
     """
 
-    VIKINGDB_ACCESS_KEY: Optional[str] = Field(
+    VIKINGDB_ACCESS_KEY: str | None = Field(
         description="The Access Key provided by Volcengine VikingDB for API authentication."
         "Refer to the following documentation for details on obtaining credentials:"
         "https://www.volcengine.com/docs/6291/65568",
         default=None,
     )
 
-    VIKINGDB_SECRET_KEY: Optional[str] = Field(
+    VIKINGDB_SECRET_KEY: str | None = Field(
         description="The Secret Key provided by Volcengine VikingDB for API authentication.",
         default=None,
     )

api/configs/middleware/vdb/weaviate_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class WeaviateConfig(BaseSettings):
     Configuration settings for Weaviate vector database
     """
 
-    WEAVIATE_ENDPOINT: Optional[str] = Field(
+    WEAVIATE_ENDPOINT: str | None = Field(
         description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
         default=None,
     )
 
-    WEAVIATE_API_KEY: Optional[str] = Field(
+    WEAVIATE_API_KEY: str | None = Field(
         description="API key for authenticating with the Weaviate server",
         default=None,
     )

api/configs/remote_settings_sources/apollo/__init__.py

@@ -1,5 +1,5 @@
 from collections.abc import Mapping
-from typing import Any, Optional
+from typing import Any
 
 from pydantic import Field
 from pydantic.fields import FieldInfo
@@ -15,22 +15,22 @@ class ApolloSettingsSourceInfo(BaseSettings):
     Packaging build information
     """
 
-    APOLLO_APP_ID: Optional[str] = Field(
+    APOLLO_APP_ID: str | None = Field(
         description="apollo app_id",
         default=None,
     )
 
-    APOLLO_CLUSTER: Optional[str] = Field(
+    APOLLO_CLUSTER: str | None = Field(
         description="apollo cluster",
         default=None,
     )
 
-    APOLLO_CONFIG_URL: Optional[str] = Field(
+    APOLLO_CONFIG_URL: str | None = Field(
         description="apollo config url",
         default=None,
     )
 
-    APOLLO_NAMESPACE: Optional[str] = Field(
+    APOLLO_NAMESPACE: str | None = Field(
         description="apollo namespace",
         default=None,
     )

api/configs/remote_settings_sources/apollo/utils.py

@@ -29,7 +29,7 @@ def no_key_cache_key(namespace: str, key: str) -> str:
 
 
 # Returns whether the obtained value is obtained, and None if it does not
-def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any | None:
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any:
     if namespace_cache:
         kv_data = namespace_cache.get(CONFIGURATIONS)
         if kv_data is None:

api/configs/remote_settings_sources/nacos/http_request.py

@@ -5,7 +5,7 @@ import logging
 import os
 import time
 
-import requests
+import httpx
 
 logger = logging.getLogger(__name__)
 
@@ -30,10 +30,10 @@ class NacosHttpClient:
             params = {}
         try:
             self._inject_auth_info(headers, params)
-            response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
+            response = httpx.request(method, url="http://" + self.server + url, headers=headers, params=params)
             response.raise_for_status()
             return response.text
-        except requests.RequestException as e:
+        except httpx.RequestError as e:
             return f"Request to Nacos failed: {e}"
 
     def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
@@ -78,7 +78,7 @@ class NacosHttpClient:
         params = {"username": self.username, "password": self.password}
         url = "http://" + self.server + "/nacos/v1/auth/login"
         try:
-            resp = requests.request("POST", url, headers=None, params=params)
+            resp = httpx.request("POST", url, headers=None, params=params)
             resp.raise_for_status()
             response_data = resp.json()
             self.token = response_data.get("accessToken")

api/constants/model_template.py

@@ -7,7 +7,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # workflow default mode
     AppMode.WORKFLOW: {
         "app": {
-            "mode": AppMode.WORKFLOW.value,
+            "mode": AppMode.WORKFLOW,
             "enable_site": True,
             "enable_api": True,
         }
@@ -15,7 +15,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # completion default mode
     AppMode.COMPLETION: {
         "app": {
-            "mode": AppMode.COMPLETION.value,
+            "mode": AppMode.COMPLETION,
             "enable_site": True,
             "enable_api": True,
         },
@@ -44,7 +44,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # chat default mode
     AppMode.CHAT: {
         "app": {
-            "mode": AppMode.CHAT.value,
+            "mode": AppMode.CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -60,7 +60,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # advanced-chat default mode
     AppMode.ADVANCED_CHAT: {
         "app": {
-            "mode": AppMode.ADVANCED_CHAT.value,
+            "mode": AppMode.ADVANCED_CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -68,7 +68,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # agent-chat default mode
     AppMode.AGENT_CHAT: {
         "app": {
-            "mode": AppMode.AGENT_CHAT.value,
+            "mode": AppMode.AGENT_CHAT,
             "enable_site": True,
             "enable_api": True,
         },

api/contexts/__init__.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from contexts.wrapper import RecyclableContextVar
 
 if TYPE_CHECKING:
+    from core.datasource.__base.datasource_provider import DatasourcePluginProviderController
     from core.model_runtime.entities.model_entities import AIModelEntity
     from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
     from core.tools.plugin_tool.provider import PluginToolProviderController
@@ -32,3 +33,11 @@ plugin_model_schema_lock: RecyclableContextVar[Lock] = RecyclableContextVar(Cont
 plugin_model_schemas: RecyclableContextVar[dict[str, "AIModelEntity"]] = RecyclableContextVar(
     ContextVar("plugin_model_schemas")
 )
+
+datasource_plugin_providers: RecyclableContextVar[dict[str, "DatasourcePluginProviderController"]] = (
+    RecyclableContextVar(ContextVar("datasource_plugin_providers"))
+)
+
+datasource_plugin_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("datasource_plugin_providers_lock")
+)

api/controllers/console/__init__.py

@@ -54,89 +54,100 @@ api.add_resource(AppImportCheckDependenciesApi, "/apps/imports/<string:app_id>/c
 
 # Import other controllers
 from . import (
-    admin,  # pyright: ignore[reportUnusedImport]
-    apikey,  # pyright: ignore[reportUnusedImport]
-    extension,  # pyright: ignore[reportUnusedImport]
-    feature,  # pyright: ignore[reportUnusedImport]
-    init_validate,  # pyright: ignore[reportUnusedImport]
-    ping,  # pyright: ignore[reportUnusedImport]
-    setup,  # pyright: ignore[reportUnusedImport]
-    version,  # pyright: ignore[reportUnusedImport]
+    admin,
+    apikey,
+    extension,
+    feature,
+    init_validate,
+    ping,
+    setup,
+    spec,
+    version,
 )
 
 # Import app controllers
 from .app import (
-    advanced_prompt_template,  # pyright: ignore[reportUnusedImport]
-    agent,  # pyright: ignore[reportUnusedImport]
-    annotation,  # pyright: ignore[reportUnusedImport]
-    app,  # pyright: ignore[reportUnusedImport]
-    audio,  # pyright: ignore[reportUnusedImport]
-    completion,  # pyright: ignore[reportUnusedImport]
-    conversation,  # pyright: ignore[reportUnusedImport]
-    conversation_variables,  # pyright: ignore[reportUnusedImport]
-    generator,  # pyright: ignore[reportUnusedImport]
-    mcp_server,  # pyright: ignore[reportUnusedImport]
-    message,  # pyright: ignore[reportUnusedImport]
-    model_config,  # pyright: ignore[reportUnusedImport]
-    ops_trace,  # pyright: ignore[reportUnusedImport]
-    site,  # pyright: ignore[reportUnusedImport]
-    statistic,  # pyright: ignore[reportUnusedImport]
-    workflow,  # pyright: ignore[reportUnusedImport]
-    workflow_app_log,  # pyright: ignore[reportUnusedImport]
-    workflow_draft_variable,  # pyright: ignore[reportUnusedImport]
-    workflow_run,  # pyright: ignore[reportUnusedImport]
-    workflow_statistic,  # pyright: ignore[reportUnusedImport]
+    advanced_prompt_template,
+    agent,
+    annotation,
+    app,
+    audio,
+    completion,
+    conversation,
+    conversation_variables,
+    generator,
+    mcp_server,
+    message,
+    model_config,
+    ops_trace,
+    site,
+    statistic,
+    workflow,
+    workflow_app_log,
+    workflow_draft_variable,
+    workflow_run,
+    workflow_statistic,
 )
 
 # Import auth controllers
 from .auth import (
-    activate,  # pyright: ignore[reportUnusedImport]
-    data_source_bearer_auth,  # pyright: ignore[reportUnusedImport]
-    data_source_oauth,  # pyright: ignore[reportUnusedImport]
-    forgot_password,  # pyright: ignore[reportUnusedImport]
-    login,  # pyright: ignore[reportUnusedImport]
-    oauth,  # pyright: ignore[reportUnusedImport]
-    oauth_server,  # pyright: ignore[reportUnusedImport]
+    activate,
+    data_source_bearer_auth,
+    data_source_oauth,
+    email_register,
+    forgot_password,
+    login,
+    oauth,
+    oauth_server,
 )
 
 # Import billing controllers
-from .billing import billing, compliance  # pyright: ignore[reportUnusedImport]
+from .billing import billing, compliance
 
 # Import datasets controllers
 from .datasets import (
-    data_source,  # pyright: ignore[reportUnusedImport]
-    datasets,  # pyright: ignore[reportUnusedImport]
-    datasets_document,  # pyright: ignore[reportUnusedImport]
-    datasets_segments,  # pyright: ignore[reportUnusedImport]
-    external,  # pyright: ignore[reportUnusedImport]
-    hit_testing,  # pyright: ignore[reportUnusedImport]
-    metadata,  # pyright: ignore[reportUnusedImport]
-    website,  # pyright: ignore[reportUnusedImport]
+    data_source,
+    datasets,
+    datasets_document,
+    datasets_segments,
+    external,
+    hit_testing,
+    metadata,
+    website,
+)
+from .datasets.rag_pipeline import (
+    datasource_auth,
+    datasource_content_preview,
+    rag_pipeline,
+    rag_pipeline_datasets,
+    rag_pipeline_draft_variable,
+    rag_pipeline_import,
+    rag_pipeline_workflow,
 )
 
 # Import explore controllers
 from .explore import (
-    installed_app,  # pyright: ignore[reportUnusedImport]
-    parameter,  # pyright: ignore[reportUnusedImport]
-    recommended_app,  # pyright: ignore[reportUnusedImport]
-    saved_message,  # pyright: ignore[reportUnusedImport]
+    installed_app,
+    parameter,
+    recommended_app,
+    saved_message,
 )
 
 # Import tag controllers
-from .tag import tags  # pyright: ignore[reportUnusedImport]
+from .tag import tags
 
 # Import workspace controllers
 from .workspace import (
-    account,  # pyright: ignore[reportUnusedImport]
-    agent_providers,  # pyright: ignore[reportUnusedImport]
-    endpoint,  # pyright: ignore[reportUnusedImport]
-    load_balancing_config,  # pyright: ignore[reportUnusedImport]
-    members,  # pyright: ignore[reportUnusedImport]
-    model_providers,  # pyright: ignore[reportUnusedImport]
-    models,  # pyright: ignore[reportUnusedImport]
-    plugin,  # pyright: ignore[reportUnusedImport]
-    tool_providers,  # pyright: ignore[reportUnusedImport]
-    workspace,  # pyright: ignore[reportUnusedImport]
+    account,
+    agent_providers,
+    endpoint,
+    load_balancing_config,
+    members,
+    model_providers,
+    models,
+    plugin,
+    tool_providers,
+    workspace,
 )
 
 # Explore Audio
@@ -211,3 +222,78 @@ api.add_resource(
 )
 
 api.add_namespace(console_ns)
+
+__all__ = [
+    "account",
+    "activate",
+    "admin",
+    "advanced_prompt_template",
+    "agent",
+    "agent_providers",
+    "annotation",
+    "api",
+    "apikey",
+    "app",
+    "audio",
+    "billing",
+    "bp",
+    "completion",
+    "compliance",
+    "console_ns",
+    "conversation",
+    "conversation_variables",
+    "data_source",
+    "data_source_bearer_auth",
+    "data_source_oauth",
+    "datasets",
+    "datasets_document",
+    "datasets_segments",
+    "datasource_auth",
+    "datasource_content_preview",
+    "email_register",
+    "endpoint",
+    "extension",
+    "external",
+    "feature",
+    "forgot_password",
+    "generator",
+    "hit_testing",
+    "init_validate",
+    "installed_app",
+    "load_balancing_config",
+    "login",
+    "mcp_server",
+    "members",
+    "message",
+    "metadata",
+    "model_config",
+    "model_providers",
+    "models",
+    "oauth",
+    "oauth_server",
+    "ops_trace",
+    "parameter",
+    "ping",
+    "plugin",
+    "rag_pipeline",
+    "rag_pipeline_datasets",
+    "rag_pipeline_draft_variable",
+    "rag_pipeline_import",
+    "rag_pipeline_workflow",
+    "recommended_app",
+    "saved_message",
+    "setup",
+    "site",
+    "spec",
+    "statistic",
+    "tags",
+    "tool_providers",
+    "version",
+    "website",
+    "workflow",
+    "workflow_app_log",
+    "workflow_draft_variable",
+    "workflow_run",
+    "workflow_statistic",
+    "workspace",
+]

api/controllers/console/apikey.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 import flask_restx
 from flask_login import current_user
 from flask_restx import Resource, fields, marshal_with
@@ -50,7 +48,7 @@ class BaseApiKeyListResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
     resource_type: str | None = None
-    resource_model: Optional[type] = None
+    resource_model: type | None = None
     resource_id_field: str | None = None
     token_prefix: str | None = None
     max_keys = 10
@@ -103,7 +101,7 @@ class BaseApiKeyResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
     resource_type: str | None = None
-    resource_model: Optional[type] = None
+    resource_model: type | None = None
     resource_id_field: str | None = None
 
     def delete(self, resource_id, api_key_id):

api/controllers/console/app/advanced_prompt_template.py

@@ -1,12 +1,26 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 
 
+@console_ns.route("/app/prompt-templates")
 class AdvancedPromptTemplateList(Resource):
+    @api.doc("get_advanced_prompt_templates")
+    @api.doc(description="Get advanced prompt templates based on app mode and model configuration")
+    @api.expect(
+        api.parser()
+        .add_argument("app_mode", type=str, required=True, location="args", help="Application mode")
+        .add_argument("model_mode", type=str, required=True, location="args", help="Model mode")
+        .add_argument("has_context", type=str, default="true", location="args", help="Whether has context")
+        .add_argument("model_name", type=str, required=True, location="args", help="Model name")
+    )
+    @api.response(
+        200, "Prompt templates retrieved successfully", fields.List(fields.Raw(description="Prompt template data"))
+    )
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -19,6 +33,3 @@ class AdvancedPromptTemplateList(Resource):
         args = parser.parse_args()
 
         return AdvancedPromptTemplateService.get_prompt(args)
-
-
-api.add_resource(AdvancedPromptTemplateList, "/app/prompt-templates")

api/controllers/console/app/agent.py

@@ -1,6 +1,6 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.helper import uuid_value
@@ -9,7 +9,18 @@ from models.model import AppMode
 from services.agent_service import AgentService
 
 
+@console_ns.route("/apps/<uuid:app_id>/agent/logs")
 class AgentLogApi(Resource):
+    @api.doc("get_agent_logs")
+    @api.doc(description="Get agent execution logs for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("message_id", type=str, required=True, location="args", help="Message UUID")
+        .add_argument("conversation_id", type=str, required=True, location="args", help="Conversation UUID")
+    )
+    @api.response(200, "Agent logs retrieved successfully", fields.List(fields.Raw(description="Agent log entries")))
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -23,6 +34,3 @@ class AgentLogApi(Resource):
         args = parser.parse_args()
 
         return AgentService.get_agent_logs(app_model, args["conversation_id"], args["message_id"])
-
-
-api.add_resource(AgentLogApi, "/apps/<uuid:app_id>/agent/logs")

api/controllers/console/app/annotation.py

@@ -2,11 +2,11 @@ from typing import Literal
 
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.common.errors import NoFileUploadedError, TooManyFilesError
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_resource_check,
@@ -21,7 +21,23 @@ from libs.login import login_required
 from services.annotation_service import AppAnnotationService
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>")
 class AnnotationReplyActionApi(Resource):
+    @api.doc("annotation_reply_action")
+    @api.doc(description="Enable or disable annotation reply for an app")
+    @api.doc(params={"app_id": "Application ID", "action": "Action to perform (enable/disable)"})
+    @api.expect(
+        api.model(
+            "AnnotationReplyActionRequest",
+            {
+                "score_threshold": fields.Float(required=True, description="Score threshold for annotation matching"),
+                "embedding_provider_name": fields.String(required=True, description="Embedding provider name"),
+                "embedding_model_name": fields.String(required=True, description="Embedding model name"),
+            },
+        )
+    )
+    @api.response(200, "Action completed successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -43,7 +59,13 @@ class AnnotationReplyActionApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-setting")
 class AppAnnotationSettingDetailApi(Resource):
+    @api.doc("get_annotation_setting")
+    @api.doc(description="Get annotation settings for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Annotation settings retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -56,7 +78,23 @@ class AppAnnotationSettingDetailApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-settings/<uuid:annotation_setting_id>")
 class AppAnnotationSettingUpdateApi(Resource):
+    @api.doc("update_annotation_setting")
+    @api.doc(description="Update annotation settings for an app")
+    @api.doc(params={"app_id": "Application ID", "annotation_setting_id": "Annotation setting ID"})
+    @api.expect(
+        api.model(
+            "AnnotationSettingUpdateRequest",
+            {
+                "score_threshold": fields.Float(required=True, description="Score threshold"),
+                "embedding_provider_name": fields.String(required=True, description="Embedding provider"),
+                "embedding_model_name": fields.String(required=True, description="Embedding model"),
+            },
+        )
+    )
+    @api.response(200, "Settings updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -75,7 +113,13 @@ class AppAnnotationSettingUpdateApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>/status/<uuid:job_id>")
 class AnnotationReplyActionStatusApi(Resource):
+    @api.doc("get_annotation_reply_action_status")
+    @api.doc(description="Get status of annotation reply action job")
+    @api.doc(params={"app_id": "Application ID", "job_id": "Job ID", "action": "Action type"})
+    @api.response(200, "Job status retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -99,7 +143,19 @@ class AnnotationReplyActionStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations")
 class AnnotationApi(Resource):
+    @api.doc("list_annotations")
+    @api.doc(description="Get annotations for an app with pagination")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size")
+        .add_argument("keyword", type=str, location="args", default="", help="Search keyword")
+    )
+    @api.response(200, "Annotations retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -122,6 +178,21 @@ class AnnotationApi(Resource):
         }
         return response, 200
 
+    @api.doc("create_annotation")
+    @api.doc(description="Create a new annotation for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "CreateAnnotationRequest",
+            {
+                "question": fields.String(required=True, description="Question text"),
+                "answer": fields.String(required=True, description="Answer text"),
+                "annotation_reply": fields.Raw(description="Annotation reply data"),
+            },
+        )
+    )
+    @api.response(201, "Annotation created successfully", annotation_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -168,7 +239,13 @@ class AnnotationApi(Resource):
             return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/export")
 class AnnotationExportApi(Resource):
+    @api.doc("export_annotations")
+    @api.doc(description="Export all annotations for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Annotations exported successfully", fields.List(fields.Nested(annotation_fields)))
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -182,7 +259,14 @@ class AnnotationExportApi(Resource):
         return response, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>")
 class AnnotationUpdateDeleteApi(Resource):
+    @api.doc("update_delete_annotation")
+    @api.doc(description="Update or delete an annotation")
+    @api.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
+    @api.response(200, "Annotation updated successfully", annotation_fields)
+    @api.response(204, "Annotation deleted successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -214,7 +298,14 @@ class AnnotationUpdateDeleteApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/batch-import")
 class AnnotationBatchImportApi(Resource):
+    @api.doc("batch_import_annotations")
+    @api.doc(description="Batch import annotations from CSV file")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Batch import started successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "No file uploaded or too many files")
     @setup_required
     @login_required
     @account_initialization_required
@@ -239,7 +330,13 @@ class AnnotationBatchImportApi(Resource):
         return AppAnnotationService.batch_import_app_annotations(app_id, file)
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/batch-import-status/<uuid:job_id>")
 class AnnotationBatchImportStatusApi(Resource):
+    @api.doc("get_batch_import_status")
+    @api.doc(description="Get status of batch import job")
+    @api.doc(params={"app_id": "Application ID", "job_id": "Job ID"})
+    @api.response(200, "Job status retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -262,7 +359,20 @@ class AnnotationBatchImportStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>/hit-histories")
 class AnnotationHitHistoryListApi(Resource):
+    @api.doc("list_annotation_hit_histories")
+    @api.doc(description="Get hit histories for an annotation")
+    @api.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size")
+    )
+    @api.response(
+        200, "Hit histories retrieved successfully", fields.List(fields.Nested(annotation_hit_history_fields))
+    )
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -285,17 +395,3 @@ class AnnotationHitHistoryListApi(Resource):
             "page": page,
         }
         return response
-
-
-api.add_resource(AnnotationReplyActionApi, "/apps/<uuid:app_id>/annotation-reply/<string:action>")
-api.add_resource(
-    AnnotationReplyActionStatusApi, "/apps/<uuid:app_id>/annotation-reply/<string:action>/status/<uuid:job_id>"
-)
-api.add_resource(AnnotationApi, "/apps/<uuid:app_id>/annotations")
-api.add_resource(AnnotationExportApi, "/apps/<uuid:app_id>/annotations/export")
-api.add_resource(AnnotationUpdateDeleteApi, "/apps/<uuid:app_id>/annotations/<uuid:annotation_id>")
-api.add_resource(AnnotationBatchImportApi, "/apps/<uuid:app_id>/annotations/batch-import")
-api.add_resource(AnnotationBatchImportStatusApi, "/apps/<uuid:app_id>/annotations/batch-import-status/<uuid:job_id>")
-api.add_resource(AnnotationHitHistoryListApi, "/apps/<uuid:app_id>/annotations/<uuid:annotation_id>/hit-histories")
-api.add_resource(AppAnnotationSettingDetailApi, "/apps/<uuid:app_id>/annotation-setting")
-api.add_resource(AppAnnotationSettingUpdateApi, "/apps/<uuid:app_id>/annotation-settings/<uuid:annotation_setting_id>")

api/controllers/console/app/app.py

@@ -2,12 +2,12 @@ import uuid
 from typing import cast
 
 from flask_login import current_user
-from flask_restx import Resource, inputs, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, abort
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     account_initialization_required,
@@ -34,7 +34,27 @@ def _validate_description_length(description):
     return description
 
 
+@console_ns.route("/apps")
 class AppListApi(Resource):
+    @api.doc("list_apps")
+    @api.doc(description="Get list of applications with pagination and filtering")
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", help="Page number (1-99999)", default=1)
+        .add_argument("limit", type=int, location="args", help="Page size (1-100)", default=20)
+        .add_argument(
+            "mode",
+            type=str,
+            location="args",
+            choices=["completion", "chat", "advanced-chat", "workflow", "agent-chat", "channel", "all"],
+            default="all",
+            help="App mode filter",
+        )
+        .add_argument("name", type=str, location="args", help="Filter by app name")
+        .add_argument("tag_ids", type=str, location="args", help="Comma-separated tag IDs")
+        .add_argument("is_created_by_me", type=bool, location="args", help="Filter by creator")
+    )
+    @api.response(200, "Success", app_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -91,6 +111,24 @@ class AppListApi(Resource):
 
         return marshal(app_pagination, app_pagination_fields), 200
 
+    @api.doc("create_app")
+    @api.doc(description="Create a new application")
+    @api.expect(
+        api.model(
+            "CreateAppRequest",
+            {
+                "name": fields.String(required=True, description="App name"),
+                "description": fields.String(description="App description (max 400 chars)"),
+                "mode": fields.String(required=True, enum=ALLOW_CREATE_APP_MODES, description="App mode"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(201, "App created successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -124,7 +162,12 @@ class AppListApi(Resource):
         return app, 201
 
 
+@console_ns.route("/apps/<uuid:app_id>")
 class AppApi(Resource):
+    @api.doc("get_app_detail")
+    @api.doc(description="Get application details")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Success", app_detail_fields_with_site)
     @setup_required
     @login_required
     @account_initialization_required
@@ -143,6 +186,26 @@ class AppApi(Resource):
 
         return app_model
 
+    @api.doc("update_app")
+    @api.doc(description="Update application details")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "UpdateAppRequest",
+            {
+                "name": fields.String(required=True, description="App name"),
+                "description": fields.String(description="App description (max 400 chars)"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+                "use_icon_as_answer_icon": fields.Boolean(description="Use icon as answer icon"),
+                "max_active_requests": fields.Integer(description="Maximum active requests"),
+            },
+        )
+    )
+    @api.response(200, "App updated successfully", app_detail_fields_with_site)
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -181,6 +244,11 @@ class AppApi(Resource):
 
         return app_model
 
+    @api.doc("delete_app")
+    @api.doc(description="Delete application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(204, "App deleted successfully")
+    @api.response(403, "Insufficient permissions")
     @get_app_model
     @setup_required
     @login_required
@@ -197,7 +265,25 @@ class AppApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/copy")
 class AppCopyApi(Resource):
+    @api.doc("copy_app")
+    @api.doc(description="Create a copy of an existing application")
+    @api.doc(params={"app_id": "Application ID to copy"})
+    @api.expect(
+        api.model(
+            "CopyAppRequest",
+            {
+                "name": fields.String(description="Name for the copied app"),
+                "description": fields.String(description="Description for the copied app"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(201, "App copied successfully", app_detail_fields_with_site)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -239,7 +325,22 @@ class AppCopyApi(Resource):
         return app, 201
 
 
+@console_ns.route("/apps/<uuid:app_id>/export")
 class AppExportApi(Resource):
+    @api.doc("export_app")
+    @api.doc(description="Export application configuration as DSL")
+    @api.doc(params={"app_id": "Application ID to export"})
+    @api.expect(
+        api.parser()
+        .add_argument("include_secret", type=bool, location="args", default=False, help="Include secrets in export")
+        .add_argument("workflow_id", type=str, location="args", help="Specific workflow ID to export")
+    )
+    @api.response(
+        200,
+        "App exported successfully",
+        api.model("AppExportResponse", {"data": fields.String(description="DSL export data")}),
+    )
+    @api.response(403, "Insufficient permissions")
     @get_app_model
     @setup_required
     @login_required
@@ -263,7 +364,13 @@ class AppExportApi(Resource):
         }
 
 
+@console_ns.route("/apps/<uuid:app_id>/name")
 class AppNameApi(Resource):
+    @api.doc("check_app_name")
+    @api.doc(description="Check if app name is available")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(api.parser().add_argument("name", type=str, required=True, location="args", help="Name to check"))
+    @api.response(200, "Name availability checked")
     @setup_required
     @login_required
     @account_initialization_required
@@ -284,7 +391,23 @@ class AppNameApi(Resource):
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/icon")
 class AppIconApi(Resource):
+    @api.doc("update_app_icon")
+    @api.doc(description="Update application icon")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppIconRequest",
+            {
+                "icon": fields.String(required=True, description="Icon data"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(200, "Icon updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -306,7 +429,18 @@ class AppIconApi(Resource):
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/site-enable")
 class AppSiteStatus(Resource):
+    @api.doc("update_app_site_status")
+    @api.doc(description="Enable or disable app site")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppSiteStatusRequest", {"enable_site": fields.Boolean(required=True, description="Enable or disable site")}
+        )
+    )
+    @api.response(200, "Site status updated successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -327,7 +461,18 @@ class AppSiteStatus(Resource):
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/api-enable")
 class AppApiStatus(Resource):
+    @api.doc("update_app_api_status")
+    @api.doc(description="Enable or disable app API")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppApiStatusRequest", {"enable_api": fields.Boolean(required=True, description="Enable or disable API")}
+        )
+    )
+    @api.response(200, "API status updated successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -348,7 +493,12 @@ class AppApiStatus(Resource):
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/trace")
 class AppTraceApi(Resource):
+    @api.doc("get_app_trace")
+    @api.doc(description="Get app tracing configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Trace configuration retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -358,6 +508,20 @@ class AppTraceApi(Resource):
 
         return app_trace_config
 
+    @api.doc("update_app_trace")
+    @api.doc(description="Update app tracing configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppTraceRequest",
+            {
+                "enabled": fields.Boolean(required=True, description="Enable or disable tracing"),
+                "tracing_provider": fields.String(required=True, description="Tracing provider"),
+            },
+        )
+    )
+    @api.response(200, "Trace configuration updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -377,14 +541,3 @@ class AppTraceApi(Resource):
         )
 
         return {"result": "success"}
-
-
-api.add_resource(AppListApi, "/apps")
-api.add_resource(AppApi, "/apps/<uuid:app_id>")
-api.add_resource(AppCopyApi, "/apps/<uuid:app_id>/copy")
-api.add_resource(AppExportApi, "/apps/<uuid:app_id>/export")
-api.add_resource(AppNameApi, "/apps/<uuid:app_id>/name")
-api.add_resource(AppIconApi, "/apps/<uuid:app_id>/icon")
-api.add_resource(AppSiteStatus, "/apps/<uuid:app_id>/site-enable")
-api.add_resource(AppApiStatus, "/apps/<uuid:app_id>/api-enable")
-api.add_resource(AppTraceApi, "/apps/<uuid:app_id>/trace")

api/controllers/console/app/audio.py

@@ -1,11 +1,11 @@
 import logging
 
 from flask import request
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 from werkzeug.exceptions import InternalServerError
 
 import services
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
     AudioTooLargeError,
@@ -34,7 +34,18 @@ from services.errors.audio import (
 logger = logging.getLogger(__name__)
 
 
+@console_ns.route("/apps/<uuid:app_id>/audio-to-text")
 class ChatMessageAudioApi(Resource):
+    @api.doc("chat_message_audio_transcript")
+    @api.doc(description="Transcript audio to text for chat messages")
+    @api.doc(params={"app_id": "App ID"})
+    @api.response(
+        200,
+        "Audio transcription successful",
+        api.model("AudioTranscriptResponse", {"text": fields.String(description="Transcribed text from audio")}),
+    )
+    @api.response(400, "Bad request - No audio uploaded or unsupported type")
+    @api.response(413, "Audio file too large")
     @setup_required
     @login_required
     @account_initialization_required
@@ -76,7 +87,24 @@ class ChatMessageAudioApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/text-to-audio")
 class ChatMessageTextApi(Resource):
+    @api.doc("chat_message_text_to_speech")
+    @api.doc(description="Convert text to speech for chat messages")
+    @api.doc(params={"app_id": "App ID"})
+    @api.expect(
+        api.model(
+            "TextToSpeechRequest",
+            {
+                "message_id": fields.String(description="Message ID"),
+                "text": fields.String(required=True, description="Text to convert to speech"),
+                "voice": fields.String(description="Voice to use for TTS"),
+                "streaming": fields.Boolean(description="Whether to stream the audio"),
+            },
+        )
+    )
+    @api.response(200, "Text to speech conversion successful")
+    @api.response(400, "Bad request - Invalid parameters")
     @get_app_model
     @setup_required
     @login_required
@@ -124,7 +152,14 @@ class ChatMessageTextApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/text-to-audio/voices")
 class TextModesApi(Resource):
+    @api.doc("get_text_to_speech_voices")
+    @api.doc(description="Get available TTS voices for a specific language")
+    @api.doc(params={"app_id": "App ID"})
+    @api.expect(api.parser().add_argument("language", type=str, required=True, location="args", help="Language code"))
+    @api.response(200, "TTS voices retrieved successfully", fields.List(fields.Raw(description="Available voices")))
+    @api.response(400, "Invalid language parameter")
     @get_app_model
     @setup_required
     @login_required
@@ -164,8 +199,3 @@ class TextModesApi(Resource):
         except Exception as e:
             logger.exception("Failed to handle get request to TextModesApi")
             raise InternalServerError()
-
-
-api.add_resource(ChatMessageAudioApi, "/apps/<uuid:app_id>/audio-to-text")
-api.add_resource(ChatMessageTextApi, "/apps/<uuid:app_id>/text-to-audio")
-api.add_resource(TextModesApi, "/apps/<uuid:app_id>/text-to-audio/voices")

api/controllers/console/app/completion.py

@@ -1,11 +1,11 @@
 import logging
 
 from flask import request
-from flask_restx import Resource, reqparse
-from werkzeug.exceptions import InternalServerError, NotFound
+from flask_restx import Resource, fields, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
     CompletionRequestError,
@@ -38,7 +38,27 @@ logger = logging.getLogger(__name__)
 
 
 # define completion message api for user
+@console_ns.route("/apps/<uuid:app_id>/completion-messages")
 class CompletionMessageApi(Resource):
+    @api.doc("create_completion_message")
+    @api.doc(description="Generate completion message for debugging")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "CompletionMessageRequest",
+            {
+                "inputs": fields.Raw(required=True, description="Input variables"),
+                "query": fields.String(description="Query text", default=""),
+                "files": fields.List(fields.Raw(), description="Uploaded files"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "response_mode": fields.String(enum=["blocking", "streaming"], description="Response mode"),
+                "retriever_from": fields.String(default="dev", description="Retriever source"),
+            },
+        )
+    )
+    @api.response(200, "Completion generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(404, "App not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -86,7 +106,12 @@ class CompletionMessageApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop")
 class CompletionMessageStopApi(Resource):
+    @api.doc("stop_completion_message")
+    @api.doc(description="Stop a running completion message generation")
+    @api.doc(params={"app_id": "Application ID", "task_id": "Task ID to stop"})
+    @api.response(200, "Task stopped successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -99,12 +124,40 @@ class CompletionMessageStopApi(Resource):
         return {"result": "success"}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages")
 class ChatMessageApi(Resource):
+    @api.doc("create_chat_message")
+    @api.doc(description="Generate chat message for debugging")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "ChatMessageRequest",
+            {
+                "inputs": fields.Raw(required=True, description="Input variables"),
+                "query": fields.String(required=True, description="User query"),
+                "files": fields.List(fields.Raw(), description="Uploaded files"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "conversation_id": fields.String(description="Conversation ID"),
+                "parent_message_id": fields.String(description="Parent message ID"),
+                "response_mode": fields.String(enum=["blocking", "streaming"], description="Response mode"),
+                "retriever_from": fields.String(default="dev", description="Retriever source"),
+            },
+        )
+    )
+    @api.response(200, "Chat message generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(404, "App or conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT])
     def post(self, app_model):
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        if not current_user.has_edit_permission:
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("inputs", type=dict, required=True, location="json")
         parser.add_argument("query", type=str, required=True, location="json")
@@ -155,7 +208,12 @@ class ChatMessageApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")
 class ChatMessageStopApi(Resource):
+    @api.doc("stop_chat_message")
+    @api.doc(description="Stop a running chat message generation")
+    @api.doc(params={"app_id": "Application ID", "task_id": "Task ID to stop"})
+    @api.response(200, "Task stopped successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -166,9 +224,3 @@ class ChatMessageStopApi(Resource):
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
 
         return {"result": "success"}, 200
-
-
-api.add_resource(CompletionMessageApi, "/apps/<uuid:app_id>/completion-messages")
-api.add_resource(CompletionMessageStopApi, "/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop")
-api.add_resource(ChatMessageApi, "/apps/<uuid:app_id>/chat-messages")
-api.add_resource(ChatMessageStopApi, "/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")

api/controllers/console/app/conversation.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 import pytz  # pip install pytz
+import sqlalchemy as sa
 from flask_login import current_user
 from flask_restx import Resource, marshal_with, reqparse
 from flask_restx.inputs import int_range
@@ -8,7 +9,7 @@ from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
 from werkzeug.exceptions import Forbidden, NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -28,7 +29,29 @@ from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-conversations")
 class CompletionConversationApi(Resource):
+    @api.doc("list_completion_conversations")
+    @api.doc(description="Get completion conversations with pagination and filtering")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("keyword", type=str, location="args", help="Search keyword")
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+        .add_argument(
+            "annotation_status",
+            type=str,
+            location="args",
+            choices=["annotated", "not_annotated", "all"],
+            default="all",
+            help="Annotation status filter",
+        )
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size (1-100)")
+    )
+    @api.response(200, "Success", conversation_pagination_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -48,7 +71,7 @@ class CompletionConversationApi(Resource):
         parser.add_argument("limit", type=int_range(1, 100), default=20, location="args")
         args = parser.parse_args()
 
-        query = db.select(Conversation).where(
+        query = sa.select(Conversation).where(
             Conversation.app_id == app_model.id, Conversation.mode == "completion", Conversation.is_deleted.is_(False)
         )
 
@@ -101,7 +124,14 @@ class CompletionConversationApi(Resource):
         return conversations
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>")
 class CompletionConversationDetailApi(Resource):
+    @api.doc("get_completion_conversation")
+    @api.doc(description="Get completion conversation details with messages")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(200, "Success", conversation_message_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -114,6 +144,12 @@ class CompletionConversationDetailApi(Resource):
 
         return _get_conversation(app_model, conversation_id)
 
+    @api.doc("delete_completion_conversation")
+    @api.doc(description="Delete a completion conversation")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(204, "Conversation deleted successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -133,7 +169,38 @@ class CompletionConversationDetailApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-conversations")
 class ChatConversationApi(Resource):
+    @api.doc("list_chat_conversations")
+    @api.doc(description="Get chat conversations with pagination, filtering and summary")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("keyword", type=str, location="args", help="Search keyword")
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+        .add_argument(
+            "annotation_status",
+            type=str,
+            location="args",
+            choices=["annotated", "not_annotated", "all"],
+            default="all",
+            help="Annotation status filter",
+        )
+        .add_argument("message_count_gte", type=int, location="args", help="Minimum message count")
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size (1-100)")
+        .add_argument(
+            "sort_by",
+            type=str,
+            location="args",
+            choices=["created_at", "-created_at", "updated_at", "-updated_at"],
+            default="-updated_at",
+            help="Sort field and direction",
+        )
+    )
+    @api.response(200, "Success", conversation_with_summary_pagination_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -170,7 +237,7 @@ class ChatConversationApi(Resource):
             .subquery()
         )
 
-        query = db.select(Conversation).where(Conversation.app_id == app_model.id, Conversation.is_deleted.is_(False))
+        query = sa.select(Conversation).where(Conversation.app_id == app_model.id, Conversation.is_deleted.is_(False))
 
         if args["keyword"]:
             keyword_filter = f"%{args['keyword']}%"
@@ -241,7 +308,7 @@ class ChatConversationApi(Resource):
                 .having(func.count(Message.id) >= args["message_count_gte"])
             )
 
-        if app_model.mode == AppMode.ADVANCED_CHAT.value:
+        if app_model.mode == AppMode.ADVANCED_CHAT:
             query = query.where(Conversation.invoke_from != InvokeFrom.DEBUGGER.value)
 
         match args["sort_by"]:
@@ -261,7 +328,14 @@ class ChatConversationApi(Resource):
         return conversations
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>")
 class ChatConversationDetailApi(Resource):
+    @api.doc("get_chat_conversation")
+    @api.doc(description="Get chat conversation details")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(200, "Success", conversation_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -274,6 +348,12 @@ class ChatConversationDetailApi(Resource):
 
         return _get_conversation(app_model, conversation_id)
 
+    @api.doc("delete_chat_conversation")
+    @api.doc(description="Delete a chat conversation")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(204, "Conversation deleted successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
@@ -293,12 +373,6 @@ class ChatConversationDetailApi(Resource):
         return {"result": "success"}, 204
 
 
-api.add_resource(CompletionConversationApi, "/apps/<uuid:app_id>/completion-conversations")
-api.add_resource(CompletionConversationDetailApi, "/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>")
-api.add_resource(ChatConversationApi, "/apps/<uuid:app_id>/chat-conversations")
-api.add_resource(ChatConversationDetailApi, "/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>")
-
-
 def _get_conversation(app_model, conversation_id):
     conversation = (
         db.session.query(Conversation)

api/controllers/console/app/conversation_variables.py

@@ -2,7 +2,7 @@ from flask_restx import Resource, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -12,7 +12,17 @@ from models import ConversationVariable
 from models.model import AppMode
 
 
+@console_ns.route("/apps/<uuid:app_id>/conversation-variables")
 class ConversationVariablesApi(Resource):
+    @api.doc("get_conversation_variables")
+    @api.doc(description="Get conversation variables for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "conversation_id", type=str, location="args", help="Conversation ID to filter variables"
+        )
+    )
+    @api.response(200, "Conversation variables retrieved successfully", paginated_conversation_variable_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -55,6 +65,3 @@ class ConversationVariablesApi(Resource):
                 for row in rows
             ],
         }
-
-
-api.add_resource(ConversationVariablesApi, "/apps/<uuid:app_id>/conversation-variables")

api/controllers/console/app/generator.py

@@ -1,9 +1,9 @@
 from collections.abc import Sequence
 
 from flask_login import current_user
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -16,10 +16,29 @@ from core.helper.code_executor.javascript.javascript_code_provider import Javasc
 from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
+from extensions.ext_database import db
 from libs.login import login_required
+from models import App
+from services.workflow_service import WorkflowService
 
 
+@console_ns.route("/rule-generate")
 class RuleGenerateApi(Resource):
+    @api.doc("generate_rule_config")
+    @api.doc(description="Generate rule configuration using LLM")
+    @api.expect(
+        api.model(
+            "RuleGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Rule generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "no_variable": fields.Boolean(required=True, default=False, description="Whether to exclude variables"),
+            },
+        )
+    )
+    @api.response(200, "Rule configuration generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -50,7 +69,26 @@ class RuleGenerateApi(Resource):
         return rules
 
 
+@console_ns.route("/rule-code-generate")
 class RuleCodeGenerateApi(Resource):
+    @api.doc("generate_rule_code")
+    @api.doc(description="Generate code rules using LLM")
+    @api.expect(
+        api.model(
+            "RuleCodeGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Code generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "no_variable": fields.Boolean(required=True, default=False, description="Whether to exclude variables"),
+                "code_language": fields.String(
+                    default="javascript", description="Programming language for code generation"
+                ),
+            },
+        )
+    )
+    @api.response(200, "Code rules generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -82,7 +120,22 @@ class RuleCodeGenerateApi(Resource):
         return code_result
 
 
+@console_ns.route("/rule-structured-output-generate")
 class RuleStructuredOutputGenerateApi(Resource):
+    @api.doc("generate_structured_output")
+    @api.doc(description="Generate structured output rules using LLM")
+    @api.expect(
+        api.model(
+            "StructuredOutputGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Structured output generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+            },
+        )
+    )
+    @api.response(200, "Structured output generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -111,7 +164,27 @@ class RuleStructuredOutputGenerateApi(Resource):
         return structured_output
 
 
+@console_ns.route("/instruction-generate")
 class InstructionGenerateApi(Resource):
+    @api.doc("generate_instruction")
+    @api.doc(description="Generate instruction for workflow nodes or general use")
+    @api.expect(
+        api.model(
+            "InstructionGenerateRequest",
+            {
+                "flow_id": fields.String(required=True, description="Workflow/Flow ID"),
+                "node_id": fields.String(description="Node ID for workflow context"),
+                "current": fields.String(description="Current instruction text"),
+                "language": fields.String(default="javascript", description="Programming language (javascript/python)"),
+                "instruction": fields.String(required=True, description="Instruction for generation"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "ideal_output": fields.String(description="Expected ideal output"),
+            },
+        )
+    )
+    @api.response(200, "Instruction generated successfully")
+    @api.response(400, "Invalid request parameters or flow/workflow not found")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -135,9 +208,6 @@ class InstructionGenerateApi(Resource):
         try:
             # Generate from nothing for a workflow node
             if (args["current"] == code_template or args["current"] == "") and args["node_id"] != "":
-                from models import App, db
-                from services.workflow_service import WorkflowService
-
                 app = db.session.query(App).where(App.id == args["flow_id"]).first()
                 if not app:
                     return {"error": f"app {args['flow_id']} not found"}, 400
@@ -191,6 +261,7 @@ class InstructionGenerateApi(Resource):
                     instruction=args["instruction"],
                     model_config=args["model_config"],
                     ideal_output=args["ideal_output"],
+                    workflow_service=WorkflowService(),
                 )
             return {"error": "incompatible parameters"}, 400
         except ProviderTokenNotInitError as ex:
@@ -203,7 +274,21 @@ class InstructionGenerateApi(Resource):
             raise CompletionRequestError(e.description)
 
 
+@console_ns.route("/instruction-generate/template")
 class InstructionGenerationTemplateApi(Resource):
+    @api.doc("get_instruction_template")
+    @api.doc(description="Get instruction generation template")
+    @api.expect(
+        api.model(
+            "InstructionTemplateRequest",
+            {
+                "instruction": fields.String(required=True, description="Template instruction"),
+                "ideal_output": fields.String(description="Expected ideal output"),
+            },
+        )
+    )
+    @api.response(200, "Template retrieved successfully")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -222,10 +307,3 @@ class InstructionGenerationTemplateApi(Resource):
                 return {"data": INSTRUCTION_GENERATE_TEMPLATE_CODE}
             case _:
                 raise ValueError(f"Invalid type: {args['type']}")
-
-
-api.add_resource(RuleGenerateApi, "/rule-generate")
-api.add_resource(RuleCodeGenerateApi, "/rule-code-generate")
-api.add_resource(RuleStructuredOutputGenerateApi, "/rule-structured-output-generate")
-api.add_resource(InstructionGenerateApi, "/instruction-generate")
-api.add_resource(InstructionGenerationTemplateApi, "/instruction-generate/template")

api/controllers/console/app/mcp_server.py

@@ -2,10 +2,10 @@ import json
 from enum import StrEnum
 
 from flask_login import current_user
-from flask_restx import Resource, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -19,7 +19,12 @@ class AppMCPServerStatus(StrEnum):
     INACTIVE = "inactive"
 
 
+@console_ns.route("/apps/<uuid:app_id>/server")
 class AppMCPServerController(Resource):
+    @api.doc("get_app_mcp_server")
+    @api.doc(description="Get MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "MCP server configuration retrieved successfully", app_server_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -29,6 +34,20 @@ class AppMCPServerController(Resource):
         server = db.session.query(AppMCPServer).where(AppMCPServer.app_id == app_model.id).first()
         return server
 
+    @api.doc("create_app_mcp_server")
+    @api.doc(description="Create MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MCPServerCreateRequest",
+            {
+                "description": fields.String(description="Server description"),
+                "parameters": fields.Raw(required=True, description="Server parameters configuration"),
+            },
+        )
+    )
+    @api.response(201, "MCP server configuration created successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -59,6 +78,23 @@ class AppMCPServerController(Resource):
         db.session.commit()
         return server
 
+    @api.doc("update_app_mcp_server")
+    @api.doc(description="Update MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MCPServerUpdateRequest",
+            {
+                "id": fields.String(required=True, description="Server ID"),
+                "description": fields.String(description="Server description"),
+                "parameters": fields.Raw(required=True, description="Server parameters configuration"),
+                "status": fields.String(description="Server status"),
+            },
+        )
+    )
+    @api.response(200, "MCP server configuration updated successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Server not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -94,7 +130,14 @@ class AppMCPServerController(Resource):
         return server
 
 
+@console_ns.route("/apps/<uuid:server_id>/server/refresh")
 class AppMCPServerRefreshController(Resource):
+    @api.doc("refresh_app_mcp_server")
+    @api.doc(description="Refresh MCP server configuration and regenerate server code")
+    @api.doc(params={"server_id": "Server ID"})
+    @api.response(200, "MCP server refreshed successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Server not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -113,7 +156,3 @@ class AppMCPServerRefreshController(Resource):
         server.server_code = AppMCPServer.generate_server_code(16)
         db.session.commit()
         return server
-
-
-api.add_resource(AppMCPServerController, "/apps/<uuid:app_id>/server")
-api.add_resource(AppMCPServerRefreshController, "/apps/<uuid:server_id>/server/refresh")

api/controllers/console/app/message.py

@@ -5,7 +5,7 @@ from flask_restx.inputs import int_range
 from sqlalchemy import exists, select
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -37,6 +37,7 @@ from services.message_service import MessageService
 logger = logging.getLogger(__name__)
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages")
 class ChatMessageListApi(Resource):
     message_infinite_scroll_pagination_fields = {
         "limit": fields.Integer,
@@ -44,12 +45,26 @@ class ChatMessageListApi(Resource):
         "data": fields.List(fields.Nested(message_detail_fields)),
     }
 
+    @api.doc("list_chat_messages")
+    @api.doc(description="Get chat messages for a conversation with pagination")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("conversation_id", type=str, required=True, location="args", help="Conversation ID")
+        .add_argument("first_id", type=str, location="args", help="First message ID for pagination")
+        .add_argument("limit", type=int, location="args", default=20, help="Number of messages to return (1-100)")
+    )
+    @api.response(200, "Success", message_infinite_scroll_pagination_fields)
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @account_initialization_required
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model):
+        if not isinstance(current_user, Account) or not current_user.has_edit_permission:
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("conversation_id", required=True, type=uuid_value, location="args")
         parser.add_argument("first_id", type=uuid_value, location="args")
@@ -117,7 +132,23 @@ class ChatMessageListApi(Resource):
         return InfiniteScrollPagination(data=history_messages, limit=args["limit"], has_more=has_more)
 
 
+@console_ns.route("/apps/<uuid:app_id>/feedbacks")
 class MessageFeedbackApi(Resource):
+    @api.doc("create_message_feedback")
+    @api.doc(description="Create or update message feedback (like/dislike)")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MessageFeedbackRequest",
+            {
+                "message_id": fields.String(required=True, description="Message ID"),
+                "rating": fields.String(enum=["like", "dislike"], description="Feedback rating"),
+            },
+        )
+    )
+    @api.response(200, "Feedback updated successfully")
+    @api.response(404, "Message not found")
+    @api.response(403, "Insufficient permissions")
     @get_app_model
     @setup_required
     @login_required
@@ -162,7 +193,24 @@ class MessageFeedbackApi(Resource):
         return {"result": "success"}
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations")
 class MessageAnnotationApi(Resource):
+    @api.doc("create_message_annotation")
+    @api.doc(description="Create message annotation")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MessageAnnotationRequest",
+            {
+                "message_id": fields.String(description="Message ID"),
+                "question": fields.String(required=True, description="Question text"),
+                "answer": fields.String(required=True, description="Answer text"),
+                "annotation_reply": fields.Raw(description="Annotation reply"),
+            },
+        )
+    )
+    @api.response(200, "Annotation created successfully", annotation_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -172,7 +220,7 @@ class MessageAnnotationApi(Resource):
     def post(self, app_model):
         if not isinstance(current_user, Account):
             raise Forbidden()
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -186,7 +234,16 @@ class MessageAnnotationApi(Resource):
         return annotation
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/count")
 class MessageAnnotationCountApi(Resource):
+    @api.doc("get_annotation_count")
+    @api.doc(description="Get count of message annotations for the app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(
+        200,
+        "Annotation count retrieved successfully",
+        api.model("AnnotationCountResponse", {"count": fields.Integer(description="Number of annotations")}),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -197,7 +254,17 @@ class MessageAnnotationCountApi(Resource):
         return {"count": count}
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions")
 class MessageSuggestedQuestionApi(Resource):
+    @api.doc("get_message_suggested_questions")
+    @api.doc(description="Get suggested questions for a message")
+    @api.doc(params={"app_id": "Application ID", "message_id": "Message ID"})
+    @api.response(
+        200,
+        "Suggested questions retrieved successfully",
+        api.model("SuggestedQuestionsResponse", {"data": fields.List(fields.String(description="Suggested question"))}),
+    )
+    @api.response(404, "Message or conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -230,7 +297,13 @@ class MessageSuggestedQuestionApi(Resource):
         return {"data": questions}
 
 
+@console_ns.route("/apps/<uuid:app_id>/messages/<uuid:message_id>")
 class MessageApi(Resource):
+    @api.doc("get_message")
+    @api.doc(description="Get message details by ID")
+    @api.doc(params={"app_id": "Application ID", "message_id": "Message ID"})
+    @api.response(200, "Message retrieved successfully", message_detail_fields)
+    @api.response(404, "Message not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -245,11 +318,3 @@ class MessageApi(Resource):
             raise NotFound("Message Not Exists.")
 
         return message
-
-
-api.add_resource(MessageSuggestedQuestionApi, "/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions")
-api.add_resource(ChatMessageListApi, "/apps/<uuid:app_id>/chat-messages", endpoint="console_chat_messages")
-api.add_resource(MessageFeedbackApi, "/apps/<uuid:app_id>/feedbacks")
-api.add_resource(MessageAnnotationApi, "/apps/<uuid:app_id>/annotations")
-api.add_resource(MessageAnnotationCountApi, "/apps/<uuid:app_id>/annotations/count")
-api.add_resource(MessageApi, "/apps/<uuid:app_id>/messages/<uuid:message_id>", endpoint="console_message")

api/controllers/console/app/model_config.py

@@ -3,9 +3,10 @@ from typing import cast
 
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource
+from flask_restx import Resource, fields
+from werkzeug.exceptions import Forbidden
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.agent.entities import AgentToolEntity
@@ -14,17 +15,51 @@ from core.tools.utils.configuration import ToolParameterConfigurationManager
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
 from libs.login import login_required
+from models.account import Account
 from models.model import AppMode, AppModelConfig
 from services.app_model_config_service import AppModelConfigService
 
 
+@console_ns.route("/apps/<uuid:app_id>/model-config")
 class ModelConfigResource(Resource):
+    @api.doc("update_app_model_config")
+    @api.doc(description="Update application model configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "ModelConfigRequest",
+            {
+                "provider": fields.String(description="Model provider"),
+                "model": fields.String(description="Model name"),
+                "configs": fields.Raw(description="Model configuration parameters"),
+                "opening_statement": fields.String(description="Opening statement"),
+                "suggested_questions": fields.List(fields.String(), description="Suggested questions"),
+                "more_like_this": fields.Raw(description="More like this configuration"),
+                "speech_to_text": fields.Raw(description="Speech to text configuration"),
+                "text_to_speech": fields.Raw(description="Text to speech configuration"),
+                "retrieval_model": fields.Raw(description="Retrieval model configuration"),
+                "tools": fields.List(fields.Raw(), description="Available tools"),
+                "dataset_configs": fields.Raw(description="Dataset configurations"),
+                "agent_mode": fields.Raw(description="Agent mode configuration"),
+            },
+        )
+    )
+    @api.response(200, "Model configuration updated successfully")
+    @api.response(400, "Invalid configuration")
+    @api.response(404, "App not found")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
     def post(self, app_model):
         """Modify app model config"""
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        if not current_user.has_edit_permission:
+            raise Forbidden()
+
+        assert current_user.current_tenant_id is not None, "The tenant information should be loaded."
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(
             tenant_id=current_user.current_tenant_id,
@@ -39,7 +74,7 @@ class ModelConfigResource(Resource):
         )
         new_app_model_config = new_app_model_config.from_model_config_dict(model_configuration)
 
-        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+        if app_model.mode == AppMode.AGENT_CHAT or app_model.is_agent:
             # get original app model config
             original_app_model_config = (
                 db.session.query(AppModelConfig).where(AppModelConfig.id == app_model.app_model_config_id).first()
@@ -142,6 +177,3 @@ class ModelConfigResource(Resource):
         app_model_config_was_updated.send(app_model, app_model_config=new_app_model_config)
 
         return {"result": "success"}
-
-
-api.add_resource(ModelConfigResource, "/apps/<uuid:app_id>/model-config")

api/controllers/console/app/ops_trace.py

@@ -1,18 +1,31 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 from werkzeug.exceptions import BadRequest
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.ops_service import OpsService
 
 
+@console_ns.route("/apps/<uuid:app_id>/trace-config")
 class TraceAppConfigApi(Resource):
     """
     Manage trace app configurations
     """
 
+    @api.doc("get_trace_app_config")
+    @api.doc(description="Get tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "tracing_provider", type=str, required=True, location="args", help="Tracing provider name"
+        )
+    )
+    @api.response(
+        200, "Tracing configuration retrieved successfully", fields.Raw(description="Tracing configuration data")
+    )
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -29,6 +42,22 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("create_trace_app_config")
+    @api.doc(description="Create a new tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "TraceConfigCreateRequest",
+            {
+                "tracing_provider": fields.String(required=True, description="Tracing provider name"),
+                "tracing_config": fields.Raw(required=True, description="Tracing configuration data"),
+            },
+        )
+    )
+    @api.response(
+        201, "Tracing configuration created successfully", fields.Raw(description="Created configuration data")
+    )
+    @api.response(400, "Invalid request parameters or configuration already exists")
     @setup_required
     @login_required
     @account_initialization_required
@@ -51,6 +80,20 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("update_trace_app_config")
+    @api.doc(description="Update an existing tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "TraceConfigUpdateRequest",
+            {
+                "tracing_provider": fields.String(required=True, description="Tracing provider name"),
+                "tracing_config": fields.Raw(required=True, description="Updated tracing configuration data"),
+            },
+        )
+    )
+    @api.response(200, "Tracing configuration updated successfully", fields.Raw(description="Success response"))
+    @api.response(400, "Invalid request parameters or configuration not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -71,6 +114,16 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("delete_trace_app_config")
+    @api.doc(description="Delete an existing tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "tracing_provider", type=str, required=True, location="args", help="Tracing provider name"
+        )
+    )
+    @api.response(204, "Tracing configuration deleted successfully")
+    @api.response(400, "Invalid request parameters or configuration not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -87,6 +140,3 @@ class TraceAppConfigApi(Resource):
             return {"result": "success"}, 204
         except Exception as e:
             raise BadRequest(str(e))
-
-
-api.add_resource(TraceAppConfigApi, "/apps/<uuid:app_id>/trace-config")

api/controllers/console/app/site.py

@@ -1,9 +1,9 @@
 from flask_login import current_user
-from flask_restx import Resource, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 from constants.languages import supported_language
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -36,7 +36,39 @@ def parse_app_site_args():
     return parser.parse_args()
 
 
+@console_ns.route("/apps/<uuid:app_id>/site")
 class AppSite(Resource):
+    @api.doc("update_app_site")
+    @api.doc(description="Update application site configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppSiteRequest",
+            {
+                "title": fields.String(description="Site title"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+                "description": fields.String(description="Site description"),
+                "default_language": fields.String(description="Default language"),
+                "chat_color_theme": fields.String(description="Chat color theme"),
+                "chat_color_theme_inverted": fields.Boolean(description="Inverted chat color theme"),
+                "customize_domain": fields.String(description="Custom domain"),
+                "copyright": fields.String(description="Copyright text"),
+                "privacy_policy": fields.String(description="Privacy policy"),
+                "custom_disclaimer": fields.String(description="Custom disclaimer"),
+                "customize_token_strategy": fields.String(
+                    enum=["must", "allow", "not_allow"], description="Token strategy"
+                ),
+                "prompt_public": fields.Boolean(description="Make prompt public"),
+                "show_workflow_steps": fields.Boolean(description="Show workflow steps"),
+                "use_icon_as_answer_icon": fields.Boolean(description="Use icon as answer icon"),
+            },
+        )
+    )
+    @api.response(200, "Site configuration updated successfully", app_site_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "App not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -84,7 +116,14 @@ class AppSite(Resource):
         return site
 
 
+@console_ns.route("/apps/<uuid:app_id>/site/access-token-reset")
 class AppSiteAccessTokenReset(Resource):
+    @api.doc("reset_app_site_access_token")
+    @api.doc(description="Reset access token for application site")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Access token reset successfully", app_site_fields)
+    @api.response(403, "Insufficient permissions (admin/owner required)")
+    @api.response(404, "App or site not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -108,7 +147,3 @@ class AppSiteAccessTokenReset(Resource):
         db.session.commit()
 
         return site
-
-
-api.add_resource(AppSite, "/apps/<uuid:app_id>/site")
-api.add_resource(AppSiteAccessTokenReset, "/apps/<uuid:app_id>/site/access-token-reset")

api/controllers/console/app/statistic.py

@@ -5,9 +5,9 @@ import pytz
 import sqlalchemy as sa
 from flask import jsonify
 from flask_login import current_user
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -17,7 +17,21 @@ from libs.login import login_required
 from models import AppMode, Message
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/daily-messages")
 class DailyMessageStatistic(Resource):
+    @api.doc("get_daily_message_statistics")
+    @api.doc(description="Get daily message statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Daily message statistics retrieved successfully",
+        fields.List(fields.Raw(description="Daily message count data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -36,8 +50,9 @@ class DailyMessageStatistic(Resource):
 FROM
     messages
 WHERE
-    app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    app_id = :app_id
+    AND invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -74,7 +89,21 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/daily-conversations")
 class DailyConversationStatistic(Resource):
+    @api.doc("get_daily_conversation_statistics")
+    @api.doc(description="Get daily conversation statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Daily conversation statistics retrieved successfully",
+        fields.List(fields.Raw(description="Daily conversation count data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -126,7 +155,21 @@ class DailyConversationStatistic(Resource):
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/daily-end-users")
 class DailyTerminalsStatistic(Resource):
+    @api.doc("get_daily_terminals_statistics")
+    @api.doc(description="Get daily terminal/end-user statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Daily terminal statistics retrieved successfully",
+        fields.List(fields.Raw(description="Daily terminal count data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -145,8 +188,9 @@ class DailyTerminalsStatistic(Resource):
 FROM
     messages
 WHERE
-    app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    app_id = :app_id
+    AND invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -183,7 +227,21 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/token-costs")
 class DailyTokenCostStatistic(Resource):
+    @api.doc("get_daily_token_cost_statistics")
+    @api.doc(description="Get daily token cost statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Daily token cost statistics retrieved successfully",
+        fields.List(fields.Raw(description="Daily token cost data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -203,8 +261,9 @@ class DailyTokenCostStatistic(Resource):
 FROM
     messages
 WHERE
-    app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    app_id = :app_id
+    AND invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -243,7 +302,21 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/average-session-interactions")
 class AverageSessionInteractionStatistic(Resource):
+    @api.doc("get_average_session_interaction_statistics")
+    @api.doc(description="Get average session interaction statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Average session interaction statistics retrieved successfully",
+        fields.List(fields.Raw(description="Average session interaction data")),
+    )
     @setup_required
     @login_required
     @account_initialization_required
@@ -270,8 +343,9 @@ FROM
             messages m
             ON c.id = m.conversation_id
         WHERE
-            c.app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+            c.app_id = :app_id
+            AND m.invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -319,7 +393,21 @@ ORDER BY
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/user-satisfaction-rate")
 class UserSatisfactionRateStatistic(Resource):
+    @api.doc("get_user_satisfaction_rate_statistics")
+    @api.doc(description="Get user satisfaction rate statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "User satisfaction rate statistics retrieved successfully",
+        fields.List(fields.Raw(description="User satisfaction rate data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -342,8 +430,9 @@ LEFT JOIN
     message_feedbacks mf
     ON mf.message_id=m.id AND mf.rating='like'
 WHERE
-    m.app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    m.app_id = :app_id
+    AND m.invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -385,7 +474,21 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/average-response-time")
 class AverageResponseTimeStatistic(Resource):
+    @api.doc("get_average_response_time_statistics")
+    @api.doc(description="Get average response time statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Average response time statistics retrieved successfully",
+        fields.List(fields.Raw(description="Average response time data")),
+    )
     @setup_required
     @login_required
     @account_initialization_required
@@ -404,8 +507,9 @@ class AverageResponseTimeStatistic(Resource):
 FROM
     messages
 WHERE
-    app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    app_id = :app_id
+    AND invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -442,7 +546,21 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/statistics/tokens-per-second")
 class TokensPerSecondStatistic(Resource):
+    @api.doc("get_tokens_per_second_statistics")
+    @api.doc(description="Get tokens per second statistics for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+    )
+    @api.response(
+        200,
+        "Tokens per second statistics retrieved successfully",
+        fields.List(fields.Raw(description="Tokens per second data")),
+    )
     @get_app_model
     @setup_required
     @login_required
@@ -464,8 +582,9 @@ class TokensPerSecondStatistic(Resource):
 FROM
     messages
 WHERE
-    app_id = :app_id"""
-        arg_dict = {"tz": account.timezone, "app_id": app_model.id}
+    app_id = :app_id
+    AND invoke_from != :invoke_from"""
+        arg_dict = {"tz": account.timezone, "app_id": app_model.id, "invoke_from": InvokeFrom.DEBUGGER.value}
 
         timezone = pytz.timezone(account.timezone)
         utc_timezone = pytz.utc
@@ -500,13 +619,3 @@ WHERE
                 response_data.append({"date": str(i.date), "tps": round(i.tokens_per_second, 4)})
 
         return jsonify({"data": response_data})
-
-
-api.add_resource(DailyMessageStatistic, "/apps/<uuid:app_id>/statistics/daily-messages")
-api.add_resource(DailyConversationStatistic, "/apps/<uuid:app_id>/statistics/daily-conversations")
-api.add_resource(DailyTerminalsStatistic, "/apps/<uuid:app_id>/statistics/daily-end-users")
-api.add_resource(DailyTokenCostStatistic, "/apps/<uuid:app_id>/statistics/token-costs")
-api.add_resource(AverageSessionInteractionStatistic, "/apps/<uuid:app_id>/statistics/average-session-interactions")
-api.add_resource(UserSatisfactionRateStatistic, "/apps/<uuid:app_id>/statistics/user-satisfaction-rate")
-api.add_resource(AverageResponseTimeStatistic, "/apps/<uuid:app_id>/statistics/average-response-time")
-api.add_resource(TokensPerSecondStatistic, "/apps/<uuid:app_id>/statistics/tokens-per-second")

api/controllers/console/app/workflow.py

@@ -4,13 +4,12 @@ from collections.abc import Sequence
 from typing import cast
 
 from flask import abort, request
-from flask_restx import Resource, inputs, marshal_with, reqparse
+from flask_restx import Resource, fields, inputs, marshal_with, reqparse
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
-from configs import dify_config
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
@@ -20,6 +19,7 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.file.models import File
 from core.helper.trace_id_helper import get_external_trace_id
+from core.workflow.graph_engine.manager import GraphEngineManager
 from extensions.ext_database import db
 from factories import file_factory, variable_factory
 from fields.workflow_fields import workflow_fields, workflow_pagination_fields
@@ -57,7 +57,13 @@ def _parse_file(workflow: Workflow, files: list[dict] | None = None) -> Sequence
     return file_objs
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft")
 class DraftWorkflowApi(Resource):
+    @api.doc("get_draft_workflow")
+    @api.doc(description="Get draft workflow for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Draft workflow retrieved successfully", workflow_fields)
+    @api.response(404, "Draft workflow not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -69,7 +75,7 @@ class DraftWorkflowApi(Resource):
         """
         # The role of the current user in the ta table must be admin, owner, or editor
         assert isinstance(current_user, Account)
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         # fetch draft workflow by app_model
@@ -86,13 +92,30 @@ class DraftWorkflowApi(Resource):
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @api.doc("sync_draft_workflow")
+    @api.doc(description="Sync draft workflow configuration")
+    @api.expect(
+        api.model(
+            "SyncDraftWorkflowRequest",
+            {
+                "graph": fields.Raw(required=True, description="Workflow graph configuration"),
+                "features": fields.Raw(required=True, description="Workflow features configuration"),
+                "hash": fields.String(description="Workflow hash for validation"),
+                "environment_variables": fields.List(fields.Raw, required=True, description="Environment variables"),
+                "conversation_variables": fields.List(fields.Raw, description="Conversation variables"),
+            },
+        )
+    )
+    @api.response(200, "Draft workflow synced successfully", workflow_fields)
+    @api.response(400, "Invalid workflow configuration")
+    @api.response(403, "Permission denied")
     def post(self, app_model: App):
         """
         Sync draft workflow
         """
         # The role of the current user in the ta table must be admin, owner, or editor
         assert isinstance(current_user, Account)
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         content_type = request.headers.get("Content-Type", "")
@@ -159,7 +182,25 @@ class DraftWorkflowApi(Resource):
         }
 
 
+@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflows/draft/run")
 class AdvancedChatDraftWorkflowRunApi(Resource):
+    @api.doc("run_advanced_chat_draft_workflow")
+    @api.doc(description="Run draft workflow for advanced chat application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AdvancedChatWorkflowRunRequest",
+            {
+                "query": fields.String(required=True, description="User query"),
+                "inputs": fields.Raw(description="Input variables"),
+                "files": fields.List(fields.Raw, description="File uploads"),
+                "conversation_id": fields.String(description="Conversation ID"),
+            },
+        )
+    )
+    @api.response(200, "Workflow run started successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -170,7 +211,7 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         """
         # The role of the current user in the ta table must be admin, owner, or editor
         assert isinstance(current_user, Account)
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         if not isinstance(current_user, Account):
@@ -208,7 +249,23 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflows/draft/iteration/nodes/<string:node_id>/run")
 class AdvancedChatDraftRunIterationNodeApi(Resource):
+    @api.doc("run_advanced_chat_draft_iteration_node")
+    @api.doc(description="Run draft workflow iteration node for advanced chat")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.expect(
+        api.model(
+            "IterationNodeRunRequest",
+            {
+                "task_id": fields.String(required=True, description="Task ID"),
+                "inputs": fields.Raw(description="Input variables"),
+            },
+        )
+    )
+    @api.response(200, "Iteration node run started successfully")
+    @api.response(403, "Permission denied")
+    @api.response(404, "Node not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -220,7 +277,7 @@ class AdvancedChatDraftRunIterationNodeApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -244,7 +301,23 @@ class AdvancedChatDraftRunIterationNodeApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/iteration/nodes/<string:node_id>/run")
 class WorkflowDraftRunIterationNodeApi(Resource):
+    @api.doc("run_workflow_draft_iteration_node")
+    @api.doc(description="Run draft workflow iteration node")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.expect(
+        api.model(
+            "WorkflowIterationNodeRunRequest",
+            {
+                "task_id": fields.String(required=True, description="Task ID"),
+                "inputs": fields.Raw(description="Input variables"),
+            },
+        )
+    )
+    @api.response(200, "Workflow iteration node run started successfully")
+    @api.response(403, "Permission denied")
+    @api.response(404, "Node not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -256,7 +329,7 @@ class WorkflowDraftRunIterationNodeApi(Resource):
         # The role of the current user in the ta table must be admin, owner, or editor
         if not isinstance(current_user, Account):
             raise Forbidden()
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -280,7 +353,23 @@ class WorkflowDraftRunIterationNodeApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflows/draft/loop/nodes/<string:node_id>/run")
 class AdvancedChatDraftRunLoopNodeApi(Resource):
+    @api.doc("run_advanced_chat_draft_loop_node")
+    @api.doc(description="Run draft workflow loop node for advanced chat")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.expect(
+        api.model(
+            "LoopNodeRunRequest",
+            {
+                "task_id": fields.String(required=True, description="Task ID"),
+                "inputs": fields.Raw(description="Input variables"),
+            },
+        )
+    )
+    @api.response(200, "Loop node run started successfully")
+    @api.response(403, "Permission denied")
+    @api.response(404, "Node not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -293,7 +382,7 @@ class AdvancedChatDraftRunLoopNodeApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -317,7 +406,23 @@ class AdvancedChatDraftRunLoopNodeApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/loop/nodes/<string:node_id>/run")
 class WorkflowDraftRunLoopNodeApi(Resource):
+    @api.doc("run_workflow_draft_loop_node")
+    @api.doc(description="Run draft workflow loop node")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.expect(
+        api.model(
+            "WorkflowLoopNodeRunRequest",
+            {
+                "task_id": fields.String(required=True, description="Task ID"),
+                "inputs": fields.Raw(description="Input variables"),
+            },
+        )
+    )
+    @api.response(200, "Workflow loop node run started successfully")
+    @api.response(403, "Permission denied")
+    @api.response(404, "Node not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -330,7 +435,7 @@ class WorkflowDraftRunLoopNodeApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -354,7 +459,22 @@ class WorkflowDraftRunLoopNodeApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/run")
 class DraftWorkflowRunApi(Resource):
+    @api.doc("run_draft_workflow")
+    @api.doc(description="Run draft workflow")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "DraftWorkflowRunRequest",
+            {
+                "inputs": fields.Raw(required=True, description="Input variables"),
+                "files": fields.List(fields.Raw, description="File uploads"),
+            },
+        )
+    )
+    @api.response(200, "Draft workflow run started successfully")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -367,7 +487,7 @@ class DraftWorkflowRunApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -393,7 +513,14 @@ class DraftWorkflowRunApi(Resource):
             raise InvokeRateLimitHttpError(ex.description)
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop")
 class WorkflowTaskStopApi(Resource):
+    @api.doc("stop_workflow_task")
+    @api.doc(description="Stop running workflow task")
+    @api.doc(params={"app_id": "Application ID", "task_id": "Task ID"})
+    @api.response(200, "Task stopped successfully")
+    @api.response(404, "Task not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -406,15 +533,35 @@ class WorkflowTaskStopApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
-        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
+        # Stop using both mechanisms for backward compatibility
+        # Legacy stop flag mechanism (without user check)
+        AppQueueManager.set_stop_flag_no_user_check(task_id)
+
+        # New graph engine command channel mechanism
+        GraphEngineManager.send_stop_command(task_id)
 
         return {"result": "success"}
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/run")
 class DraftWorkflowNodeRunApi(Resource):
+    @api.doc("run_draft_workflow_node")
+    @api.doc(description="Run draft workflow node")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.expect(
+        api.model(
+            "DraftWorkflowNodeRunRequest",
+            {
+                "inputs": fields.Raw(description="Input variables"),
+            },
+        )
+    )
+    @api.response(200, "Node run started successfully", workflow_run_node_execution_fields)
+    @api.response(403, "Permission denied")
+    @api.response(404, "Node not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -428,7 +575,7 @@ class DraftWorkflowNodeRunApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -462,7 +609,13 @@ class DraftWorkflowNodeRunApi(Resource):
         return workflow_node_execution
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/publish")
 class PublishedWorkflowApi(Resource):
+    @api.doc("get_published_workflow")
+    @api.doc(description="Get published workflow for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Published workflow retrieved successfully", workflow_fields)
+    @api.response(404, "Published workflow not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -476,7 +629,7 @@ class PublishedWorkflowApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         # fetch published workflow by app_model
@@ -497,7 +650,7 @@ class PublishedWorkflowApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -534,7 +687,12 @@ class PublishedWorkflowApi(Resource):
         }
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/default-workflow-block-configs")
 class DefaultBlockConfigsApi(Resource):
+    @api.doc("get_default_block_configs")
+    @api.doc(description="Get default block configurations for workflow")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Default block configurations retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -547,7 +705,7 @@ class DefaultBlockConfigsApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         # Get default block configs
@@ -555,7 +713,13 @@ class DefaultBlockConfigsApi(Resource):
         return workflow_service.get_default_block_configs()
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>")
 class DefaultBlockConfigApi(Resource):
+    @api.doc("get_default_block_config")
+    @api.doc(description="Get default block configuration by type")
+    @api.doc(params={"app_id": "Application ID", "block_type": "Block type"})
+    @api.response(200, "Default block configuration retrieved successfully")
+    @api.response(404, "Block type not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -567,7 +731,7 @@ class DefaultBlockConfigApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -588,7 +752,14 @@ class DefaultBlockConfigApi(Resource):
         return workflow_service.get_default_block_config(node_type=block_type, filters=filters)
 
 
+@console_ns.route("/apps/<uuid:app_id>/convert-to-workflow")
 class ConvertToWorkflowApi(Resource):
+    @api.doc("convert_to_workflow")
+    @api.doc(description="Convert application to workflow mode")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Application converted to workflow successfully")
+    @api.response(400, "Application cannot be converted")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -602,7 +773,7 @@ class ConvertToWorkflowApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         if request.data:
@@ -625,20 +796,12 @@ class ConvertToWorkflowApi(Resource):
         }
 
 
-class WorkflowConfigApi(Resource):
-    """Resource for workflow configuration."""
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
-    def get(self, app_model: App):
-        return {
-            "parallel_depth_limit": dify_config.WORKFLOW_PARALLEL_DEPTH_LIMIT,
-        }
-
-
+@console_ns.route("/apps/<uuid:app_id>/workflows")
 class PublishedAllWorkflowApi(Resource):
+    @api.doc("get_all_published_workflows")
+    @api.doc(description="Get all published workflows for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Published workflows retrieved successfully", workflow_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -651,7 +814,7 @@ class PublishedAllWorkflowApi(Resource):
 
         if not isinstance(current_user, Account):
             raise Forbidden()
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -689,7 +852,23 @@ class PublishedAllWorkflowApi(Resource):
             }
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/<string:workflow_id>")
 class WorkflowByIdApi(Resource):
+    @api.doc("update_workflow_by_id")
+    @api.doc(description="Update workflow by ID")
+    @api.doc(params={"app_id": "Application ID", "workflow_id": "Workflow ID"})
+    @api.expect(
+        api.model(
+            "UpdateWorkflowRequest",
+            {
+                "environment_variables": fields.List(fields.Raw, description="Environment variables"),
+                "conversation_variables": fields.List(fields.Raw, description="Conversation variables"),
+            },
+        )
+    )
+    @api.response(200, "Workflow updated successfully", workflow_fields)
+    @api.response(404, "Workflow not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -702,7 +881,7 @@ class WorkflowByIdApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # Check permission
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -715,7 +894,6 @@ class WorkflowByIdApi(Resource):
             raise ValueError("Marked name cannot exceed 20 characters")
         if args.marked_comment and len(args.marked_comment) > 100:
             raise ValueError("Marked comment cannot exceed 100 characters")
-        args = parser.parse_args()
 
         # Prepare update data
         update_data = {}
@@ -758,7 +936,7 @@ class WorkflowByIdApi(Resource):
         if not isinstance(current_user, Account):
             raise Forbidden()
         # Check permission
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         workflow_service = WorkflowService()
@@ -781,7 +959,14 @@ class WorkflowByIdApi(Resource):
         return None, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/last-run")
 class DraftWorkflowNodeLastRunApi(Resource):
+    @api.doc("get_draft_workflow_node_last_run")
+    @api.doc(description="Get last run result for draft workflow node")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.response(200, "Node last run retrieved successfully", workflow_run_node_execution_fields)
+    @api.response(404, "Node last run not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -800,73 +985,3 @@ class DraftWorkflowNodeLastRunApi(Resource):
         if node_exec is None:
             raise NotFound("last run not found")
         return node_exec
-
-
-api.add_resource(
-    DraftWorkflowApi,
-    "/apps/<uuid:app_id>/workflows/draft",
-)
-api.add_resource(
-    WorkflowConfigApi,
-    "/apps/<uuid:app_id>/workflows/draft/config",
-)
-api.add_resource(
-    AdvancedChatDraftWorkflowRunApi,
-    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/run",
-)
-api.add_resource(
-    DraftWorkflowRunApi,
-    "/apps/<uuid:app_id>/workflows/draft/run",
-)
-api.add_resource(
-    WorkflowTaskStopApi,
-    "/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop",
-)
-api.add_resource(
-    DraftWorkflowNodeRunApi,
-    "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/run",
-)
-api.add_resource(
-    AdvancedChatDraftRunIterationNodeApi,
-    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/iteration/nodes/<string:node_id>/run",
-)
-api.add_resource(
-    WorkflowDraftRunIterationNodeApi,
-    "/apps/<uuid:app_id>/workflows/draft/iteration/nodes/<string:node_id>/run",
-)
-api.add_resource(
-    AdvancedChatDraftRunLoopNodeApi,
-    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/loop/nodes/<string:node_id>/run",
-)
-api.add_resource(
-    WorkflowDraftRunLoopNodeApi,
-    "/apps/<uuid:app_id>/workflows/draft/loop/nodes/<string:node_id>/run",
-)
-api.add_resource(
-    PublishedWorkflowApi,
-    "/apps/<uuid:app_id>/workflows/publish",
-)
-api.add_resource(
-    PublishedAllWorkflowApi,
-    "/apps/<uuid:app_id>/workflows",
-)
-api.add_resource(
-    DefaultBlockConfigsApi,
-    "/apps/<uuid:app_id>/workflows/default-workflow-block-configs",
-)
-api.add_resource(
-    DefaultBlockConfigApi,
-    "/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>",
-)
-api.add_resource(
-    ConvertToWorkflowApi,
-    "/apps/<uuid:app_id>/convert-to-workflow",
-)
-api.add_resource(
-    WorkflowByIdApi,
-    "/apps/<uuid:app_id>/workflows/<string:workflow_id>",
-)
-api.add_resource(
-    DraftWorkflowNodeLastRunApi,
-    "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/last-run",
-)

api/controllers/console/app/workflow_app_log.py

@@ -3,10 +3,10 @@ from flask_restx import Resource, marshal_with, reqparse
 from flask_restx.inputs import int_range
 from sqlalchemy.orm import Session
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
-from core.workflow.entities.workflow_execution import WorkflowExecutionStatus
+from core.workflow.enums import WorkflowExecutionStatus
 from extensions.ext_database import db
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
@@ -15,7 +15,24 @@ from models.model import AppMode
 from services.workflow_app_service import WorkflowAppService
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow-app-logs")
 class WorkflowAppLogApi(Resource):
+    @api.doc("get_workflow_app_logs")
+    @api.doc(description="Get workflow application execution logs")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(
+        params={
+            "keyword": "Search keyword for filtering logs",
+            "status": "Filter by execution status (succeeded, failed, stopped, partial-succeeded)",
+            "created_at__before": "Filter logs created before this timestamp",
+            "created_at__after": "Filter logs created after this timestamp",
+            "created_by_end_user_session_id": "Filter by end user session ID",
+            "created_by_account": "Filter by account",
+            "page": "Page number (1-99999)",
+            "limit": "Number of items per page (1-100)",
+        }
+    )
+    @api.response(200, "Workflow app logs retrieved successfully", workflow_app_log_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -78,6 +95,3 @@ class WorkflowAppLogApi(Resource):
             )
 
             return workflow_app_log_pagination
-
-
-api.add_resource(WorkflowAppLogApi, "/apps/<uuid:app_id>/workflow-app-logs")

api/controllers/console/app/workflow_draft_variable.py

@@ -6,21 +6,23 @@ from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqpars
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     DraftWorkflowNotExist,
 )
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from controllers.web.error import InvalidArgumentError, NotFoundError
+from core.file import helpers as file_helpers
 from core.variables.segment_group import SegmentGroup
 from core.variables.segments import ArrayFileSegment, FileSegment, Segment
 from core.variables.types import SegmentType
 from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
+from extensions.ext_database import db
 from factories.file_factory import build_from_mapping, build_from_mappings
 from factories.variable_factory import build_segment_with_type
 from libs.login import current_user, login_required
-from models import App, AppMode, db
+from models import App, AppMode
 from models.account import Account
 from models.workflow import WorkflowDraftVariable
 from services.workflow_draft_variable_service import WorkflowDraftVariableList, WorkflowDraftVariableService
@@ -74,6 +76,22 @@ def _serialize_variable_type(workflow_draft_var: WorkflowDraftVariable) -> str:
     return value_type.exposed_type().value
 
 
+def _serialize_full_content(variable: WorkflowDraftVariable) -> dict | None:
+    """Serialize full_content information for large variables."""
+    if not variable.is_truncated():
+        return None
+
+    variable_file = variable.variable_file
+    assert variable_file is not None
+
+    return {
+        "size_bytes": variable_file.size,
+        "value_type": variable_file.value_type.exposed_type().value,
+        "length": variable_file.length,
+        "download_url": file_helpers.get_signed_file_url(variable_file.upload_file_id, as_attachment=True),
+    }
+
+
 _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS = {
     "id": fields.String,
     "type": fields.String(attribute=lambda model: model.get_variable_type()),
@@ -83,11 +101,13 @@ _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS = {
     "value_type": fields.String(attribute=_serialize_variable_type),
     "edited": fields.Boolean(attribute=lambda model: model.edited),
     "visible": fields.Boolean,
+    "is_truncated": fields.Boolean(attribute=lambda model: model.file_id is not None),
 }
 
 _WORKFLOW_DRAFT_VARIABLE_FIELDS = dict(
     _WORKFLOW_DRAFT_VARIABLE_WITHOUT_VALUE_FIELDS,
     value=fields.Raw(attribute=_serialize_var_value),
+    full_content=fields.Raw(attribute=_serialize_full_content),
 )
 
 _WORKFLOW_DRAFT_ENV_VARIABLE_FIELDS = {
@@ -137,14 +157,20 @@ def _api_prerequisite(f):
     @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
     def wrapper(*args, **kwargs):
         assert isinstance(current_user, Account)
-        if not current_user.is_editor:
+        if not current_user.has_edit_permission:
             raise Forbidden()
         return f(*args, **kwargs)
 
     return wrapper
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/variables")
 class WorkflowVariableCollectionApi(Resource):
+    @api.doc("get_workflow_variables")
+    @api.doc(description="Get draft workflow variables")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"page": "Page number (1-100000)", "limit": "Number of items per page (1-100)"})
+    @api.response(200, "Workflow variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS)
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_WITHOUT_VALUE_FIELDS)
     def get(self, app_model: App):
@@ -173,6 +199,9 @@ class WorkflowVariableCollectionApi(Resource):
 
         return workflow_vars
 
+    @api.doc("delete_workflow_variables")
+    @api.doc(description="Delete all draft workflow variables")
+    @api.response(204, "Workflow variables deleted successfully")
     @_api_prerequisite
     def delete(self, app_model: App):
         draft_var_srv = WorkflowDraftVariableService(
@@ -201,7 +230,12 @@ def validate_node_id(node_id: str) -> NoReturn | None:
     return None
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/variables")
 class NodeVariableCollectionApi(Resource):
+    @api.doc("get_node_variables")
+    @api.doc(description="Get variables for a specific node")
+    @api.doc(params={"app_id": "Application ID", "node_id": "Node ID"})
+    @api.response(200, "Node variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
     def get(self, app_model: App, node_id: str):
@@ -214,6 +248,9 @@ class NodeVariableCollectionApi(Resource):
 
         return node_vars
 
+    @api.doc("delete_node_variables")
+    @api.doc(description="Delete all variables for a specific node")
+    @api.response(204, "Node variables deleted successfully")
     @_api_prerequisite
     def delete(self, app_model: App, node_id: str):
         validate_node_id(node_id)
@@ -223,10 +260,16 @@ class NodeVariableCollectionApi(Resource):
         return Response("", 204)
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>")
 class VariableApi(Resource):
     _PATCH_NAME_FIELD = "name"
     _PATCH_VALUE_FIELD = "value"
 
+    @api.doc("get_variable")
+    @api.doc(description="Get a specific workflow variable")
+    @api.doc(params={"app_id": "Application ID", "variable_id": "Variable ID"})
+    @api.response(200, "Variable retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS)
+    @api.response(404, "Variable not found")
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS)
     def get(self, app_model: App, variable_id: str):
@@ -240,6 +283,19 @@ class VariableApi(Resource):
             raise NotFoundError(description=f"variable not found, id={variable_id}")
         return variable
 
+    @api.doc("update_variable")
+    @api.doc(description="Update a workflow variable")
+    @api.expect(
+        api.model(
+            "UpdateVariableRequest",
+            {
+                "name": fields.String(description="Variable name"),
+                "value": fields.Raw(description="Variable value"),
+            },
+        )
+    )
+    @api.response(200, "Variable updated successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS)
+    @api.response(404, "Variable not found")
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_FIELDS)
     def patch(self, app_model: App, variable_id: str):
@@ -302,6 +358,10 @@ class VariableApi(Resource):
         db.session.commit()
         return variable
 
+    @api.doc("delete_variable")
+    @api.doc(description="Delete a workflow variable")
+    @api.response(204, "Variable deleted successfully")
+    @api.response(404, "Variable not found")
     @_api_prerequisite
     def delete(self, app_model: App, variable_id: str):
         draft_var_srv = WorkflowDraftVariableService(
@@ -317,7 +377,14 @@ class VariableApi(Resource):
         return Response("", 204)
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>/reset")
 class VariableResetApi(Resource):
+    @api.doc("reset_variable")
+    @api.doc(description="Reset a workflow variable to its default value")
+    @api.doc(params={"app_id": "Application ID", "variable_id": "Variable ID"})
+    @api.response(200, "Variable reset successfully", _WORKFLOW_DRAFT_VARIABLE_FIELDS)
+    @api.response(204, "Variable reset (no content)")
+    @api.response(404, "Variable not found")
     @_api_prerequisite
     def put(self, app_model: App, variable_id: str):
         draft_var_srv = WorkflowDraftVariableService(
@@ -358,7 +425,13 @@ def _get_variable_list(app_model: App, node_id) -> WorkflowDraftVariableList:
     return draft_vars
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/conversation-variables")
 class ConversationVariableCollectionApi(Resource):
+    @api.doc("get_conversation_variables")
+    @api.doc(description="Get conversation variables for workflow")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Conversation variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
+    @api.response(404, "Draft workflow not found")
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
     def get(self, app_model: App):
@@ -374,14 +447,25 @@ class ConversationVariableCollectionApi(Resource):
         return _get_variable_list(app_model, CONVERSATION_VARIABLE_NODE_ID)
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/system-variables")
 class SystemVariableCollectionApi(Resource):
+    @api.doc("get_system_variables")
+    @api.doc(description="Get system variables for workflow")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "System variables retrieved successfully", _WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
     @_api_prerequisite
     @marshal_with(_WORKFLOW_DRAFT_VARIABLE_LIST_FIELDS)
     def get(self, app_model: App):
         return _get_variable_list(app_model, SYSTEM_VARIABLE_NODE_ID)
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/environment-variables")
 class EnvironmentVariableCollectionApi(Resource):
+    @api.doc("get_environment_variables")
+    @api.doc(description="Get environment variables for workflow")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Environment variables retrieved successfully")
+    @api.response(404, "Draft workflow not found")
     @_api_prerequisite
     def get(self, app_model: App):
         """
@@ -413,16 +497,3 @@ class EnvironmentVariableCollectionApi(Resource):
             )
 
         return {"items": env_vars_list}
-
-
-api.add_resource(
-    WorkflowVariableCollectionApi,
-    "/apps/<uuid:app_id>/workflows/draft/variables",
-)
-api.add_resource(NodeVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/variables")
-api.add_resource(VariableApi, "/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>")
-api.add_resource(VariableResetApi, "/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>/reset")
-
-api.add_resource(ConversationVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/conversation-variables")
-api.add_resource(SystemVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/system-variables")
-api.add_resource(EnvironmentVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/environment-variables")

api/controllers/console/app/workflow_run.py

@@ -4,7 +4,7 @@ from flask_login import current_user
 from flask_restx import Resource, marshal_with, reqparse
 from flask_restx.inputs import int_range
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from fields.workflow_run_fields import (
@@ -19,7 +19,13 @@ from models import Account, App, AppMode, EndUser
 from services.workflow_run_service import WorkflowRunService
 
 
+@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflow-runs")
 class AdvancedChatAppWorkflowRunListApi(Resource):
+    @api.doc("get_advanced_chat_workflow_runs")
+    @api.doc(description="Get advanced chat workflow run list")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"last_id": "Last run ID for pagination", "limit": "Number of items per page (1-100)"})
+    @api.response(200, "Workflow runs retrieved successfully", advanced_chat_workflow_run_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -40,7 +46,13 @@ class AdvancedChatAppWorkflowRunListApi(Resource):
         return result
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow-runs")
 class WorkflowRunListApi(Resource):
+    @api.doc("get_workflow_runs")
+    @api.doc(description="Get workflow run list")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"last_id": "Last run ID for pagination", "limit": "Number of items per page (1-100)"})
+    @api.response(200, "Workflow runs retrieved successfully", workflow_run_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -61,7 +73,13 @@ class WorkflowRunListApi(Resource):
         return result
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>")
 class WorkflowRunDetailApi(Resource):
+    @api.doc("get_workflow_run_detail")
+    @api.doc(description="Get workflow run detail")
+    @api.doc(params={"app_id": "Application ID", "run_id": "Workflow run ID"})
+    @api.response(200, "Workflow run detail retrieved successfully", workflow_run_detail_fields)
+    @api.response(404, "Workflow run not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -79,7 +97,13 @@ class WorkflowRunDetailApi(Resource):
         return workflow_run
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>/node-executions")
 class WorkflowRunNodeExecutionListApi(Resource):
+    @api.doc("get_workflow_run_node_executions")
+    @api.doc(description="Get workflow run node execution list")
+    @api.doc(params={"app_id": "Application ID", "run_id": "Workflow run ID"})
+    @api.response(200, "Node executions retrieved successfully", workflow_run_node_execution_list_fields)
+    @api.response(404, "Workflow run not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -100,9 +124,3 @@ class WorkflowRunNodeExecutionListApi(Resource):
         )
 
         return {"data": node_executions}
-
-
-api.add_resource(AdvancedChatAppWorkflowRunListApi, "/apps/<uuid:app_id>/advanced-chat/workflow-runs")
-api.add_resource(WorkflowRunListApi, "/apps/<uuid:app_id>/workflow-runs")
-api.add_resource(WorkflowRunDetailApi, "/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>")
-api.add_resource(WorkflowRunNodeExecutionListApi, "/apps/<uuid:app_id>/workflow-runs/<uuid:run_id>/node-executions")

api/controllers/console/app/workflow_statistic.py

@@ -7,7 +7,7 @@ from flask import jsonify
 from flask_login import current_user
 from flask_restx import Resource, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -17,7 +17,13 @@ from models.enums import WorkflowRunTriggeredFrom
 from models.model import AppMode
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow/statistics/daily-conversations")
 class WorkflowDailyRunsStatistic(Resource):
+    @api.doc("get_workflow_daily_runs_statistic")
+    @api.doc(description="Get workflow daily runs statistics")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"start": "Start date and time (YYYY-MM-DD HH:MM)", "end": "End date and time (YYYY-MM-DD HH:MM)"})
+    @api.response(200, "Daily runs statistics retrieved successfully")
     @get_app_model
     @setup_required
     @login_required
@@ -79,7 +85,13 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow/statistics/daily-terminals")
 class WorkflowDailyTerminalsStatistic(Resource):
+    @api.doc("get_workflow_daily_terminals_statistic")
+    @api.doc(description="Get workflow daily terminals statistics")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"start": "Start date and time (YYYY-MM-DD HH:MM)", "end": "End date and time (YYYY-MM-DD HH:MM)"})
+    @api.response(200, "Daily terminals statistics retrieved successfully")
     @get_app_model
     @setup_required
     @login_required
@@ -141,7 +153,13 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow/statistics/token-costs")
 class WorkflowDailyTokenCostStatistic(Resource):
+    @api.doc("get_workflow_daily_token_cost_statistic")
+    @api.doc(description="Get workflow daily token cost statistics")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"start": "Start date and time (YYYY-MM-DD HH:MM)", "end": "End date and time (YYYY-MM-DD HH:MM)"})
+    @api.response(200, "Daily token cost statistics retrieved successfully")
     @get_app_model
     @setup_required
     @login_required
@@ -208,7 +226,13 @@ WHERE
         return jsonify({"data": response_data})
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflow/statistics/average-app-interactions")
 class WorkflowAverageAppInteractionStatistic(Resource):
+    @api.doc("get_workflow_average_app_interaction_statistic")
+    @api.doc(description="Get workflow average app interaction statistics")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.doc(params={"start": "Start date and time (YYYY-MM-DD HH:MM)", "end": "End date and time (YYYY-MM-DD HH:MM)"})
+    @api.response(200, "Average app interaction statistics retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -285,11 +309,3 @@ GROUP BY
                 )
 
         return jsonify({"data": response_data})
-
-
-api.add_resource(WorkflowDailyRunsStatistic, "/apps/<uuid:app_id>/workflow/statistics/daily-conversations")
-api.add_resource(WorkflowDailyTerminalsStatistic, "/apps/<uuid:app_id>/workflow/statistics/daily-terminals")
-api.add_resource(WorkflowDailyTokenCostStatistic, "/apps/<uuid:app_id>/workflow/statistics/token-costs")
-api.add_resource(
-    WorkflowAverageAppInteractionStatistic, "/apps/<uuid:app_id>/workflow/statistics/average-app-interactions"
-)

api/controllers/console/app/wraps.py

@@ -1,6 +1,6 @@
 from collections.abc import Callable
 from functools import wraps
-from typing import Optional, ParamSpec, TypeVar, Union
+from typing import ParamSpec, TypeVar, Union
 
 from controllers.console.app.error import AppNotFoundError
 from extensions.ext_database import db
@@ -12,7 +12,7 @@ P = ParamSpec("P")
 R = TypeVar("R")
 
 
-def _load_app_model(app_id: str) -> Optional[App]:
+def _load_app_model(app_id: str) -> App | None:
     assert isinstance(current_user, Account)
     app_model = (
         db.session.query(App)
@@ -22,7 +22,7 @@ def _load_app_model(app_id: str) -> Optional[App]:
     return app_model
 
 
-def get_app_model(view: Optional[Callable[P, R]] = None, *, mode: Union[AppMode, list[AppMode], None] = None):
+def get_app_model(view: Callable[P, R] | None = None, *, mode: Union[AppMode, list[AppMode], None] = None):
     def decorator(view_func: Callable[P, R]):
         @wraps(view_func)
         def decorated_view(*args: P.args, **kwargs: P.kwargs):

api/controllers/console/auth/data_source_oauth.py

@@ -1,6 +1,6 @@
 import logging
 
-import requests
+import httpx
 from flask import current_app, redirect, request
 from flask_login import current_user
 from flask_restx import Resource, fields
@@ -119,7 +119,7 @@ class OAuthDataSourceBinding(Resource):
                 return {"error": "Invalid code"}, 400
             try:
                 oauth_provider.get_access_token(code)
-            except requests.HTTPError as e:
+            except httpx.HTTPStatusError as e:
                 logger.exception(
                     "An error occurred during the OAuthCallback process with %s: %s", provider, e.response.text
                 )
@@ -152,7 +152,7 @@ class OAuthDataSourceSync(Resource):
             return {"error": "Invalid provider"}, 400
         try:
             oauth_provider.sync_data_source(binding_id)
-        except requests.HTTPError as e:
+        except httpx.HTTPStatusError as e:
             logger.exception(
                 "An error occurred during the OAuthCallback process with %s: %s", provider, e.response.text
             )

api/controllers/console/auth/email_register.py

@@ -0,0 +1,155 @@
+from flask import request
+from flask_restx import Resource, reqparse
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from configs import dify_config
+from constants.languages import languages
+from controllers.console import api
+from controllers.console.auth.error import (
+    EmailAlreadyInUseError,
+    EmailCodeError,
+    EmailRegisterLimitError,
+    InvalidEmailError,
+    InvalidTokenError,
+    PasswordMismatchError,
+)
+from controllers.console.error import AccountInFreezeError, EmailSendIpLimitError
+from controllers.console.wraps import email_password_login_enabled, email_register_enabled, setup_required
+from extensions.ext_database import db
+from libs.helper import email, extract_remote_ip
+from libs.password import valid_password
+from models.account import Account
+from services.account_service import AccountService
+from services.billing_service import BillingService
+from services.errors.account import AccountNotFoundError, AccountRegisterError
+
+
+class EmailRegisterSendEmailApi(Resource):
+    @setup_required
+    @email_password_login_enabled
+    @email_register_enabled
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=email, required=True, location="json")
+        parser.add_argument("language", type=str, required=False, location="json")
+        args = parser.parse_args()
+
+        ip_address = extract_remote_ip(request)
+        if AccountService.is_email_send_ip_limit(ip_address):
+            raise EmailSendIpLimitError()
+        language = "en-US"
+        if args["language"] in languages:
+            language = args["language"]
+
+        if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(args["email"]):
+            raise AccountInFreezeError()
+
+        with Session(db.engine) as session:
+            account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none()
+        token = None
+        token = AccountService.send_email_register_email(email=args["email"], account=account, language=language)
+        return {"result": "success", "data": token}
+
+
+class EmailRegisterCheckApi(Resource):
+    @setup_required
+    @email_password_login_enabled
+    @email_register_enabled
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=str, required=True, location="json")
+        parser.add_argument("code", type=str, required=True, location="json")
+        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+
+        user_email = args["email"]
+
+        is_email_register_error_rate_limit = AccountService.is_email_register_error_rate_limit(args["email"])
+        if is_email_register_error_rate_limit:
+            raise EmailRegisterLimitError()
+
+        token_data = AccountService.get_email_register_data(args["token"])
+        if token_data is None:
+            raise InvalidTokenError()
+
+        if user_email != token_data.get("email"):
+            raise InvalidEmailError()
+
+        if args["code"] != token_data.get("code"):
+            AccountService.add_email_register_error_rate_limit(args["email"])
+            raise EmailCodeError()
+
+        # Verified, revoke the first token
+        AccountService.revoke_email_register_token(args["token"])
+
+        # Refresh token data by generating a new token
+        _, new_token = AccountService.generate_email_register_token(
+            user_email, code=args["code"], additional_data={"phase": "register"}
+        )
+
+        AccountService.reset_email_register_error_rate_limit(args["email"])
+        return {"is_valid": True, "email": token_data.get("email"), "token": new_token}
+
+
+class EmailRegisterResetApi(Resource):
+    @setup_required
+    @email_password_login_enabled
+    @email_register_enabled
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("token", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("new_password", type=valid_password, required=True, nullable=False, location="json")
+        parser.add_argument("password_confirm", type=valid_password, required=True, nullable=False, location="json")
+        args = parser.parse_args()
+
+        # Validate passwords match
+        if args["new_password"] != args["password_confirm"]:
+            raise PasswordMismatchError()
+
+        # Validate token and get register data
+        register_data = AccountService.get_email_register_data(args["token"])
+        if not register_data:
+            raise InvalidTokenError()
+        # Must use token in reset phase
+        if register_data.get("phase", "") != "register":
+            raise InvalidTokenError()
+
+        # Revoke token to prevent reuse
+        AccountService.revoke_email_register_token(args["token"])
+
+        email = register_data.get("email", "")
+
+        with Session(db.engine) as session:
+            account = session.execute(select(Account).filter_by(email=email)).scalar_one_or_none()
+
+            if account:
+                raise EmailAlreadyInUseError()
+            else:
+                account = self._create_new_account(email, args["password_confirm"])
+                if not account:
+                    raise AccountNotFoundError()
+                token_pair = AccountService.login(account=account, ip_address=extract_remote_ip(request))
+                AccountService.reset_login_error_rate_limit(email)
+
+        return {"result": "success", "data": token_pair.model_dump()}
+
+    def _create_new_account(self, email, password) -> Account | None:
+        # Create new account if allowed
+        account = None
+        try:
+            account = AccountService.create_account_and_tenant(
+                email=email,
+                name=email,
+                password=password,
+                interface_language=languages[0],
+            )
+        except AccountRegisterError:
+            raise AccountInFreezeError()
+
+        return account
+
+
+api.add_resource(EmailRegisterSendEmailApi, "/email-register/send-email")
+api.add_resource(EmailRegisterCheckApi, "/email-register/validity")
+api.add_resource(EmailRegisterResetApi, "/email-register")

api/controllers/console/auth/error.py

@@ -27,21 +27,43 @@ class InvalidTokenError(BaseHTTPException):
 
 class PasswordResetRateLimitExceededError(BaseHTTPException):
     error_code = "password_reset_rate_limit_exceeded"
-    description = "Too many password reset emails have been sent. Please try again in 1 minute."
+    description = "Too many password reset emails have been sent. Please try again in {minutes} minutes."
     code = 429
 
+    def __init__(self, minutes: int = 1):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
+
+class EmailRegisterRateLimitExceededError(BaseHTTPException):
+    error_code = "email_register_rate_limit_exceeded"
+    description = "Too many email register emails have been sent. Please try again in {minutes} minutes."
+    code = 429
+
+    def __init__(self, minutes: int = 1):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
 
 class EmailChangeRateLimitExceededError(BaseHTTPException):
     error_code = "email_change_rate_limit_exceeded"
-    description = "Too many email change emails have been sent. Please try again in 1 minute."
+    description = "Too many email change emails have been sent. Please try again in {minutes} minutes."
     code = 429
 
+    def __init__(self, minutes: int = 1):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
 
 class OwnerTransferRateLimitExceededError(BaseHTTPException):
     error_code = "owner_transfer_rate_limit_exceeded"
-    description = "Too many owner transfer emails have been sent. Please try again in 1 minute."
+    description = "Too many owner transfer emails have been sent. Please try again in {minutes} minutes."
     code = 429
 
+    def __init__(self, minutes: int = 1):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
 
 class EmailCodeError(BaseHTTPException):
     error_code = "email_code_error"
@@ -69,15 +91,23 @@ class EmailPasswordLoginLimitError(BaseHTTPException):
 
 class EmailCodeLoginRateLimitExceededError(BaseHTTPException):
     error_code = "email_code_login_rate_limit_exceeded"
-    description = "Too many login emails have been sent. Please try again in 5 minutes."
+    description = "Too many login emails have been sent. Please try again in {minutes} minutes."
     code = 429
 
+    def __init__(self, minutes: int = 5):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
 
 class EmailCodeAccountDeletionRateLimitExceededError(BaseHTTPException):
     error_code = "email_code_account_deletion_rate_limit_exceeded"
-    description = "Too many account deletion emails have been sent. Please try again in 5 minutes."
+    description = "Too many account deletion emails have been sent. Please try again in {minutes} minutes."
     code = 429
 
+    def __init__(self, minutes: int = 5):
+        description = self.description.format(minutes=int(minutes)) if self.description else None
+        super().__init__(description=description)
+
 
 class EmailPasswordResetLimitError(BaseHTTPException):
     error_code = "email_password_reset_limit"
@@ -85,6 +115,12 @@ class EmailPasswordResetLimitError(BaseHTTPException):
     code = 429
 
 
+class EmailRegisterLimitError(BaseHTTPException):
+    error_code = "email_register_limit"
+    description = "Too many failed email register attempts. Please try again in 24 hours."
+    code = 429
+
+
 class EmailChangeLimitError(BaseHTTPException):
     error_code = "email_change_limit"
     description = "Too many failed email change attempts. Please try again in 24 hours."

api/controllers/console/auth/forgot_password.py

@@ -6,7 +6,6 @@ from flask_restx import Resource, fields, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from constants.languages import languages
 from controllers.console import api, console_ns
 from controllers.console.auth.error import (
     EmailCodeError,
@@ -15,7 +14,7 @@ from controllers.console.auth.error import (
     InvalidTokenError,
     PasswordMismatchError,
 )
-from controllers.console.error import AccountInFreezeError, AccountNotFound, EmailSendIpLimitError
+from controllers.console.error import AccountNotFound, EmailSendIpLimitError
 from controllers.console.wraps import email_password_login_enabled, setup_required
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
@@ -23,8 +22,6 @@ from libs.helper import email, extract_remote_ip
 from libs.password import hash_password, valid_password
 from models.account import Account
 from services.account_service import AccountService, TenantService
-from services.errors.account import AccountRegisterError
-from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkspacesLimitExceededError
 from services.feature_service import FeatureService
 
 
@@ -73,15 +70,13 @@ class ForgotPasswordSendEmailApi(Resource):
 
         with Session(db.engine) as session:
             account = session.execute(select(Account).filter_by(email=args["email"])).scalar_one_or_none()
-        token = None
-        if account is None:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-                return {"result": "fail", "data": token, "code": "account_not_found"}
-            else:
-                raise AccountNotFound()
-        else:
-            token = AccountService.send_reset_password_email(account=account, email=args["email"], language=language)
+
+        token = AccountService.send_reset_password_email(
+            account=account,
+            email=args["email"],
+            language=language,
+            is_allow_register=FeatureService.get_system_features().is_allow_register,
+        )
 
         return {"result": "success", "data": token}
 
@@ -207,7 +202,7 @@ class ForgotPasswordResetApi(Resource):
             if account:
                 self._update_existing_account(account, password_hashed, salt, session)
             else:
-                self._create_new_account(email, args["password_confirm"])
+                raise AccountNotFound()
 
         return {"result": "success"}
 
@@ -227,18 +222,7 @@ class ForgotPasswordResetApi(Resource):
             account.current_tenant = tenant
             tenant_was_created.send(tenant)
 
-    def _create_new_account(self, email, password):
-        # Create new account if allowed
-        try:
-            AccountService.create_account_and_tenant(
-                email=email,
-                name=email,
-                password=password,
-                interface_language=languages[0],
-            )
-        except WorkSpaceNotAllowedCreateError:
-            pass
-        except WorkspacesLimitExceededError:
-            pass
-        except AccountRegisterError:
-            raise AccountInFreezeError()
+
+api.add_resource(ForgotPasswordSendEmailApi, "/forgot-password")
+api.add_resource(ForgotPasswordCheckApi, "/forgot-password/validity")
+api.add_resource(ForgotPasswordResetApi, "/forgot-password/resets")

api/controllers/console/auth/login.py

@@ -26,7 +26,6 @@ from controllers.console.error import (
 from controllers.console.wraps import email_password_login_enabled, setup_required
 from events.tenant_event import tenant_was_created
 from libs.helper import email, extract_remote_ip
-from libs.password import valid_password
 from models.account import Account
 from services.account_service import AccountService, RegisterService, TenantService
 from services.billing_service import BillingService
@@ -44,10 +43,9 @@ class LoginApi(Resource):
         """Authenticate user and login."""
         parser = reqparse.RequestParser()
         parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("password", type=valid_password, required=True, location="json")
+        parser.add_argument("password", type=str, required=True, location="json")
         parser.add_argument("remember_me", type=bool, required=False, default=False, location="json")
         parser.add_argument("invite_token", type=str, required=False, default=None, location="json")
-        parser.add_argument("language", type=str, required=False, default="en-US", location="json")
         args = parser.parse_args()
 
         if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(args["email"]):
@@ -61,11 +59,6 @@ class LoginApi(Resource):
         if invitation:
             invitation = RegisterService.get_invitation_if_token_valid(None, args["email"], invitation)
 
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
-
         try:
             if invitation:
                 data = invitation.get("data", {})
@@ -80,12 +73,6 @@ class LoginApi(Resource):
         except services.errors.account.AccountPasswordError:
             AccountService.add_login_error_rate_limit(args["email"])
             raise AuthenticationFailedError()
-        except services.errors.account.AccountNotFoundError:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-                return {"result": "fail", "data": token, "code": "account_not_found"}
-            else:
-                raise AccountNotFound()
         # SELF_HOSTED only have one workspace
         tenants = TenantService.get_join_tenants(account)
         if len(tenants) == 0:
@@ -133,13 +120,12 @@ class ResetPasswordSendEmailApi(Resource):
         except AccountRegisterError:
             raise AccountInFreezeError()
 
-        if account is None:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-            else:
-                raise AccountNotFound()
-        else:
-            token = AccountService.send_reset_password_email(account=account, language=language)
+        token = AccountService.send_reset_password_email(
+            email=args["email"],
+            account=account,
+            language=language,
+            is_allow_register=FeatureService.get_system_features().is_allow_register,
+        )
 
         return {"result": "success", "data": token}
 

api/controllers/console/auth/oauth.py

@@ -1,7 +1,6 @@
 import logging
-from typing import Optional
 
-import requests
+import httpx
 from flask import current_app, redirect, request
 from flask_restx import Resource
 from sqlalchemy import select
@@ -18,6 +17,7 @@ from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models import Account
 from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
+from services.billing_service import BillingService
 from services.errors.account import AccountNotFoundError, AccountRegisterError
 from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError
 from services.feature_service import FeatureService
@@ -101,8 +101,10 @@ class OAuthCallback(Resource):
         try:
             token = oauth_provider.get_access_token(code)
             user_info = oauth_provider.get_user_info(token)
-        except requests.RequestException as e:
-            error_text = e.response.text if e.response else str(e)
+        except httpx.RequestError as e:
+            error_text = str(e)
+            if isinstance(e, httpx.HTTPStatusError):
+                error_text = e.response.text
             logger.exception("An error occurred during the OAuth process with %s: %s", provider, error_text)
             return {"error": "OAuth process failed"}, 400
 
@@ -156,8 +158,8 @@ class OAuthCallback(Resource):
         )
 
 
-def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:
-    account: Optional[Account] = Account.get_by_openid(provider, user_info.id)
+def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Account | None:
+    account: Account | None = Account.get_by_openid(provider, user_info.id)
 
     if not account:
         with Session(db.engine) as session:
@@ -183,7 +185,15 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
 
     if not account:
         if not FeatureService.get_system_features().is_allow_register:
-            raise AccountNotFoundError()
+            if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(user_info.email):
+                raise AccountRegisterError(
+                    description=(
+                        "This email account has been deleted within the past "
+                        "30 days and is temporarily unavailable for new account registration"
+                    )
+                )
+            else:
+                raise AccountRegisterError(description=("Invalid email or password"))
         account_name = user_info.name or "Dify"
         account = RegisterService.register(
             email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider

api/controllers/console/datasets/data_source.py

@@ -1,4 +1,6 @@
 import json
+from collections.abc import Generator
+from typing import cast
 
 from flask import request
 from flask_login import current_user
@@ -9,6 +11,8 @@ from werkzeug.exceptions import NotFound
 
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required, setup_required
+from core.datasource.entities.datasource_entities import DatasourceProviderType, OnlineDocumentPagesMessage
+from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
 from core.indexing_runner import IndexingRunner
 from core.rag.extractor.entity.datasource_type import DatasourceType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
@@ -19,6 +23,7 @@ from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import DataSourceOauthBinding, Document
 from services.dataset_service import DatasetService, DocumentService
+from services.datasource_provider_service import DatasourceProviderService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
 
 
@@ -111,6 +116,18 @@ class DataSourceNotionListApi(Resource):
     @marshal_with(integrate_notion_info_list_fields)
     def get(self):
         dataset_id = request.args.get("dataset_id", default=None, type=str)
+        credential_id = request.args.get("credential_id", default=None, type=str)
+        if not credential_id:
+            raise ValueError("Credential id is required.")
+        datasource_provider_service = DatasourceProviderService()
+        credential = datasource_provider_service.get_datasource_credentials(
+            tenant_id=current_user.current_tenant_id,
+            credential_id=credential_id,
+            provider="notion_datasource",
+            plugin_id="langgenius/notion_datasource",
+        )
+        if not credential:
+            raise NotFound("Credential not found.")
         exist_page_ids = []
         with Session(db.engine) as session:
             # import notion in the exist dataset
@@ -134,31 +151,49 @@ class DataSourceNotionListApi(Resource):
                         data_source_info = json.loads(document.data_source_info)
                         exist_page_ids.append(data_source_info["notion_page_id"])
             # get all authorized pages
-            data_source_bindings = session.scalars(
-                select(DataSourceOauthBinding).filter_by(
-                    tenant_id=current_user.current_tenant_id, provider="notion", disabled=False
+            from core.datasource.datasource_manager import DatasourceManager
+
+            datasource_runtime = DatasourceManager.get_datasource_runtime(
+                provider_id="langgenius/notion_datasource/notion_datasource",
+                datasource_name="notion_datasource",
+                tenant_id=current_user.current_tenant_id,
+                datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
+            )
+            datasource_provider_service = DatasourceProviderService()
+            if credential:
+                datasource_runtime.runtime.credentials = credential
+            datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
+            online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
+                datasource_runtime.get_online_document_pages(
+                    user_id=current_user.id,
+                    datasource_parameters={},
+                    provider_type=datasource_runtime.datasource_provider_type(),
                 )
-            ).all()
-            if not data_source_bindings:
-                return {"notion_info": []}, 200
-            pre_import_info_list = []
-            for data_source_binding in data_source_bindings:
-                source_info = data_source_binding.source_info
-                pages = source_info["pages"]
-                # Filter out already bound pages
-                for page in pages:
-                    if page["page_id"] in exist_page_ids:
-                        page["is_bound"] = True
-                    else:
-                        page["is_bound"] = False
-                pre_import_info = {
-                    "workspace_name": source_info["workspace_name"],
-                    "workspace_icon": source_info["workspace_icon"],
-                    "workspace_id": source_info["workspace_id"],
-                    "pages": pages,
-                }
-                pre_import_info_list.append(pre_import_info)
-            return {"notion_info": pre_import_info_list}, 200
+            )
+            try:
+                pages = []
+                workspace_info = {}
+                for message in online_document_result:
+                    result = message.result
+                    for info in result:
+                        workspace_info = {
+                            "workspace_id": info.workspace_id,
+                            "workspace_name": info.workspace_name,
+                            "workspace_icon": info.workspace_icon,
+                        }
+                        for page in info.pages:
+                            page_info = {
+                                "page_id": page.page_id,
+                                "page_name": page.page_name,
+                                "type": page.type,
+                                "parent_id": page.parent_id,
+                                "is_bound": page.page_id in exist_page_ids,
+                                "page_icon": page.page_icon,
+                            }
+                            pages.append(page_info)
+            except Exception as e:
+                raise e
+            return {"notion_info": {**workspace_info, "pages": pages}}, 200
 
 
 class DataSourceNotionApi(Resource):
@@ -166,27 +201,25 @@ class DataSourceNotionApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, workspace_id, page_id, page_type):
+        credential_id = request.args.get("credential_id", default=None, type=str)
+        if not credential_id:
+            raise ValueError("Credential id is required.")
+        datasource_provider_service = DatasourceProviderService()
+        credential = datasource_provider_service.get_datasource_credentials(
+            tenant_id=current_user.current_tenant_id,
+            credential_id=credential_id,
+            provider="notion_datasource",
+            plugin_id="langgenius/notion_datasource",
+        )
+
         workspace_id = str(workspace_id)
         page_id = str(page_id)
-        with Session(db.engine) as session:
-            data_source_binding = session.execute(
-                select(DataSourceOauthBinding).where(
-                    db.and_(
-                        DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
-                        DataSourceOauthBinding.provider == "notion",
-                        DataSourceOauthBinding.disabled == False,
-                        DataSourceOauthBinding.source_info["workspace_id"] == f'"{workspace_id}"',
-                    )
-                )
-            ).scalar_one_or_none()
-        if not data_source_binding:
-            raise NotFound("Data source binding not found.")
 
         extractor = NotionExtractor(
             notion_workspace_id=workspace_id,
             notion_obj_id=page_id,
             notion_page_type=page_type,
-            notion_access_token=data_source_binding.access_token,
+            notion_access_token=credential.get("integration_secret"),
             tenant_id=current_user.current_tenant_id,
         )
 
@@ -211,10 +244,12 @@ class DataSourceNotionApi(Resource):
         extract_settings = []
         for notion_info in notion_info_list:
             workspace_id = notion_info["workspace_id"]
+            credential_id = notion_info.get("credential_id")
             for page in notion_info["pages"]:
                 extract_setting = ExtractSetting(
                     datasource_type=DatasourceType.NOTION.value,
                     notion_info={
+                        "credential_id": credential_id,
                         "notion_workspace_id": workspace_id,
                         "notion_obj_id": page["page_id"],
                         "notion_page_type": page["type"],

api/controllers/console/datasets/datasets.py

@@ -1,13 +1,13 @@
 import flask_restx
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal, marshal_with, reqparse
 from sqlalchemy import select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
 from configs import dify_config
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
@@ -20,7 +20,6 @@ from controllers.console.wraps import (
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
-from core.plugin.entities.plugin import ModelProviderID
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.datasource_type import DatasourceType
@@ -33,6 +32,7 @@ from fields.document_fields import document_status_fields
 from libs.login import login_required
 from models import ApiToken, Dataset, Document, DocumentSegment, UploadFile
 from models.dataset import DatasetPermissionEnum
+from models.provider_ids import ModelProviderID
 from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
 
@@ -48,7 +48,21 @@ def _validate_description_length(description):
     return description
 
 
+@console_ns.route("/datasets")
 class DatasetListApi(Resource):
+    @api.doc("get_datasets")
+    @api.doc(description="Get list of datasets")
+    @api.doc(
+        params={
+            "page": "Page number (default: 1)",
+            "limit": "Number of items per page (default: 20)",
+            "ids": "Filter by dataset IDs (list)",
+            "keyword": "Search keyword",
+            "tag_ids": "Filter by tag IDs (list)",
+            "include_all": "Include all datasets (default: false)",
+        }
+    )
+    @api.response(200, "Datasets retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -100,6 +114,24 @@ class DatasetListApi(Resource):
         response = {"data": data, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page}
         return response, 200
 
+    @api.doc("create_dataset")
+    @api.doc(description="Create a new dataset")
+    @api.expect(
+        api.model(
+            "CreateDatasetRequest",
+            {
+                "name": fields.String(required=True, description="Dataset name (1-40 characters)"),
+                "description": fields.String(description="Dataset description (max 400 characters)"),
+                "indexing_technique": fields.String(description="Indexing technique"),
+                "permission": fields.String(description="Dataset permission"),
+                "provider": fields.String(description="Provider"),
+                "external_knowledge_api_id": fields.String(description="External knowledge API ID"),
+                "external_knowledge_id": fields.String(description="External knowledge ID"),
+            },
+        )
+    )
+    @api.response(201, "Dataset created successfully")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -172,7 +204,14 @@ class DatasetListApi(Resource):
         return marshal(dataset, dataset_detail_fields), 201
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>")
 class DatasetApi(Resource):
+    @api.doc("get_dataset")
+    @api.doc(description="Get dataset details")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Dataset retrieved successfully", dataset_detail_fields)
+    @api.response(404, "Dataset not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -215,6 +254,23 @@ class DatasetApi(Resource):
 
         return data, 200
 
+    @api.doc("update_dataset")
+    @api.doc(description="Update dataset details")
+    @api.expect(
+        api.model(
+            "UpdateDatasetRequest",
+            {
+                "name": fields.String(description="Dataset name"),
+                "description": fields.String(description="Dataset description"),
+                "permission": fields.String(description="Dataset permission"),
+                "indexing_technique": fields.String(description="Indexing technique"),
+                "external_retrieval_model": fields.Raw(description="External retrieval model settings"),
+            },
+        )
+    )
+    @api.response(200, "Dataset updated successfully", dataset_detail_fields)
+    @api.response(404, "Dataset not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -281,6 +337,15 @@ class DatasetApi(Resource):
             location="json",
             help="Invalid external knowledge api id.",
         )
+
+        parser.add_argument(
+            "icon_info",
+            type=dict,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid icon info.",
+        )
         args = parser.parse_args()
         data = request.get_json()
 
@@ -331,7 +396,7 @@ class DatasetApi(Resource):
         dataset_id_str = str(dataset_id)
 
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor or current_user.is_dataset_operator:
+        if not (current_user.is_editor or current_user.is_dataset_operator):
             raise Forbidden()
 
         try:
@@ -344,7 +409,12 @@ class DatasetApi(Resource):
             raise DatasetInUseError()
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/use-check")
 class DatasetUseCheckApi(Resource):
+    @api.doc("check_dataset_use")
+    @api.doc(description="Check if dataset is in use")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Dataset use status retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -355,7 +425,12 @@ class DatasetUseCheckApi(Resource):
         return {"is_using": dataset_is_using}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/queries")
 class DatasetQueryApi(Resource):
+    @api.doc("get_dataset_queries")
+    @api.doc(description="Get dataset query history")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Query history retrieved successfully", dataset_query_detail_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -385,7 +460,11 @@ class DatasetQueryApi(Resource):
         return response, 200
 
 
+@console_ns.route("/datasets/indexing-estimate")
 class DatasetIndexingEstimateApi(Resource):
+    @api.doc("estimate_dataset_indexing")
+    @api.doc(description="Estimate dataset indexing cost")
+    @api.response(200, "Indexing estimate calculated successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -433,10 +512,12 @@ class DatasetIndexingEstimateApi(Resource):
             notion_info_list = args["info_list"]["notion_info_list"]
             for notion_info in notion_info_list:
                 workspace_id = notion_info["workspace_id"]
+                credential_id = notion_info.get("credential_id")
                 for page in notion_info["pages"]:
                     extract_setting = ExtractSetting(
                         datasource_type=DatasourceType.NOTION.value,
                         notion_info={
+                            "credential_id": credential_id,
                             "notion_workspace_id": workspace_id,
                             "notion_obj_id": page["page_id"],
                             "notion_page_type": page["type"],
@@ -486,7 +567,12 @@ class DatasetIndexingEstimateApi(Resource):
         return response.model_dump(), 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/related-apps")
 class DatasetRelatedAppListApi(Resource):
+    @api.doc("get_dataset_related_apps")
+    @api.doc(description="Get applications related to dataset")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Related apps retrieved successfully", related_app_list)
     @setup_required
     @login_required
     @account_initialization_required
@@ -513,7 +599,12 @@ class DatasetRelatedAppListApi(Resource):
         return {"data": related_apps, "total": len(related_apps)}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/indexing-status")
 class DatasetIndexingStatusApi(Resource):
+    @api.doc("get_dataset_indexing_status")
+    @api.doc(description="Get dataset indexing status")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Indexing status retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -560,11 +651,15 @@ class DatasetIndexingStatusApi(Resource):
         return data, 200
 
 
+@console_ns.route("/datasets/api-keys")
 class DatasetApiKeyApi(Resource):
     max_keys = 10
     token_prefix = "dataset-"
     resource_type = "dataset"
 
+    @api.doc("get_dataset_api_keys")
+    @api.doc(description="Get dataset API keys")
+    @api.response(200, "API keys retrieved successfully", api_key_list)
     @setup_required
     @login_required
     @account_initialization_required
@@ -609,9 +704,14 @@ class DatasetApiKeyApi(Resource):
         return api_token, 200
 
 
+@console_ns.route("/datasets/api-keys/<uuid:api_key_id>")
 class DatasetApiDeleteApi(Resource):
     resource_type = "dataset"
 
+    @api.doc("delete_dataset_api_key")
+    @api.doc(description="Delete dataset API key")
+    @api.doc(params={"api_key_id": "API key ID"})
+    @api.response(204, "API key deleted successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -641,7 +741,24 @@ class DatasetApiDeleteApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/api-keys/<string:status>")
+class DatasetEnableApiApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id, status):
+        dataset_id_str = str(dataset_id)
+
+        DatasetService.update_dataset_api_status(dataset_id_str, status == "enable")
+
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/datasets/api-base-info")
 class DatasetApiBaseUrlApi(Resource):
+    @api.doc("get_dataset_api_base_info")
+    @api.doc(description="Get dataset API base information")
+    @api.response(200, "API base info retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -649,7 +766,11 @@ class DatasetApiBaseUrlApi(Resource):
         return {"api_base_url": (dify_config.SERVICE_API_URL or request.host_url.rstrip("/")) + "/v1"}
 
 
+@console_ns.route("/datasets/retrieval-setting")
 class DatasetRetrievalSettingApi(Resource):
+    @api.doc("get_dataset_retrieval_setting")
+    @api.doc(description="Get dataset retrieval settings")
+    @api.response(200, "Retrieval settings retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -661,7 +782,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.PGVECTO_RS
-                | VectorType.BAIDU
                 | VectorType.VIKINGDB
                 | VectorType.UPSTASH
             ):
@@ -688,6 +808,7 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.TENCENT
                 | VectorType.MATRIXONE
                 | VectorType.CLICKZETTA
+                | VectorType.BAIDU
             ):
                 return {
                     "retrieval_method": [
@@ -700,7 +821,12 @@ class DatasetRetrievalSettingApi(Resource):
                 raise ValueError(f"Unsupported vector db type {vector_type}.")
 
 
+@console_ns.route("/datasets/retrieval-setting/<string:vector_type>")
 class DatasetRetrievalSettingMockApi(Resource):
+    @api.doc("get_dataset_retrieval_setting_mock")
+    @api.doc(description="Get mock dataset retrieval settings by vector type")
+    @api.doc(params={"vector_type": "Vector store type"})
+    @api.response(200, "Mock retrieval settings retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -712,7 +838,6 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.PGVECTO_RS
-                | VectorType.BAIDU
                 | VectorType.VIKINGDB
                 | VectorType.UPSTASH
             ):
@@ -737,6 +862,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.HUAWEI_CLOUD
                 | VectorType.MATRIXONE
                 | VectorType.CLICKZETTA
+                | VectorType.BAIDU
             ):
                 return {
                     "retrieval_method": [
@@ -749,7 +875,13 @@ class DatasetRetrievalSettingMockApi(Resource):
                 raise ValueError(f"Unsupported vector db type {vector_type}.")
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/error-docs")
 class DatasetErrorDocs(Resource):
+    @api.doc("get_dataset_error_docs")
+    @api.doc(description="Get dataset error documents")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Error documents retrieved successfully")
+    @api.response(404, "Dataset not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -763,7 +895,14 @@ class DatasetErrorDocs(Resource):
         return {"data": [marshal(item, document_status_fields) for item in results], "total": len(results)}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/permission-part-users")
 class DatasetPermissionUserListApi(Resource):
+    @api.doc("get_dataset_permission_users")
+    @api.doc(description="Get dataset permission user list")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Permission users retrieved successfully")
+    @api.response(404, "Dataset not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -784,7 +923,13 @@ class DatasetPermissionUserListApi(Resource):
         }, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/auto-disable-logs")
 class DatasetAutoDisableLogApi(Resource):
+    @api.doc("get_dataset_auto_disable_logs")
+    @api.doc(description="Get dataset auto disable logs")
+    @api.doc(params={"dataset_id": "Dataset ID"})
+    @api.response(200, "Auto disable logs retrieved successfully")
+    @api.response(404, "Dataset not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -794,20 +939,3 @@ class DatasetAutoDisableLogApi(Resource):
         if dataset is None:
             raise NotFound("Dataset not found.")
         return DatasetService.get_dataset_auto_disable_logs(dataset_id_str), 200
-
-
-api.add_resource(DatasetListApi, "/datasets")
-api.add_resource(DatasetApi, "/datasets/<uuid:dataset_id>")
-api.add_resource(DatasetUseCheckApi, "/datasets/<uuid:dataset_id>/use-check")
-api.add_resource(DatasetQueryApi, "/datasets/<uuid:dataset_id>/queries")
-api.add_resource(DatasetErrorDocs, "/datasets/<uuid:dataset_id>/error-docs")
-api.add_resource(DatasetIndexingEstimateApi, "/datasets/indexing-estimate")
-api.add_resource(DatasetRelatedAppListApi, "/datasets/<uuid:dataset_id>/related-apps")
-api.add_resource(DatasetIndexingStatusApi, "/datasets/<uuid:dataset_id>/indexing-status")
-api.add_resource(DatasetApiKeyApi, "/datasets/api-keys")
-api.add_resource(DatasetApiDeleteApi, "/datasets/api-keys/<uuid:api_key_id>")
-api.add_resource(DatasetApiBaseUrlApi, "/datasets/api-base-info")
-api.add_resource(DatasetRetrievalSettingApi, "/datasets/retrieval-setting")
-api.add_resource(DatasetRetrievalSettingMockApi, "/datasets/retrieval-setting/<string:vector_type>")
-api.add_resource(DatasetPermissionUserListApi, "/datasets/<uuid:dataset_id>/permission-part-users")
-api.add_resource(DatasetAutoDisableLogApi, "/datasets/<uuid:dataset_id>/auto-disable-logs")

api/controllers/console/datasets/datasets_document.py

@@ -1,16 +1,18 @@
+import json
 import logging
 from argparse import ArgumentTypeError
 from collections.abc import Sequence
 from typing import Literal, cast
 
+import sqlalchemy as sa
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal, marshal_with, reqparse
 from sqlalchemy import asc, desc, select
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     ProviderModelCurrentlyNotSupportError,
     ProviderNotInitializeError,
@@ -53,6 +55,7 @@ from fields.document_fields import (
 from libs.datetime_utils import naive_utc_now
 from libs.login import login_required
 from models import Dataset, DatasetProcessRule, Document, DocumentSegment, UploadFile
+from models.dataset import DocumentPipelineExecutionLog
 from services.dataset_service import DatasetService, DocumentService
 from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig
 
@@ -98,7 +101,12 @@ class DocumentResource(Resource):
         return documents
 
 
+@console_ns.route("/datasets/process-rule")
 class GetProcessRuleApi(Resource):
+    @api.doc("get_process_rule")
+    @api.doc(description="Get dataset document processing rules")
+    @api.doc(params={"document_id": "Document ID (optional)"})
+    @api.response(200, "Process rules retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -140,7 +148,21 @@ class GetProcessRuleApi(Resource):
         return {"mode": mode, "rules": rules, "limits": limits}
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents")
 class DatasetDocumentListApi(Resource):
+    @api.doc("get_dataset_documents")
+    @api.doc(description="Get documents in a dataset")
+    @api.doc(
+        params={
+            "dataset_id": "Dataset ID",
+            "page": "Page number (default: 1)",
+            "limit": "Number of items per page (default: 20)",
+            "keyword": "Search keyword",
+            "sort": "Sort order (default: -created_at)",
+            "fetch": "Fetch full details (default: false)",
+        }
+    )
+    @api.response(200, "Documents retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -190,13 +212,13 @@ class DatasetDocumentListApi(Resource):
 
         if sort == "hit_count":
             sub_query = (
-                db.select(DocumentSegment.document_id, db.func.sum(DocumentSegment.hit_count).label("total_hit_count"))
+                sa.select(DocumentSegment.document_id, sa.func.sum(DocumentSegment.hit_count).label("total_hit_count"))
                 .group_by(DocumentSegment.document_id)
                 .subquery()
             )
 
             query = query.outerjoin(sub_query, sub_query.c.document_id == Document.id).order_by(
-                sort_logic(db.func.coalesce(sub_query.c.total_hit_count, 0)),
+                sort_logic(sa.func.coalesce(sub_query.c.total_hit_count, 0)),
                 sort_logic(Document.position),
             )
         elif sort == "created_at":
@@ -324,7 +346,23 @@ class DatasetDocumentListApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/init")
 class DatasetInitApi(Resource):
+    @api.doc("init_dataset")
+    @api.doc(description="Initialize dataset with documents")
+    @api.expect(
+        api.model(
+            "DatasetInitRequest",
+            {
+                "upload_file_id": fields.String(required=True, description="Upload file ID"),
+                "indexing_technique": fields.String(description="Indexing technique"),
+                "process_rule": fields.Raw(description="Processing rules"),
+                "data_source": fields.Raw(description="Data source configuration"),
+            },
+        )
+    )
+    @api.response(201, "Dataset initialized successfully", dataset_and_document_fields)
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -394,7 +432,14 @@ class DatasetInitApi(Resource):
         return response
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-estimate")
 class DocumentIndexingEstimateApi(DocumentResource):
+    @api.doc("estimate_document_indexing")
+    @api.doc(description="Estimate document indexing cost")
+    @api.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @api.response(200, "Indexing estimate calculated successfully")
+    @api.response(404, "Document not found")
+    @api.response(400, "Document already finished")
     @setup_required
     @login_required
     @account_initialization_required
@@ -457,6 +502,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
         return response, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-estimate")
 class DocumentBatchIndexingEstimateApi(DocumentResource):
     @setup_required
     @login_required
@@ -499,6 +545,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 extract_setting = ExtractSetting(
                     datasource_type=DatasourceType.NOTION.value,
                     notion_info={
+                        "credential_id": data_source_info["credential_id"],
                         "notion_workspace_id": data_source_info["notion_workspace_id"],
                         "notion_obj_id": data_source_info["notion_page_id"],
                         "notion_page_type": data_source_info["type"],
@@ -549,6 +596,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 raise IndexingEstimateError(str(e))
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-status")
 class DocumentBatchIndexingStatusApi(DocumentResource):
     @setup_required
     @login_required
@@ -593,7 +641,13 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
         return data
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-status")
 class DocumentIndexingStatusApi(DocumentResource):
+    @api.doc("get_document_indexing_status")
+    @api.doc(description="Get document indexing status")
+    @api.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @api.response(200, "Indexing status retrieved successfully")
+    @api.response(404, "Document not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -635,9 +689,21 @@ class DocumentIndexingStatusApi(DocumentResource):
         return marshal(document_dict, document_status_fields)
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 class DocumentApi(DocumentResource):
     METADATA_CHOICES = {"all", "only", "without"}
 
+    @api.doc("get_document")
+    @api.doc(description="Get document details")
+    @api.doc(
+        params={
+            "dataset_id": "Dataset ID",
+            "document_id": "Document ID",
+            "metadata": "Metadata inclusion (all/only/without)",
+        }
+    )
+    @api.response(200, "Document retrieved successfully")
+    @api.response(404, "Document not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -654,7 +720,7 @@ class DocumentApi(DocumentResource):
             response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
         elif metadata == "without":
             dataset_process_rules = DatasetService.get_process_rules(dataset_id)
-            document_process_rules = document.dataset_process_rule.to_dict()
+            document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             data_source_info = document.data_source_detail_dict
             response = {
                 "id": document.id,
@@ -746,7 +812,16 @@ class DocumentApi(DocumentResource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/<string:action>")
 class DocumentProcessingApi(DocumentResource):
+    @api.doc("update_document_processing")
+    @api.doc(description="Update document processing status (pause/resume)")
+    @api.doc(
+        params={"dataset_id": "Dataset ID", "document_id": "Document ID", "action": "Action to perform (pause/resume)"}
+    )
+    @api.response(200, "Processing status updated successfully")
+    @api.response(404, "Document not found")
+    @api.response(400, "Invalid action")
     @setup_required
     @login_required
     @account_initialization_required
@@ -781,7 +856,23 @@ class DocumentProcessingApi(DocumentResource):
         return {"result": "success"}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")
 class DocumentMetadataApi(DocumentResource):
+    @api.doc("update_document_metadata")
+    @api.doc(description="Update document metadata")
+    @api.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @api.expect(
+        api.model(
+            "UpdateDocumentMetadataRequest",
+            {
+                "doc_type": fields.String(description="Document type"),
+                "doc_metadata": fields.Raw(description="Document metadata"),
+            },
+        )
+    )
+    @api.response(200, "Document metadata updated successfully")
+    @api.response(404, "Document not found")
+    @api.response(403, "Permission denied")
     @setup_required
     @login_required
     @account_initialization_required
@@ -825,6 +916,7 @@ class DocumentMetadataApi(DocumentResource):
         return {"result": "success", "message": "Document metadata updated."}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")
 class DocumentStatusApi(DocumentResource):
     @setup_required
     @login_required
@@ -861,6 +953,7 @@ class DocumentStatusApi(DocumentResource):
         return {"result": "success"}, 200
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")
 class DocumentPauseApi(DocumentResource):
     @setup_required
     @login_required
@@ -894,6 +987,7 @@ class DocumentPauseApi(DocumentResource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/resume")
 class DocumentRecoverApi(DocumentResource):
     @setup_required
     @login_required
@@ -924,6 +1018,7 @@ class DocumentRecoverApi(DocumentResource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/retry")
 class DocumentRetryApi(DocumentResource):
     @setup_required
     @login_required
@@ -967,6 +1062,7 @@ class DocumentRetryApi(DocumentResource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/rename")
 class DocumentRenameApi(DocumentResource):
     @setup_required
     @login_required
@@ -990,6 +1086,7 @@ class DocumentRenameApi(DocumentResource):
         return document
 
 
+@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/website-sync")
 class WebsiteDocumentSyncApi(DocumentResource):
     @setup_required
     @login_required
@@ -1017,6 +1114,41 @@ class WebsiteDocumentSyncApi(DocumentResource):
         return {"result": "success"}, 200
 
 
+class DocumentPipelineExecutionLogApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        log = (
+            db.session.query(DocumentPipelineExecutionLog)
+            .filter_by(document_id=document_id)
+            .order_by(DocumentPipelineExecutionLog.created_at.desc())
+            .first()
+        )
+        if not log:
+            return {
+                "datasource_info": None,
+                "datasource_type": None,
+                "input_data": None,
+                "datasource_node_id": None,
+            }, 200
+        return {
+            "datasource_info": json.loads(log.datasource_info),
+            "datasource_type": log.datasource_type,
+            "input_data": log.input_data,
+            "datasource_node_id": log.datasource_node_id,
+        }, 200
+
+
 api.add_resource(GetProcessRuleApi, "/datasets/process-rule")
 api.add_resource(DatasetDocumentListApi, "/datasets/<uuid:dataset_id>/documents")
 api.add_resource(DatasetInitApi, "/datasets/init")
@@ -1038,3 +1170,6 @@ api.add_resource(DocumentRetryApi, "/datasets/<uuid:dataset_id>/retry")
 api.add_resource(DocumentRenameApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/rename")
 
 api.add_resource(WebsiteDocumentSyncApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/website-sync")
+api.add_resource(
+    DocumentPipelineExecutionLogApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/pipeline-execution-log"
+)

api/controllers/console/datasets/error.py

@@ -71,3 +71,9 @@ class ChildChunkDeleteIndexError(BaseHTTPException):
     error_code = "child_chunk_delete_index_error"
     description = "Delete child chunk index failed: {message}"
     code = 500
+
+
+class PipelineNotFoundError(BaseHTTPException):
+    error_code = "pipeline_not_found"
+    description = "Pipeline not found."
+    code = 404