refactor: simplify spread usage in various components (#25886)

This option would generate upgrading / table creating sql with `NULLS
DISTINCT` part and causing syntax error while running testcontainer
tests.

The `NULLS DISTINCT` syntax is only supported by PG 15+.

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/web"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2
+  - package-ecosystem: "uv"
+    directory: "/api"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2

.github/workflows/api-tests.yml

@@ -1,13 +1,7 @@
 name: Run Pytest
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - api/**
-      - docker/**
-      - .github/workflows/api-tests.yml
+  workflow_call:
 
 concurrency:
   group: api-tests-${{ github.head_ref || github.run_id }}
@@ -48,11 +42,7 @@ jobs:
       - name: Run Unit tests
         run: |
           uv run --project api bash dev/pytest/pytest_unit_tests.sh
-      - name: Run ty check
-        run: |
-          cd api
-          uv add --dev ty
-          uv run ty check || true
+
       - name: Run pyrefly check
         run: |
           cd api
@@ -72,15 +62,6 @@ jobs:
       - name: Run dify config tests
         run: uv run --project api dev/pytest/pytest_config_tests.py
 
-      - name: MyPy Cache
-        uses: actions/cache@v4
-        with:
-          path: api/.mypy_cache
-          key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }}
-
-      - name: Run MyPy Checks
-        run: dev/mypy-check
-
       - name: Set up dotenvs
         run: |
           cp docker/.env.example docker/.env

.github/workflows/autofix.yml

@@ -1,10 +1,7 @@
 name: autofix.ci
 on:
-  workflow_call:
   pull_request:
-    branches: [ "main" ]
-  push:
-    branches: [ "main" ]
+    branches: ["main"]
 permissions:
   contents: read
 
@@ -18,18 +15,65 @@ jobs:
       # Use uv to ensure we have the same ruff version in CI and locally.
       - uses: astral-sh/setup-uv@v6
         with:
-          python-version: "3.12" 
+          python-version: "3.12"
       - run: |
           cd api
           uv sync --dev
           # Fix lint errors
-          uv run ruff check --fix-only .
+          uv run ruff check --fix .
           # Format code
-          uv run ruff format .
+          uv run ruff format ..
+
       - name: ast-grep
         run: |
           uvx --from ast-grep-cli sg --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all
+          uvx --from ast-grep-cli sg --pattern 'session.query($WHATEVER).filter($HERE)' --rewrite 'session.query($WHATEVER).where($HERE)' -l py --update-all
+          # Convert Optional[T] to T | None (ignoring quoted types)
+          cat > /tmp/optional-rule.yml << 'EOF'
+          id: convert-optional-to-union
+          language: python
+          rule:
+            kind: generic_type
+            all:
+              - has:
+                  kind: identifier
+                  pattern: Optional
+              - has:
+                  kind: type_parameter
+                  has:
+                    kind: type
+                    pattern: $T
+          fix: $T | None
+          EOF
+          uvx --from ast-grep-cli sg scan --inline-rules "$(cat /tmp/optional-rule.yml)" --update-all
+          # Fix forward references that were incorrectly converted (Python doesn't support "Type" | None syntax)
+          find . -name "*.py" -type f -exec sed -i.bak -E 's/"([^"]+)" \| None/Optional["\1"]/g; s/'"'"'([^'"'"']+)'"'"' \| None/Optional['"'"'\1'"'"']/g' {} \;
+          find . -name "*.py.bak" -type f -delete
+
       - name: mdformat
         run: |
           uvx mdformat .
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: web/package.json
+          run_install: false
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: ./web/package.json
+
+      - name: Web dependencies
+        working-directory: ./web
+        run: pnpm install --frozen-lockfile
+
+      - name: oxlint
+        working-directory: ./web
+        run: |
+          pnpx oxlint --fix
+
       - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27

.github/workflows/build-push.yml

@@ -8,6 +8,8 @@ on:
       - "deploy/enterprise"
       - "build/**"
       - "release/e-*"
+      - "deploy/rag-dev"
+      - "feat/rag-2"
     tags:
       - "*"
 

.github/workflows/db-migration-test.yml

@@ -1,13 +1,7 @@
 name: DB Migration Test
 
 on:
-  pull_request:
-    branches:
-      - main
-      - plugins/beta
-    paths:
-      - api/migrations/**
-      - .github/workflows/db-migration-test.yml
+  workflow_call:
 
 concurrency:
   group: db-migration-test-${{ github.ref }}
@@ -33,6 +27,12 @@ jobs:
 
       - name: Install dependencies
         run: uv sync --project api
+      - name: Ensure Offline migration are supported
+        run: |
+          # upgrade
+          uv run --directory api flask db upgrade 'base:head' --sql
+          # downgrade
+          uv run --directory api flask db downgrade 'head:base' --sql
 
       - name: Prepare middleware env
         run: |

.github/workflows/deploy-dev.yml

@@ -4,7 +4,7 @@ on:
   workflow_run:
     workflows: ["Build and Push API & Web"]
     branches:
-      - "deploy/dev"
+      - "deploy/rag-dev"
     types:
       - completed
 
@@ -12,12 +12,13 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     if: |
-      github.event.workflow_run.conclusion == 'success'
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_branch == 'deploy/rag-dev'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v0.1.8
         with:
-          host: ${{ secrets.SSH_HOST }}
+          host: ${{ secrets.RAG_SSH_HOST }}
           username: ${{ secrets.SSH_USER }}
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           script: |

.github/workflows/deploy-enterprise.yml

@@ -19,11 +19,23 @@ jobs:
       github.event.workflow_run.head_branch == 'deploy/enterprise'
 
     steps:
-      - name: Deploy to server
-        uses: appleboy/ssh-action@v0.1.8
-        with:
-          host: ${{ secrets.ENTERPRISE_SSH_HOST }}
-          username: ${{ secrets.ENTERPRISE_SSH_USER }}
-          password: ${{ secrets.ENTERPRISE_SSH_PASSWORD }}
-          script: |
-            ${{ vars.ENTERPRISE_SSH_SCRIPT || secrets.ENTERPRISE_SSH_SCRIPT }}
+      - name: trigger deployments
+        env:
+          DEV_ENV_ADDRS: ${{ vars.DEV_ENV_ADDRS }}
+          DEPLOY_SECRET: ${{ secrets.DEPLOY_SECRET }}
+        run: |
+          IFS=',' read -ra ENDPOINTS <<< "${DEV_ENV_ADDRS:-}"
+          BODY='{"project":"dify-api","tag":"deploy-enterprise"}'
+
+          for ENDPOINT in "${ENDPOINTS[@]}"; do
+            ENDPOINT="$(echo "$ENDPOINT" | xargs)"
+            [ -z "$ENDPOINT" ] && continue
+
+            API_SIGNATURE=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "$DEPLOY_SECRET" | awk '{print "sha256="$2}')
+
+            curl -sSf -X POST \
+              -H "Content-Type: application/json" \
+              -H "X-Hub-Signature-256: $API_SIGNATURE" \
+              -d "$BODY" \
+              "$ENDPOINT"
+          done

.github/workflows/main-ci.yml

@@ -0,0 +1,78 @@
+name: Main CI Pipeline
+
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: write
+  pull-requests: write
+  checks: write
+  statuses: write
+
+concurrency:
+  group: main-ci-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Check which paths were changed to determine which tests to run
+  check-changes:
+    name: Check Changed Files
+    runs-on: ubuntu-latest
+    outputs:
+      api-changed: ${{ steps.changes.outputs.api }}
+      web-changed: ${{ steps.changes.outputs.web }}
+      vdb-changed: ${{ steps.changes.outputs.vdb }}
+      migration-changed: ${{ steps.changes.outputs.migration }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            api:
+              - 'api/**'
+              - 'docker/**'
+              - '.github/workflows/api-tests.yml'
+            web:
+              - 'web/**'
+            vdb:
+              - 'api/core/rag/datasource/**'
+              - 'docker/**'
+              - '.github/workflows/vdb-tests.yml'
+              - 'api/uv.lock'
+              - 'api/pyproject.toml'
+            migration:
+              - 'api/migrations/**'
+              - '.github/workflows/db-migration-test.yml'
+
+  # Run tests in parallel
+  api-tests:
+    name: API Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.api-changed == 'true'
+    uses: ./.github/workflows/api-tests.yml
+
+  web-tests:
+    name: Web Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.web-changed == 'true'
+    uses: ./.github/workflows/web-tests.yml
+
+  style-check:
+    name: Style Check
+    uses: ./.github/workflows/style.yml
+
+  vdb-tests:
+    name: VDB Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.vdb-changed == 'true'
+    uses: ./.github/workflows/vdb-tests.yml
+
+  db-migration-test:
+    name: DB Migration Test
+    needs: check-changes
+    if: needs.check-changes.outputs.migration-changed == 'true'
+    uses: ./.github/workflows/db-migration-test.yml

.github/workflows/style.yml

@@ -1,9 +1,7 @@
 name: Style check
 
 on:
-  pull_request:
-    branches:
-      - main
+  workflow_call:
 
 concurrency:
   group: style-${{ github.head_ref || github.run_id }}
@@ -14,7 +12,6 @@ permissions:
   statuses: write
   contents: read
 
-
 jobs:
   python-style:
     name: Python Style
@@ -46,21 +43,22 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: uv sync --project api --dev
 
-      - name: Ruff check
+      - name: Run Import Linter
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          uv run --directory api ruff --version
-          uv run --directory api ruff check ./
-          uv run --directory api ruff format --check ./
+        run: uv run --directory api --dev lint-imports
+
+      - name: Run Basedpyright Checks
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: dev/basedpyright-check
+
+      - name: Run Mypy Type Checks
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --check-untyped-defs --disable-error-code=import-untyped .
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
         run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
 
-      - name: Lint hints
-        if: failure()
-        run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
-
   web-style:
     name: Web Style
     runs-on: ubuntu-latest
@@ -102,7 +100,9 @@ jobs:
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: ./web
-        run: pnpm run lint
+        run: |
+          pnpm run lint
+          pnpm run eslint
 
   docker-compose-template:
     name: Docker Compose Template

.github/workflows/translate-i18n-base-on-english.yml

@@ -67,12 +67,22 @@ jobs:
         working-directory: ./web
         run: pnpm run auto-gen-i18n ${{ env.FILE_ARGS }}
 
+      - name: Generate i18n type definitions
+        if: env.FILES_CHANGED == 'true'
+        working-directory: ./web
+        run: pnpm run gen:i18n-types
+
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update i18n files based on en-US changes
-          title: 'chore: translate i18n files'
-          body: This PR was automatically created to update i18n files based on changes in en-US locale.
+          commit-message: Update i18n files and type definitions based on en-US changes
+          title: 'chore: translate i18n files and update type definitions'
+          body: |
+            This PR was automatically created to update i18n files and TypeScript type definitions based on changes in en-US locale.
+            
+            **Changes included:**
+            - Updated translation files for all locales
+            - Regenerated TypeScript type definitions for type safety
           branch: chore/automated-i18n-updates

.github/workflows/vdb-tests.yml

@@ -1,15 +1,7 @@
 name: Run VDB Tests
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - api/core/rag/datasource/**
-      - docker/**
-      - .github/workflows/vdb-tests.yml
-      - api/uv.lock
-      - api/pyproject.toml
+  workflow_call:
 
 concurrency:
   group: vdb-tests-${{ github.head_ref || github.run_id }}

.github/workflows/web-tests.yml

@@ -1,11 +1,7 @@
 name: Web Tests
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - web/**
+  workflow_call:
 
 concurrency:
   group: web-tests-${{ github.head_ref || github.run_id }}
@@ -51,6 +47,11 @@ jobs:
         working-directory: ./web
         run: pnpm install --frozen-lockfile
 
+      - name: Check i18n types synchronization
+        if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
+        run: pnpm run check:i18n-types
+
       - name: Run tests
         if: steps.changed-files.outputs.any_changed == 'true'
         working-directory: ./web

.gitignore

@@ -123,10 +123,12 @@ venv.bak/
 # mkdocs documentation
 /site
 
-# mypy
+# type checking
 .mypy_cache/
 .dmypy.json
 dmypy.json
+pyrightconfig.json
+!api/pyrightconfig.json
 
 # Pyre type checker
 .pyre/
@@ -195,8 +197,8 @@ sdks/python-client/dify_client.egg-info
 .vscode/*
 !.vscode/launch.json.template
 !.vscode/README.md
-pyrightconfig.json
 api/.vscode
+web/.vscode
 # vscode Code History Extension
 .history
 
@@ -214,7 +216,18 @@ mise.toml
 # Next.js build output
 .next/
 
+# PWA generated files
+web/public/sw.js
+web/public/sw.js.map
+web/public/workbox-*.js
+web/public/workbox-*.js.map
+web/public/fallback-*.js
+
 # AI Assistant
 .roo/
 api/.env.backup
 /clickzetta
+
+# Benchmark
+scripts/stress-test/setup/config/
+scripts/stress-test/reports/

AGENTS.md

@@ -1 +0,0 @@
-CLAUDE.md

AGENTS.md

@@ -0,0 +1,87 @@
+# AGENTS.md
+
+## Project Overview
+
+Dify is an open-source platform for developing LLM applications with an intuitive interface combining agentic AI workflows, RAG pipelines, agent capabilities, and model management.
+
+The codebase consists of:
+
+- **Backend API** (`/api`): Python Flask application with Domain-Driven Design architecture
+- **Frontend Web** (`/web`): Next.js 15 application with TypeScript and React 19
+- **Docker deployment** (`/docker`): Containerized deployment configurations
+
+## Development Commands
+
+### Backend (API)
+
+All Python commands must be prefixed with `uv run --project api`:
+
+```bash
+# Start development servers
+./dev/start-api                   # Start API server
+./dev/start-worker                # Start Celery worker
+
+# Run tests
+uv run --project api pytest      # Run all tests
+uv run --project api pytest tests/unit_tests/     # Unit tests only
+uv run --project api pytest tests/integration_tests/  # Integration tests
+
+# Code quality
+./dev/reformat                    # Run all formatters and linters
+uv run --project api ruff check --fix ./    # Fix linting issues
+uv run --project api ruff format ./         # Format code
+uv run --directory api basedpyright         # Type checking
+```
+
+### Frontend (Web)
+
+```bash
+cd web
+pnpm lint                         # Run ESLint
+pnpm eslint-fix                   # Fix ESLint issues
+pnpm test                         # Run Jest tests
+```
+
+## Testing Guidelines
+
+### Backend Testing
+
+- Use `pytest` for all backend tests
+- Write tests first (TDD approach)
+- Test structure: Arrange-Act-Assert
+
+## Code Style Requirements
+
+### Python
+
+- Use type hints for all functions and class attributes
+- No `Any` types unless absolutely necessary
+- Implement special methods (`__repr__`, `__str__`) appropriately
+
+### TypeScript/JavaScript
+
+- Strict TypeScript configuration
+- ESLint with Prettier integration
+- Avoid `any` type
+
+## Important Notes
+
+- **Environment Variables**: Always use UV for Python commands: `uv run --project api <command>`
+- **Comments**: Only write meaningful comments that explain "why", not "what"
+- **File Creation**: Always prefer editing existing files over creating new ones
+- **Documentation**: Don't create documentation files unless explicitly requested
+- **Code Quality**: Always run `./dev/reformat` before committing backend changes
+
+## Common Development Tasks
+
+### Adding a New API Endpoint
+
+1. Create controller in `/api/controllers/`
+1. Add service logic in `/api/services/`
+1. Update routes in controller's `__init__.py`
+1. Write tests in `/api/tests/`
+
+## Project-Specific Conventions
+
+- All async tasks use Celery with Redis as broker
+- **Internationalization**: Frontend supports multiple languages with English (`web/i18n/en-US/`) as the source. All user-facing text must use i18n keys, no hardcoded strings. Edit corresponding module files in `en-US/` directory for translations.

CLAUDE.md

@@ -1,89 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-Dify is an open-source platform for developing LLM applications with an intuitive interface combining agentic AI workflows, RAG pipelines, agent capabilities, and model management.
-
-The codebase consists of:
-
-- **Backend API** (`/api`): Python Flask application with Domain-Driven Design architecture
-- **Frontend Web** (`/web`): Next.js 15 application with TypeScript and React 19
-- **Docker deployment** (`/docker`): Containerized deployment configurations
-
-## Development Commands
-
-### Backend (API)
-
-All Python commands must be prefixed with `uv run --project api`:
-
-```bash
-# Start development servers
-./dev/start-api                   # Start API server
-./dev/start-worker                # Start Celery worker
-
-# Run tests
-uv run --project api pytest      # Run all tests
-uv run --project api pytest tests/unit_tests/     # Unit tests only
-uv run --project api pytest tests/integration_tests/  # Integration tests
-
-# Code quality
-./dev/reformat                    # Run all formatters and linters
-uv run --project api ruff check --fix ./    # Fix linting issues
-uv run --project api ruff format ./         # Format code
-uv run --project api mypy .                 # Type checking
-```
-
-### Frontend (Web)
-
-```bash
-cd web
-pnpm lint                         # Run ESLint
-pnpm eslint-fix                   # Fix ESLint issues
-pnpm test                         # Run Jest tests
-```
-
-## Testing Guidelines
-
-### Backend Testing
-
-- Use `pytest` for all backend tests
-- Write tests first (TDD approach)
-- Test structure: Arrange-Act-Assert
-
-## Code Style Requirements
-
-### Python
-
-- Use type hints for all functions and class attributes
-- No `Any` types unless absolutely necessary
-- Implement special methods (`__repr__`, `__str__`) appropriately
-
-### TypeScript/JavaScript
-
-- Strict TypeScript configuration
-- ESLint with Prettier integration
-- Avoid `any` type
-
-## Important Notes
-
-- **Environment Variables**: Always use UV for Python commands: `uv run --project api <command>`
-- **Comments**: Only write meaningful comments that explain "why", not "what"
-- **File Creation**: Always prefer editing existing files over creating new ones
-- **Documentation**: Don't create documentation files unless explicitly requested
-- **Code Quality**: Always run `./dev/reformat` before committing backend changes
-
-## Common Development Tasks
-
-### Adding a New API Endpoint
-
-1. Create controller in `/api/controllers/`
-1. Add service logic in `/api/services/`
-1. Update routes in controller's `__init__.py`
-1. Write tests in `/api/tests/`
-
-## Project-Specific Conventions
-
-- All async tasks use Celery with Redis as broker
-- **Internationalization**: Frontend supports multiple languages with English (`web/i18n/en-US/`) as the source. All user-facing text must use i18n keys, no hardcoded strings. Edit corresponding module files in `en-US/` directory for translations.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Makefile

@@ -4,6 +4,72 @@ WEB_IMAGE=$(DOCKER_REGISTRY)/dify-web
 API_IMAGE=$(DOCKER_REGISTRY)/dify-api
 VERSION=latest
 
+# Default target - show help
+.DEFAULT_GOAL := help
+
+# Backend Development Environment Setup
+.PHONY: dev-setup prepare-docker prepare-web prepare-api
+
+# Dev setup target
+dev-setup: prepare-docker prepare-web prepare-api
+	@echo "✅ Backend development environment setup complete!"
+
+# Step 1: Prepare Docker middleware
+prepare-docker:
+	@echo "🐳 Setting up Docker middleware..."
+	@cp -n docker/middleware.env.example docker/middleware.env 2>/dev/null || echo "Docker middleware.env already exists"
+	@cd docker && docker compose -f docker-compose.middleware.yaml --env-file middleware.env -p dify-middlewares-dev up -d
+	@echo "✅ Docker middleware started"
+
+# Step 2: Prepare web environment
+prepare-web:
+	@echo "🌐 Setting up web environment..."
+	@cp -n web/.env.example web/.env 2>/dev/null || echo "Web .env already exists"
+	@cd web && pnpm install
+	@cd web && pnpm build
+	@echo "✅ Web environment prepared (not started)"
+
+# Step 3: Prepare API environment
+prepare-api:
+	@echo "🔧 Setting up API environment..."
+	@cp -n api/.env.example api/.env 2>/dev/null || echo "API .env already exists"
+	@cd api && uv sync --dev
+	@cd api && uv run flask db upgrade
+	@echo "✅ API environment prepared (not started)"
+
+# Clean dev environment
+dev-clean:
+	@echo "⚠️  Stopping Docker containers..."
+	@cd docker && docker compose -f docker-compose.middleware.yaml --env-file middleware.env -p dify-middlewares-dev down
+	@echo "🗑️  Removing volumes..."
+	@rm -rf docker/volumes/db
+	@rm -rf docker/volumes/redis
+	@rm -rf docker/volumes/plugin_daemon
+	@rm -rf docker/volumes/weaviate
+	@rm -rf api/storage
+	@echo "✅ Cleanup complete"
+
+# Backend Code Quality Commands
+format:
+	@echo "🎨 Running ruff format..."
+	@uv run --project api --dev ruff format ./api
+	@echo "✅ Code formatting complete"
+
+check:
+	@echo "🔍 Running ruff check..."
+	@uv run --project api --dev ruff check ./api
+	@echo "✅ Code check complete"
+
+lint:
+	@echo "🔧 Running ruff format and check with fixes..."
+	@uv run --directory api --dev sh -c 'ruff format ./api && ruff check --fix ./api'
+	@echo "✅ Linting complete"
+
+type-check:
+	@echo "📝 Running type check with basedpyright..."
+	@uv run --directory api --dev basedpyright
+	@echo "✅ Type check complete"
+
 # Build Docker images
 build-web:
 	@echo "Building web Docker image: $(WEB_IMAGE):$(VERSION)..."
@@ -39,5 +105,27 @@ build-push-web: build-web push-web
 build-push-all: build-all push-all
 	@echo "All Docker images have been built and pushed."
 
+# Help target
+help:
+	@echo "Development Setup Targets:"
+	@echo "  make dev-setup      - Run all setup steps for backend dev environment"
+	@echo "  make prepare-docker - Set up Docker middleware"
+	@echo "  make prepare-web    - Set up web environment"
+	@echo "  make prepare-api    - Set up API environment"
+	@echo "  make dev-clean      - Stop Docker middleware containers"
+	@echo ""
+	@echo "Backend Code Quality:"
+	@echo "  make format         - Format code with ruff"
+	@echo "  make check          - Check code with ruff"
+	@echo "  make lint           - Format and fix code with ruff"
+	@echo "  make type-check     - Run type checking with basedpyright"
+	@echo ""
+	@echo "Docker Build Targets:"
+	@echo "  make build-web      - Build web Docker image"
+	@echo "  make build-api      - Build API Docker image"
+	@echo "  make build-all      - Build all Docker images"
+	@echo "  make push-all       - Push all Docker images"
+	@echo "  make build-push-all - Build and push all Docker images"
+
 # Phony targets
-.PHONY: build-web build-api push-web push-api build-all push-all build-push-all
+.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help format check lint type-check

api/.env.example

@@ -75,6 +75,7 @@ DB_PASSWORD=difyai123456
 DB_HOST=localhost
 DB_PORT=5432
 DB_DATABASE=dify
+SQLALCHEMY_POOL_PRE_PING=true
 
 # Storage configuration
 # use for store upload files, private keys...
@@ -327,7 +328,7 @@ MATRIXONE_DATABASE=dify
 LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
 LINDORM_USERNAME=admin
 LINDORM_PASSWORD=admin
-USING_UGC_INDEX=False
+LINDORM_USING_UGC=True
 LINDORM_QUERY_TIMEOUT=1
 
 # OceanBase Vector configuration
@@ -460,6 +461,16 @@ WORKFLOW_CALL_MAX_DEPTH=5
 WORKFLOW_PARALLEL_DEPTH_LIMIT=3
 MAX_VARIABLE_SIZE=204800
 
+# GraphEngine Worker Pool Configuration
+# Minimum number of workers per GraphEngine instance (default: 1)
+GRAPH_ENGINE_MIN_WORKERS=1
+# Maximum number of workers per GraphEngine instance (default: 10)
+GRAPH_ENGINE_MAX_WORKERS=10
+# Queue depth threshold that triggers worker scale up (default: 3)
+GRAPH_ENGINE_SCALE_UP_THRESHOLD=3
+# Seconds of idle time before scaling down workers (default: 5.0)
+GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0
+
 # Workflow storage configuration
 # Options: rdbms, hybrid
 # rdbms: Use only the relational database (default)
@@ -529,6 +540,7 @@ ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5
 CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
 OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
 
@@ -568,3 +580,7 @@ QUEUE_MONITOR_INTERVAL=30
 # Swagger UI configuration
 SWAGGER_UI_ENABLED=true
 SWAGGER_UI_PATH=/swagger-ui.html
+
+# Whether to encrypt dataset IDs when exporting DSL files (default: true)
+# Set to false to export dataset IDs as plain text for easier cross-environment import
+DSL_EXPORT_ENCRYPT_DATASET_ID=true

api/.importlinter

@@ -0,0 +1,105 @@
+[importlinter]
+root_packages =
+    core
+    configs
+    controllers
+    models
+    tasks
+    services
+
+[importlinter:contract:workflow]
+name = Workflow
+type=layers
+layers =
+    graph_engine
+    graph_events
+    graph
+    nodes
+    node_events
+    entities
+containers =
+    core.workflow
+ignore_imports =
+    core.workflow.nodes.base.node -> core.workflow.graph_events
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_events
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_events
+
+    core.workflow.nodes.node_factory -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine.command_channels
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels
+
+[importlinter:contract:rsc]
+name = RSC
+type = layers
+layers =
+    graph_engine
+    response_coordinator
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:worker]
+name = Worker
+type = layers
+layers =
+    graph_engine
+    worker
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:graph-engine-architecture]
+name = Graph Engine Architecture
+type = layers
+layers =
+    graph_engine
+    orchestration
+    command_processing
+    event_management
+    error_handler
+    graph_traversal
+    graph_state_manager
+    worker_management
+    domain
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:domain-isolation]
+name = Domain Model Isolation
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.domain
+forbidden_modules =
+    core.workflow.graph_engine.worker_management
+    core.workflow.graph_engine.command_channels
+    core.workflow.graph_engine.layers
+    core.workflow.graph_engine.protocols
+
+[importlinter:contract:worker-management]
+name = Worker Management
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.worker_management
+forbidden_modules =
+    core.workflow.graph_engine.orchestration
+    core.workflow.graph_engine.command_processing
+    core.workflow.graph_engine.event_management
+
+
+[importlinter:contract:graph-traversal-components]
+name = Graph Traversal Components
+type = layers
+layers =
+    edge_processor
+    skip_propagator
+containers =
+    core.workflow.graph_engine.graph_traversal
+
+[importlinter:contract:command-channels]
+name = Command Channels Independence
+type = independence
+modules =
+    core.workflow.graph_engine.command_channels.in_memory_channel
+    core.workflow.graph_engine.command_channels.redis_channel

api/.ruff.toml

@@ -5,7 +5,7 @@ line-length = 120
 quote-style = "double"
 
 [lint]
-preview = false
+preview = true
 select = [
     "B",       # flake8-bugbear rules
     "C4",      # flake8-comprehensions
@@ -45,6 +45,7 @@ select = [
     "G001", # don't use str format to logging messages
     "G003", # don't use + in logging messages
     "G004", # don't use f-strings to format logging messages
+    "UP042", # use StrEnum
 ]
 
 ignore = [
@@ -64,6 +65,7 @@ ignore = [
     "B006",    # mutable-argument-default
     "B007",    # unused-loop-control-variable
     "B026",    # star-arg-unpacking-after-keyword-arg
+    "B901",    # allow return in yield
     "B903",    # class-as-data-structure
     "B904",    # raise-without-from-inside-except
     "B905",    # zip-without-explicit-strict

api/README.md

@@ -108,5 +108,5 @@ uv run celery -A app.celery beat
    ../dev/reformat               # Run all formatters and linters
    uv run ruff check --fix ./    # Fix linting issues
    uv run ruff format ./         # Format code
-   uv run mypy .                 # Type checking
+   uv run basedpyright .         # Type checking
    ```

api/app.py

@@ -1,4 +1,3 @@
-import os
 import sys
 
 
@@ -17,20 +16,20 @@ else:
     # It seems that JetBrains Python debugger does not work well with gevent,
     # so we need to disable gevent in debug mode.
     # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
-    if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
-        from gevent import monkey
+    # if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
+    # from gevent import monkey
+    #
+    # # gevent
+    # monkey.patch_all()
+    #
+    # from grpc.experimental import gevent as grpc_gevent  # type: ignore
+    #
+    # # grpc gevent
+    # grpc_gevent.init_gevent()
 
-        # gevent
-        monkey.patch_all()
-
-        from grpc.experimental import gevent as grpc_gevent  # type: ignore
-
-        # grpc gevent
-        grpc_gevent.init_gevent()
-
-        import psycogreen.gevent  # type: ignore
-
-        psycogreen.gevent.patch_psycopg()
+    # import psycogreen.gevent  # type: ignore
+    #
+    # psycogreen.gevent.patch_psycopg()
 
     from app_factory import create_app
 

api/app_factory.py

@@ -25,6 +25,9 @@ def create_flask_app_with_configs() -> DifyApp:
         # add an unique identifier to each request
         RecyclableContextVar.increment_thread_recycles()
 
+    # Capture the decorator's return value to avoid pyright reportUnusedFunction
+    _ = before_request
+
     return dify_app
 
 

api/celery_entrypoint.py

@@ -0,0 +1,22 @@
+import logging
+
+import psycogreen.gevent as pscycogreen_gevent  # type: ignore
+from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+_logger = logging.getLogger(__name__)
+
+
+def _log(message: str):
+    print(message, flush=True)
+
+
+# grpc gevent
+grpc_gevent.init_gevent()
+_log("gRPC  patched with gevent.")
+pscycogreen_gevent.patch_psycopg()
+_log("psycopg2 patched with gevent.")
+
+
+from app import app, celery
+
+__all__ = ["app", "celery"]

api/child_class.py

@@ -1,11 +0,0 @@
-from tests.integration_tests.utils.parent_class import ParentClass
-
-
-class ChildClass(ParentClass):
-    """Test child class for module import helper tests"""
-
-    def __init__(self, name):
-        super().__init__(name)
-
-    def get_name(self):
-        return f"Child: {self.name}"

api/commands.py

@@ -2,7 +2,7 @@ import base64
 import json
 import logging
 import secrets
-from typing import Any, Optional
+from typing import Any
 
 import click
 import sqlalchemy as sa
@@ -13,11 +13,13 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from configs import dify_config
 from constants.languages import languages
-from core.plugin.entities.plugin import ToolProviderID
+from core.helper import encrypter
+from core.plugin.impl.plugin import PluginInstaller
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.rag.models.document import Document
+from core.tools.entities.tool_entities import CredentialType
 from core.tools.utils.system_oauth_encryption import encrypt_system_oauth_params
 from events.app_event import app_was_created
 from extensions.ext_database import db
@@ -30,12 +32,16 @@ from models import Tenant
 from models.dataset import Dataset, DatasetCollectionBinding, DatasetMetadata, DatasetMetadataBinding, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
+from models.oauth import DatasourceOauthParamConfig, DatasourceProvider
 from models.provider import Provider, ProviderModel
+from models.provider_ids import DatasourceProviderID, ToolProviderID
+from models.source import DataSourceApiKeyAuthBinding, DataSourceOauthBinding
 from models.tools import ToolOAuthSystemClient
 from services.account_service import AccountService, RegisterService, TenantService
 from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs
 from services.plugin.data_migration import PluginDataMigration
 from services.plugin.plugin_migration import PluginMigration
+from services.plugin.plugin_service import PluginService
 from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
 
 logger = logging.getLogger(__name__)
@@ -212,7 +218,9 @@ def migrate_annotation_vector_database():
                 if not dataset_collection_binding:
                     click.echo(f"App annotation collection binding not found: {app.id}")
                     continue
-                annotations = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app.id).all()
+                annotations = db.session.scalars(
+                    select(MessageAnnotation).where(MessageAnnotation.app_id == app.id)
+                ).all()
                 dataset = Dataset(
                     id=app.id,
                     tenant_id=app.tenant_id,
@@ -367,29 +375,25 @@ def migrate_knowledge_vector_database():
                     )
                     raise e
 
-                dataset_documents = (
-                    db.session.query(DatasetDocument)
-                    .where(
+                dataset_documents = db.session.scalars(
+                    select(DatasetDocument).where(
                         DatasetDocument.dataset_id == dataset.id,
                         DatasetDocument.indexing_status == "completed",
                         DatasetDocument.enabled == True,
                         DatasetDocument.archived == False,
                     )
-                    .all()
-                )
+                ).all()
 
                 documents = []
                 segments_count = 0
                 for dataset_document in dataset_documents:
-                    segments = (
-                        db.session.query(DocumentSegment)
-                        .where(
+                    segments = db.session.scalars(
+                        select(DocumentSegment).where(
                             DocumentSegment.document_id == dataset_document.id,
                             DocumentSegment.status == "completed",
                             DocumentSegment.enabled == True,
                         )
-                        .all()
-                    )
+                    ).all()
 
                     for segment in segments:
                         document = Document(
@@ -479,12 +483,12 @@ def convert_to_agent_apps():
             click.echo(f"Converting app: {app.id}")
 
             try:
-                app.mode = AppMode.AGENT_CHAT.value
+                app.mode = AppMode.AGENT_CHAT
                 db.session.commit()
 
                 # update conversation mode to agent
                 db.session.query(Conversation).where(Conversation.app_id == app.id).update(
-                    {Conversation.mode: AppMode.AGENT_CHAT.value}
+                    {Conversation.mode: AppMode.AGENT_CHAT}
                 )
 
                 db.session.commit()
@@ -511,7 +515,7 @@ def add_qdrant_index(field: str):
         from qdrant_client.http.exceptions import UnexpectedResponse
         from qdrant_client.http.models import PayloadSchemaType
 
-        from core.rag.datasource.vdb.qdrant.qdrant_vector import QdrantConfig
+        from core.rag.datasource.vdb.qdrant.qdrant_vector import PathQdrantParams, QdrantConfig
 
         for binding in bindings:
             if dify_config.QDRANT_URL is None:
@@ -525,7 +529,21 @@ def add_qdrant_index(field: str):
                 prefer_grpc=dify_config.QDRANT_GRPC_ENABLED,
             )
             try:
-                client = qdrant_client.QdrantClient(**qdrant_config.to_qdrant_params())
+                params = qdrant_config.to_qdrant_params()
+                # Check the type before using
+                if isinstance(params, PathQdrantParams):
+                    # PathQdrantParams case
+                    client = qdrant_client.QdrantClient(path=params.path)
+                else:
+                    # UrlQdrantParams case - params is UrlQdrantParams
+                    client = qdrant_client.QdrantClient(
+                        url=params.url,
+                        api_key=params.api_key,
+                        timeout=int(params.timeout),
+                        verify=params.verify,
+                        grpc_port=params.grpc_port,
+                        prefer_grpc=params.prefer_grpc,
+                    )
                 # create payload index
                 client.create_payload_index(binding.collection_name, field, field_schema=PayloadSchemaType.KEYWORD)
                 create_count += 1
@@ -571,7 +589,7 @@ def old_metadata_migration():
         for document in documents:
             if document.doc_metadata:
                 doc_metadata = document.doc_metadata
-                for key, value in doc_metadata.items():
+                for key in doc_metadata:
                     for field in BuiltInField:
                         if field.value == key:
                             break
@@ -627,7 +645,7 @@ def old_metadata_migration():
 @click.option("--email", prompt=True, help="Tenant account email.")
 @click.option("--name", prompt=True, help="Workspace name.")
 @click.option("--language", prompt=True, help="Account language, default: en-US.")
-def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None):
+def create_tenant(email: str, language: str | None = None, name: str | None = None):
     """
     Create tenant account
     """
@@ -941,7 +959,7 @@ def clear_orphaned_file_records(force: bool):
             click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
             query = "DELETE FROM message_files WHERE id IN :ids"
             with db.engine.begin() as conn:
-                conn.execute(sa.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+                conn.execute(sa.text(query), {"ids": tuple(record["id"] for record in orphaned_message_files)})
             click.echo(
                 click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
             )
@@ -1233,15 +1251,17 @@ def _find_orphaned_draft_variables(batch_size: int = 1000) -> list[str]:
 
 def _count_orphaned_draft_variables() -> dict[str, Any]:
     """
-    Count orphaned draft variables by app.
+    Count orphaned draft variables by app, including associated file counts.
 
     Returns:
-        Dictionary with statistics about orphaned variables
+        Dictionary with statistics about orphaned variables and files
     """
-    query = """
+    # Count orphaned variables by app
+    variables_query = """
         SELECT
             wdv.app_id,
-            COUNT(*) as variable_count
+            COUNT(*) as variable_count,
+            COUNT(wdv.file_id) as file_count
         FROM workflow_draft_variables AS wdv
         WHERE NOT EXISTS(
             SELECT 1 FROM apps WHERE apps.id = wdv.app_id
@@ -1251,14 +1271,21 @@ def _count_orphaned_draft_variables() -> dict[str, Any]:
     """
 
     with db.engine.connect() as conn:
-        result = conn.execute(sa.text(query))
-        orphaned_by_app = {row[0]: row[1] for row in result}
+        result = conn.execute(sa.text(variables_query))
+        orphaned_by_app = {}
+        total_files = 0
 
-        total_orphaned = sum(orphaned_by_app.values())
+        for row in result:
+            app_id, variable_count, file_count = row
+            orphaned_by_app[app_id] = {"variables": variable_count, "files": file_count}
+            total_files += file_count
+
+        total_orphaned = sum(app_data["variables"] for app_data in orphaned_by_app.values())
         app_count = len(orphaned_by_app)
 
         return {
             "total_orphaned_variables": total_orphaned,
+            "total_orphaned_files": total_files,
             "orphaned_app_count": app_count,
             "orphaned_by_app": orphaned_by_app,
         }
@@ -1287,6 +1314,7 @@ def cleanup_orphaned_draft_variables(
     stats = _count_orphaned_draft_variables()
 
     logger.info("Found %s orphaned draft variables", stats["total_orphaned_variables"])
+    logger.info("Found %s associated offload files", stats["total_orphaned_files"])
     logger.info("Across %s non-existent apps", stats["orphaned_app_count"])
 
     if stats["total_orphaned_variables"] == 0:
@@ -1295,10 +1323,10 @@ def cleanup_orphaned_draft_variables(
 
     if dry_run:
         logger.info("DRY RUN: Would delete the following:")
-        for app_id, count in sorted(stats["orphaned_by_app"].items(), key=lambda x: x[1], reverse=True)[
+        for app_id, data in sorted(stats["orphaned_by_app"].items(), key=lambda x: x[1]["variables"], reverse=True)[
             :10
         ]:  # Show top 10
-            logger.info("  App %s: %s variables", app_id, count)
+            logger.info("  App %s: %s variables, %s files", app_id, data["variables"], data["files"])
         if len(stats["orphaned_by_app"]) > 10:
             logger.info("  ... and %s more apps", len(stats["orphaned_by_app"]) - 10)
         return
@@ -1307,7 +1335,8 @@ def cleanup_orphaned_draft_variables(
     if not force:
         click.confirm(
             f"Are you sure you want to delete {stats['total_orphaned_variables']} "
-            f"orphaned draft variables from {stats['orphaned_app_count']} apps?",
+            f"orphaned draft variables and {stats['total_orphaned_files']} associated files "
+            f"from {stats['orphaned_app_count']} apps?",
             abort=True,
         )
 
@@ -1340,3 +1369,231 @@ def cleanup_orphaned_draft_variables(
                 continue
 
     logger.info("Cleanup completed. Total deleted: %s variables across %s apps", total_deleted, processed_apps)
+
+
+@click.command("setup-datasource-oauth-client", help="Setup datasource oauth client.")
+@click.option("--provider", prompt=True, help="Provider name")
+@click.option("--client-params", prompt=True, help="Client Params")
+def setup_datasource_oauth_client(provider, client_params):
+    """
+    Setup datasource oauth client
+    """
+    provider_id = DatasourceProviderID(provider)
+    provider_name = provider_id.provider_name
+    plugin_id = provider_id.plugin_id
+
+    try:
+        # json validate
+        click.echo(click.style(f"Validating client params: {client_params}", fg="yellow"))
+        client_params_dict = TypeAdapter(dict[str, Any]).validate_json(client_params)
+        click.echo(click.style("Client params validated successfully.", fg="green"))
+    except Exception as e:
+        click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red"))
+        return
+
+    click.echo(click.style(f"Ready to delete existing oauth client params: {provider_name}", fg="yellow"))
+    deleted_count = (
+        db.session.query(DatasourceOauthParamConfig)
+        .filter_by(
+            provider=provider_name,
+            plugin_id=plugin_id,
+        )
+        .delete()
+    )
+    if deleted_count > 0:
+        click.echo(click.style(f"Deleted {deleted_count} existing oauth client params.", fg="yellow"))
+
+    click.echo(click.style(f"Ready to setup datasource oauth client: {provider_name}", fg="yellow"))
+    oauth_client = DatasourceOauthParamConfig(
+        provider=provider_name,
+        plugin_id=plugin_id,
+        system_credentials=client_params_dict,
+    )
+    db.session.add(oauth_client)
+    db.session.commit()
+    click.echo(click.style(f"provider: {provider_name}", fg="green"))
+    click.echo(click.style(f"plugin_id: {plugin_id}", fg="green"))
+    click.echo(click.style(f"params: {json.dumps(client_params_dict, indent=2, ensure_ascii=False)}", fg="green"))
+    click.echo(click.style(f"Datasource oauth client setup successfully. id: {oauth_client.id}", fg="green"))
+
+
+@click.command("transform-datasource-credentials", help="Transform datasource credentials.")
+def transform_datasource_credentials():
+    """
+    Transform datasource credentials
+    """
+    try:
+        installer_manager = PluginInstaller()
+        plugin_migration = PluginMigration()
+
+        notion_plugin_id = "langgenius/notion_datasource"
+        firecrawl_plugin_id = "langgenius/firecrawl_datasource"
+        jina_plugin_id = "langgenius/jina_datasource"
+        notion_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(notion_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        firecrawl_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(firecrawl_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        jina_plugin_unique_identifier = plugin_migration._fetch_plugin_unique_identifier(jina_plugin_id)  # pyright: ignore[reportPrivateUsage]
+        oauth_credential_type = CredentialType.OAUTH2
+        api_key_credential_type = CredentialType.API_KEY
+
+        # deal notion credentials
+        deal_notion_count = 0
+        notion_credentials = db.session.query(DataSourceOauthBinding).filter_by(provider="notion").all()
+        if notion_credentials:
+            notion_credentials_tenant_mapping: dict[str, list[DataSourceOauthBinding]] = {}
+            for notion_credential in notion_credentials:
+                tenant_id = notion_credential.tenant_id
+                if tenant_id not in notion_credentials_tenant_mapping:
+                    notion_credentials_tenant_mapping[tenant_id] = []
+                notion_credentials_tenant_mapping[tenant_id].append(notion_credential)
+            for tenant_id, notion_tenant_credentials in notion_credentials_tenant_mapping.items():
+                # check notion plugin is installed
+                installed_plugins = installer_manager.list_plugins(tenant_id)
+                installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                if notion_plugin_id not in installed_plugins_ids:
+                    if notion_plugin_unique_identifier:
+                        # install notion plugin
+                        PluginService.install_from_marketplace_pkg(tenant_id, [notion_plugin_unique_identifier])
+                auth_count = 0
+                for notion_tenant_credential in notion_tenant_credentials:
+                    auth_count += 1
+                    # get credential oauth params
+                    access_token = notion_tenant_credential.access_token
+                    # notion info
+                    notion_info = notion_tenant_credential.source_info
+                    workspace_id = notion_info.get("workspace_id")
+                    workspace_name = notion_info.get("workspace_name")
+                    workspace_icon = notion_info.get("workspace_icon")
+                    new_credentials = {
+                        "integration_secret": encrypter.encrypt_token(tenant_id, access_token),
+                        "workspace_id": workspace_id,
+                        "workspace_name": workspace_name,
+                        "workspace_icon": workspace_icon,
+                    }
+                    datasource_provider = DatasourceProvider(
+                        provider="notion_datasource",
+                        tenant_id=tenant_id,
+                        plugin_id=notion_plugin_id,
+                        auth_type=oauth_credential_type.value,
+                        encrypted_credentials=new_credentials,
+                        name=f"Auth {auth_count}",
+                        avatar_url=workspace_icon or "default",
+                        is_default=False,
+                    )
+                    db.session.add(datasource_provider)
+                    deal_notion_count += 1
+                db.session.commit()
+        # deal firecrawl credentials
+        deal_firecrawl_count = 0
+        firecrawl_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="firecrawl").all()
+        if firecrawl_credentials:
+            firecrawl_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {}
+            for firecrawl_credential in firecrawl_credentials:
+                tenant_id = firecrawl_credential.tenant_id
+                if tenant_id not in firecrawl_credentials_tenant_mapping:
+                    firecrawl_credentials_tenant_mapping[tenant_id] = []
+                firecrawl_credentials_tenant_mapping[tenant_id].append(firecrawl_credential)
+            for tenant_id, firecrawl_tenant_credentials in firecrawl_credentials_tenant_mapping.items():
+                # check firecrawl plugin is installed
+                installed_plugins = installer_manager.list_plugins(tenant_id)
+                installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                if firecrawl_plugin_id not in installed_plugins_ids:
+                    if firecrawl_plugin_unique_identifier:
+                        # install firecrawl plugin
+                        PluginService.install_from_marketplace_pkg(tenant_id, [firecrawl_plugin_unique_identifier])
+
+                auth_count = 0
+                for firecrawl_tenant_credential in firecrawl_tenant_credentials:
+                    auth_count += 1
+                    # get credential api key
+                    credentials_json = json.loads(firecrawl_tenant_credential.credentials)
+                    api_key = credentials_json.get("config", {}).get("api_key")
+                    base_url = credentials_json.get("config", {}).get("base_url")
+                    new_credentials = {
+                        "firecrawl_api_key": api_key,
+                        "base_url": base_url,
+                    }
+                    datasource_provider = DatasourceProvider(
+                        provider="firecrawl",
+                        tenant_id=tenant_id,
+                        plugin_id=firecrawl_plugin_id,
+                        auth_type=api_key_credential_type.value,
+                        encrypted_credentials=new_credentials,
+                        name=f"Auth {auth_count}",
+                        avatar_url="default",
+                        is_default=False,
+                    )
+                    db.session.add(datasource_provider)
+                    deal_firecrawl_count += 1
+                db.session.commit()
+        # deal jina credentials
+        deal_jina_count = 0
+        jina_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="jinareader").all()
+        if jina_credentials:
+            jina_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {}
+            for jina_credential in jina_credentials:
+                tenant_id = jina_credential.tenant_id
+                if tenant_id not in jina_credentials_tenant_mapping:
+                    jina_credentials_tenant_mapping[tenant_id] = []
+                jina_credentials_tenant_mapping[tenant_id].append(jina_credential)
+            for tenant_id, jina_tenant_credentials in jina_credentials_tenant_mapping.items():
+                # check jina plugin is installed
+                installed_plugins = installer_manager.list_plugins(tenant_id)
+                installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins]
+                if jina_plugin_id not in installed_plugins_ids:
+                    if jina_plugin_unique_identifier:
+                        # install jina plugin
+                        print(jina_plugin_unique_identifier)
+                        PluginService.install_from_marketplace_pkg(tenant_id, [jina_plugin_unique_identifier])
+
+                auth_count = 0
+                for jina_tenant_credential in jina_tenant_credentials:
+                    auth_count += 1
+                    # get credential api key
+                    credentials_json = json.loads(jina_tenant_credential.credentials)
+                    api_key = credentials_json.get("config", {}).get("api_key")
+                    new_credentials = {
+                        "integration_secret": api_key,
+                    }
+                    datasource_provider = DatasourceProvider(
+                        provider="jina",
+                        tenant_id=tenant_id,
+                        plugin_id=jina_plugin_id,
+                        auth_type=api_key_credential_type.value,
+                        encrypted_credentials=new_credentials,
+                        name=f"Auth {auth_count}",
+                        avatar_url="default",
+                        is_default=False,
+                    )
+                    db.session.add(datasource_provider)
+                    deal_jina_count += 1
+                db.session.commit()
+    except Exception as e:
+        click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red"))
+        return
+    click.echo(click.style(f"Transforming notion successfully. deal_notion_count: {deal_notion_count}", fg="green"))
+    click.echo(
+        click.style(f"Transforming firecrawl successfully. deal_firecrawl_count: {deal_firecrawl_count}", fg="green")
+    )
+    click.echo(click.style(f"Transforming jina successfully. deal_jina_count: {deal_jina_count}", fg="green"))
+
+
+@click.command("install-rag-pipeline-plugins", help="Install rag pipeline plugins.")
+@click.option(
+    "--input_file", prompt=True, help="The file to store the extracted unique identifiers.", default="plugins.jsonl"
+)
+@click.option(
+    "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
+)
+@click.option("--workers", prompt=True, help="The number of workers to install plugins.", default=100)
+def install_rag_pipeline_plugins(input_file, output_file, workers):
+    """
+    Install rag pipeline plugins
+    """
+    click.echo(click.style("Installing rag pipeline plugins", fg="yellow"))
+    plugin_migration = PluginMigration()
+    plugin_migration.install_rag_pipeline_plugins(
+        input_file,
+        output_file,
+        workers,
+    )
+    click.echo(click.style("Installing rag pipeline plugins successfully", fg="green"))

api/configs/__init__.py

@@ -1,3 +1,3 @@
 from .app_config import DifyConfig
 
-dify_config = DifyConfig()
+dify_config = DifyConfig()  # type: ignore

api/configs/extra/notion_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,28 +7,28 @@ class NotionConfig(BaseSettings):
     Configuration settings for Notion integration
     """
 
-    NOTION_CLIENT_ID: Optional[str] = Field(
+    NOTION_CLIENT_ID: str | None = Field(
         description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_CLIENT_SECRET: Optional[str] = Field(
+    NOTION_CLIENT_SECRET: str | None = Field(
         description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+    NOTION_INTEGRATION_TYPE: str | None = Field(
         description="Type of Notion integration."
         " Set to 'internal' for internal integrations, or None for public integrations.",
         default=None,
     )
 
-    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+    NOTION_INTERNAL_SECRET: str | None = Field(
         description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+    NOTION_INTEGRATION_TOKEN: str | None = Field(
         description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
         default=None,
     )

api/configs/extra/sentry_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeFloat
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class SentryConfig(BaseSettings):
     Configuration settings for Sentry error tracking and performance monitoring
     """
 
-    SENTRY_DSN: Optional[str] = Field(
+    SENTRY_DSN: str | None = Field(
         description="Sentry Data Source Name (DSN)."
         " This is the unique identifier of your Sentry project, used to send events to the correct project.",
         default=None,

api/configs/feature/__init__.py

@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import (
     AliasChoices,
@@ -31,6 +31,12 @@ class SecurityConfig(BaseSettings):
         description="Duration in minutes for which a password reset token remains valid",
         default=5,
     )
+
+    EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
+        description="Duration in minutes for which a email register token remains valid",
+        default=5,
+    )
+
     CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
         description="Duration in minutes for which a change email token remains valid",
         default=5,
@@ -51,7 +57,7 @@ class SecurityConfig(BaseSettings):
         default=False,
     )
 
-    ADMIN_API_KEY: Optional[str] = Field(
+    ADMIN_API_KEY: str | None = Field(
         description="admin api key for authentication",
         default=None,
     )
@@ -91,17 +97,17 @@ class CodeExecutionSandboxConfig(BaseSettings):
         default="dify-sandbox",
     )
 
-    CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_CONNECT_TIMEOUT: float | None = Field(
         description="Connection timeout in seconds for code execution requests",
         default=10.0,
     )
 
-    CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_READ_TIMEOUT: float | None = Field(
         description="Read timeout in seconds for code execution requests",
         default=60.0,
     )
 
-    CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_WRITE_TIMEOUT: float | None = Field(
         description="Write timeout in seconds for code execution request",
         default=10.0,
     )
@@ -362,17 +368,17 @@ class HttpConfig(BaseSettings):
         default=3,
     )
 
-    SSRF_PROXY_ALL_URL: Optional[str] = Field(
+    SSRF_PROXY_ALL_URL: str | None = Field(
         description="Proxy URL for HTTP or HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTP_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTP_URL: str | None = Field(
         description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTPS_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTPS_URL: str | None = Field(
         description="Proxy URL for HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
@@ -414,7 +420,7 @@ class InnerAPIConfig(BaseSettings):
         default=False,
     )
 
-    INNER_API_KEY: Optional[str] = Field(
+    INNER_API_KEY: str | None = Field(
         description="API key for accessing the internal API",
         default=None,
     )
@@ -430,7 +436,7 @@ class LoggingConfig(BaseSettings):
         default="INFO",
     )
 
-    LOG_FILE: Optional[str] = Field(
+    LOG_FILE: str | None = Field(
         description="File path for log output.",
         default=None,
     )
@@ -450,12 +456,12 @@ class LoggingConfig(BaseSettings):
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
     )
 
-    LOG_DATEFORMAT: Optional[str] = Field(
+    LOG_DATEFORMAT: str | None = Field(
         description="Date format string for log timestamps",
         default=None,
     )
 
-    LOG_TZ: Optional[str] = Field(
+    LOG_TZ: str | None = Field(
         description="Timezone for log timestamps (e.g., 'America/New_York')",
         default="UTC",
     )
@@ -499,6 +505,22 @@ class UpdateConfig(BaseSettings):
     )
 
 
+class WorkflowVariableTruncationConfig(BaseSettings):
+    WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE: PositiveInt = Field(
+        # 100KB
+        1024_000,
+        description="Maximum size for variable to trigger final truncation.",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH: PositiveInt = Field(
+        100000,
+        description="maximum length for string to trigger tuncation, measure in number of characters",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_ARRAY_LENGTH: PositiveInt = Field(
+        1000,
+        description="maximum length for array to trigger truncation.",
+    )
+
+
 class WorkflowConfig(BaseSettings):
     """
     Configuration for workflow execution
@@ -529,6 +551,28 @@ class WorkflowConfig(BaseSettings):
         default=200 * 1024,
     )
 
+    # GraphEngine Worker Pool Configuration
+    GRAPH_ENGINE_MIN_WORKERS: PositiveInt = Field(
+        description="Minimum number of workers per GraphEngine instance",
+        default=1,
+    )
+
+    GRAPH_ENGINE_MAX_WORKERS: PositiveInt = Field(
+        description="Maximum number of workers per GraphEngine instance",
+        default=10,
+    )
+
+    GRAPH_ENGINE_SCALE_UP_THRESHOLD: PositiveInt = Field(
+        description="Queue depth threshold that triggers worker scale up",
+        default=3,
+    )
+
+    GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME: float = Field(
+        description="Seconds of idle time before scaling down workers",
+        default=5.0,
+        ge=0.1,
+    )
+
 
 class WorkflowNodeExecutionConfig(BaseSettings):
     """
@@ -589,22 +633,22 @@ class AuthConfig(BaseSettings):
         default="/console/api/oauth/authorize",
     )
 
-    GITHUB_CLIENT_ID: Optional[str] = Field(
+    GITHUB_CLIENT_ID: str | None = Field(
         description="GitHub OAuth client ID",
         default=None,
     )
 
-    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+    GITHUB_CLIENT_SECRET: str | None = Field(
         description="GitHub OAuth client secret",
         default=None,
     )
 
-    GOOGLE_CLIENT_ID: Optional[str] = Field(
+    GOOGLE_CLIENT_ID: str | None = Field(
         description="Google OAuth client ID",
         default=None,
     )
 
-    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+    GOOGLE_CLIENT_SECRET: str | None = Field(
         description="Google OAuth client secret",
         default=None,
     )
@@ -639,6 +683,11 @@ class AuthConfig(BaseSettings):
         default=86400,
     )
 
+    EMAIL_REGISTER_LOCKOUT_DURATION: PositiveInt = Field(
+        description="Time (in seconds) a user must wait before retrying email register after exceeding the rate limit.",
+        default=86400,
+    )
+
 
 class ModerationConfig(BaseSettings):
     """
@@ -667,42 +716,42 @@ class MailConfig(BaseSettings):
     Configuration for email services
     """
 
-    MAIL_TYPE: Optional[str] = Field(
+    MAIL_TYPE: str | None = Field(
         description="Email service provider type ('smtp' or 'resend' or 'sendGrid), default to None.",
         default=None,
     )
 
-    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+    MAIL_DEFAULT_SEND_FROM: str | None = Field(
         description="Default email address to use as the sender",
         default=None,
     )
 
-    RESEND_API_KEY: Optional[str] = Field(
+    RESEND_API_KEY: str | None = Field(
         description="API key for Resend email service",
         default=None,
     )
 
-    RESEND_API_URL: Optional[str] = Field(
+    RESEND_API_URL: str | None = Field(
         description="API URL for Resend email service",
         default=None,
     )
 
-    SMTP_SERVER: Optional[str] = Field(
+    SMTP_SERVER: str | None = Field(
         description="SMTP server hostname",
         default=None,
     )
 
-    SMTP_PORT: Optional[int] = Field(
+    SMTP_PORT: int | None = Field(
         description="SMTP server port number",
         default=465,
     )
 
-    SMTP_USERNAME: Optional[str] = Field(
+    SMTP_USERNAME: str | None = Field(
         description="Username for SMTP authentication",
         default=None,
     )
 
-    SMTP_PASSWORD: Optional[str] = Field(
+    SMTP_PASSWORD: str | None = Field(
         description="Password for SMTP authentication",
         default=None,
     )
@@ -722,7 +771,7 @@ class MailConfig(BaseSettings):
         default=50,
     )
 
-    SENDGRID_API_KEY: Optional[str] = Field(
+    SENDGRID_API_KEY: str | None = Field(
         description="API key for SendGrid service",
         default=None,
     )
@@ -745,17 +794,17 @@ class RagEtlConfig(BaseSettings):
         default="database",
     )
 
-    UNSTRUCTURED_API_URL: Optional[str] = Field(
+    UNSTRUCTURED_API_URL: str | None = Field(
         description="API URL for Unstructured.io service",
         default=None,
     )
 
-    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+    UNSTRUCTURED_API_KEY: str | None = Field(
         description="API key for Unstructured.io service",
         default="",
     )
 
-    SCARF_NO_ANALYTICS: Optional[str] = Field(
+    SCARF_NO_ANALYTICS: str | None = Field(
         description="This is about whether to disable Scarf analytics in Unstructured library.",
         default="false",
     )
@@ -796,6 +845,11 @@ class DataSetConfig(BaseSettings):
         default=30,
     )
 
+    DSL_EXPORT_ENCRYPT_DATASET_ID: bool = Field(
+        description="Enable or disable dataset ID encryption when exporting DSL files",
+        default=True,
+    )
+
 
 class WorkspaceConfig(BaseSettings):
     """
@@ -1025,5 +1079,6 @@ class FeatureConfig(
     CeleryBeatConfig,
     CeleryScheduleTasksConfig,
     WorkflowLogConfig,
+    WorkflowVariableTruncationConfig,
 ):
     pass

api/configs/feature/hosted_service/__init__.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt
 from pydantic_settings import BaseSettings
 
@@ -40,17 +38,17 @@ class HostedOpenAiConfig(BaseSettings):
     Configuration for hosted OpenAI service
     """
 
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted OpenAI service",
         default=None,
     )
 
-    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted OpenAI API",
         default=None,
     )
 
-    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
+    HOSTED_OPENAI_API_ORGANIZATION: str | None = Field(
         description="Organization ID for hosted OpenAI service",
         default=None,
     )
@@ -110,12 +108,12 @@ class HostedAzureOpenAiConfig(BaseSettings):
         default=False,
     )
 
-    HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted Azure OpenAI service",
         default=None,
     )
 
-    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted Azure OpenAI API",
         default=None,
     )
@@ -131,12 +129,12 @@ class HostedAnthropicConfig(BaseSettings):
     Configuration for hosted Anthropic service
     """
 
-    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_BASE: str | None = Field(
         description="Base URL for hosted Anthropic API",
         default=None,
     )
 
-    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_KEY: str | None = Field(
         description="API key for hosted Anthropic service",
         default=None,
     )
@@ -222,11 +220,28 @@ class HostedFetchAppTemplateConfig(BaseSettings):
     )
 
 
+class HostedFetchPipelineTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching pipeline templates
+    """
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching pipeline templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote pipeline templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
 class HostedServiceConfig(
     # place the configs in alphabet order
     HostedAnthropicConfig,
     HostedAzureOpenAiConfig,
     HostedFetchAppTemplateConfig,
+    HostedFetchPipelineTemplateConfig,
     HostedMinmaxConfig,
     HostedOpenAiConfig,
     HostedSparkConfig,

api/configs/middleware/__init__.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 from urllib.parse import parse_qsl, quote_plus
 
 from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
@@ -78,18 +78,18 @@ class StorageConfig(BaseSettings):
 
 
 class VectorStoreConfig(BaseSettings):
-    VECTOR_STORE: Optional[str] = Field(
+    VECTOR_STORE: str | None = Field(
         description="Type of vector store to use for efficient similarity search."
         " Set to None if not using a vector store.",
         default=None,
     )
 
-    VECTOR_STORE_WHITELIST_ENABLE: Optional[bool] = Field(
+    VECTOR_STORE_WHITELIST_ENABLE: bool | None = Field(
         description="Enable whitelist for vector store.",
         default=False,
     )
 
-    VECTOR_INDEX_NAME_PREFIX: Optional[str] = Field(
+    VECTOR_INDEX_NAME_PREFIX: str | None = Field(
         description="Prefix used to create collection name in vector database",
         default="Vector_index",
     )
@@ -225,26 +225,26 @@ class CeleryConfig(DatabaseConfig):
         default="redis",
     )
 
-    CELERY_BROKER_URL: Optional[str] = Field(
+    CELERY_BROKER_URL: str | None = Field(
         description="URL of the message broker for Celery tasks.",
         default=None,
     )
 
-    CELERY_USE_SENTINEL: Optional[bool] = Field(
+    CELERY_USE_SENTINEL: bool | None = Field(
         description="Whether to use Redis Sentinel for high availability.",
         default=False,
     )
 
-    CELERY_SENTINEL_MASTER_NAME: Optional[str] = Field(
+    CELERY_SENTINEL_MASTER_NAME: str | None = Field(
         description="Name of the Redis Sentinel master.",
         default=None,
     )
 
-    CELERY_SENTINEL_PASSWORD: Optional[str] = Field(
+    CELERY_SENTINEL_PASSWORD: str | None = Field(
         description="Password of the Redis Sentinel master.",
         default=None,
     )
-    CELERY_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    CELERY_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Timeout for Redis Sentinel socket operations in seconds.",
         default=0.1,
     )
@@ -268,12 +268,12 @@ class InternalTestConfig(BaseSettings):
     Configuration settings for Internal Test
     """
 
-    AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
+    AWS_SECRET_ACCESS_KEY: str | None = Field(
         description="Internal test AWS secret access key",
         default=None,
     )
 
-    AWS_ACCESS_KEY_ID: Optional[str] = Field(
+    AWS_ACCESS_KEY_ID: str | None = Field(
         description="Internal test AWS access key ID",
         default=None,
     )
@@ -284,15 +284,15 @@ class DatasetQueueMonitorConfig(BaseSettings):
     Configuration settings for Dataset Queue Monitor
     """
 
-    QUEUE_MONITOR_THRESHOLD: Optional[NonNegativeInt] = Field(
+    QUEUE_MONITOR_THRESHOLD: NonNegativeInt | None = Field(
         description="Threshold for dataset queue monitor",
         default=200,
     )
-    QUEUE_MONITOR_ALERT_EMAILS: Optional[str] = Field(
+    QUEUE_MONITOR_ALERT_EMAILS: str | None = Field(
         description="Emails for dataset queue monitor alert, separated by commas",
         default=None,
     )
-    QUEUE_MONITOR_INTERVAL: Optional[NonNegativeFloat] = Field(
+    QUEUE_MONITOR_INTERVAL: NonNegativeFloat | None = Field(
         description="Interval for dataset queue monitor in minutes",
         default=30,
     )
@@ -300,8 +300,7 @@ class DatasetQueueMonitorConfig(BaseSettings):
 
 class MiddlewareConfig(
     # place the configs in alphabet order
-    CeleryConfig,
-    DatabaseConfig,
+    CeleryConfig,  # Note: CeleryConfig already inherits from DatabaseConfig
     KeywordStoreConfig,
     RedisConfig,
     # configs of storage and storage providers

api/configs/middleware/cache/redis_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -19,12 +17,12 @@ class RedisConfig(BaseSettings):
         default=6379,
     )
 
-    REDIS_USERNAME: Optional[str] = Field(
+    REDIS_USERNAME: str | None = Field(
         description="Username for Redis authentication (if required)",
         default=None,
     )
 
-    REDIS_PASSWORD: Optional[str] = Field(
+    REDIS_PASSWORD: str | None = Field(
         description="Password for Redis authentication (if required)",
         default=None,
     )
@@ -44,47 +42,47 @@ class RedisConfig(BaseSettings):
         default="CERT_NONE",
     )
 
-    REDIS_SSL_CA_CERTS: Optional[str] = Field(
+    REDIS_SSL_CA_CERTS: str | None = Field(
         description="Path to the CA certificate file for SSL verification",
         default=None,
     )
 
-    REDIS_SSL_CERTFILE: Optional[str] = Field(
+    REDIS_SSL_CERTFILE: str | None = Field(
         description="Path to the client certificate file for SSL authentication",
         default=None,
     )
 
-    REDIS_SSL_KEYFILE: Optional[str] = Field(
+    REDIS_SSL_KEYFILE: str | None = Field(
         description="Path to the client private key file for SSL authentication",
         default=None,
     )
 
-    REDIS_USE_SENTINEL: Optional[bool] = Field(
+    REDIS_USE_SENTINEL: bool | None = Field(
         description="Enable Redis Sentinel mode for high availability",
         default=False,
     )
 
-    REDIS_SENTINELS: Optional[str] = Field(
+    REDIS_SENTINELS: str | None = Field(
         description="Comma-separated list of Redis Sentinel nodes (host:port)",
         default=None,
     )
 
-    REDIS_SENTINEL_SERVICE_NAME: Optional[str] = Field(
+    REDIS_SENTINEL_SERVICE_NAME: str | None = Field(
         description="Name of the Redis Sentinel service to monitor",
         default=None,
     )
 
-    REDIS_SENTINEL_USERNAME: Optional[str] = Field(
+    REDIS_SENTINEL_USERNAME: str | None = Field(
         description="Username for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_PASSWORD: Optional[str] = Field(
+    REDIS_SENTINEL_PASSWORD: str | None = Field(
         description="Password for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    REDIS_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )
@@ -94,12 +92,12 @@ class RedisConfig(BaseSettings):
         default=False,
     )
 
-    REDIS_CLUSTERS: Optional[str] = Field(
+    REDIS_CLUSTERS: str | None = Field(
         description="Comma-separated list of Redis Clusters nodes (host:port)",
         default=None,
     )
 
-    REDIS_CLUSTERS_PASSWORD: Optional[str] = Field(
+    REDIS_CLUSTERS_PASSWORD: str | None = Field(
         description="Password for Redis Clusters authentication (if required)",
         default=None,
     )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,37 +7,37 @@ class AliyunOSSStorageConfig(BaseSettings):
     Configuration settings for Aliyun Object Storage Service (OSS)
     """
 
-    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+    ALIYUN_OSS_BUCKET_NAME: str | None = Field(
         description="Name of the Aliyun OSS bucket to store and retrieve objects",
         default=None,
     )
 
-    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+    ALIYUN_OSS_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+    ALIYUN_OSS_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+    ALIYUN_OSS_ENDPOINT: str | None = Field(
         description="URL of the Aliyun OSS endpoint for your chosen region",
         default=None,
     )
 
-    ALIYUN_OSS_REGION: Optional[str] = Field(
+    ALIYUN_OSS_REGION: str | None = Field(
         description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
         default=None,
     )
 
-    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+    ALIYUN_OSS_AUTH_VERSION: str | None = Field(
         description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
         default=None,
     )
 
-    ALIYUN_OSS_PATH: Optional[str] = Field(
+    ALIYUN_OSS_PATH: str | None = Field(
         description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
         default=None,
     )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import Field
 from pydantic_settings import BaseSettings
@@ -9,27 +9,27 @@ class S3StorageConfig(BaseSettings):
     Configuration settings for S3-compatible object storage
     """
 
-    S3_ENDPOINT: Optional[str] = Field(
+    S3_ENDPOINT: str | None = Field(
         description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
         default=None,
     )
 
-    S3_REGION: Optional[str] = Field(
+    S3_REGION: str | None = Field(
         description="Region where the S3 bucket is located (e.g., 'us-east-1')",
         default=None,
     )
 
-    S3_BUCKET_NAME: Optional[str] = Field(
+    S3_BUCKET_NAME: str | None = Field(
         description="Name of the S3 bucket to store and retrieve objects",
         default=None,
     )
 
-    S3_ACCESS_KEY: Optional[str] = Field(
+    S3_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with the S3 service",
         default=None,
     )
 
-    S3_SECRET_KEY: Optional[str] = Field(
+    S3_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with the S3 service",
         default=None,
     )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class AzureBlobStorageConfig(BaseSettings):
     Configuration settings for Azure Blob Storage
     """
 
-    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_NAME: str | None = Field(
         description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_KEY: str | None = Field(
         description="Access key for authenticating with the Azure Storage account",
         default=None,
     )
 
-    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+    AZURE_BLOB_CONTAINER_NAME: str | None = Field(
         description="Name of the Azure Blob container to store and retrieve objects",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_URL: str | None = Field(
         description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
         default=None,
     )

api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class BaiduOBSStorageConfig(BaseSettings):
     Configuration settings for Baidu Object Storage Service (OBS)
     """
 
-    BAIDU_OBS_BUCKET_NAME: Optional[str] = Field(
+    BAIDU_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    BAIDU_OBS_ACCESS_KEY: Optional[str] = Field(
+    BAIDU_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_SECRET_KEY: Optional[str] = Field(
+    BAIDU_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_ENDPOINT: Optional[str] = Field(
+    BAIDU_OBS_ENDPOINT: str | None = Field(
         description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
         default=None,
     )

api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -1,7 +1,5 @@
 """ClickZetta Volume Storage Configuration"""
 
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ from pydantic_settings import BaseSettings
 class ClickZettaVolumeStorageConfig(BaseSettings):
     """Configuration for ClickZetta Volume storage."""
 
-    CLICKZETTA_VOLUME_USERNAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_USERNAME: str | None = Field(
         description="Username for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_VOLUME_PASSWORD: str | None = Field(
         description="Password for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_VOLUME_INSTANCE: str | None = Field(
         description="ClickZetta instance identifier",
         default=None,
     )
@@ -49,7 +47,7 @@ class ClickZettaVolumeStorageConfig(BaseSettings):
         default="user",
     )
 
-    CLICKZETTA_VOLUME_NAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_NAME: str | None = Field(
         description="ClickZetta volume name for external volumes",
         default=None,
     )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class GoogleCloudStorageConfig(BaseSettings):
     Configuration settings for Google Cloud Storage
     """
 
-    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+    GOOGLE_STORAGE_BUCKET_NAME: str | None = Field(
         description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
         default=None,
     )
 
-    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: str | None = Field(
         description="Base64-encoded JSON key file for Google Cloud service account authentication",
         default=None,
     )

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class HuaweiCloudOBSStorageConfig(BaseSettings):
     Configuration settings for Huawei Cloud Object Storage Service (OBS)
     """
 
-    HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
+    HUAWEI_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
+    HUAWEI_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
+    HUAWEI_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SERVER: Optional[str] = Field(
+    HUAWEI_OBS_SERVER: str | None = Field(
         description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
         default=None,
     )

api/configs/middleware/storage/oci_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OCIStorageConfig(BaseSettings):
     Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
     """
 
-    OCI_ENDPOINT: Optional[str] = Field(
+    OCI_ENDPOINT: str | None = Field(
         description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
         default=None,
     )
 
-    OCI_REGION: Optional[str] = Field(
+    OCI_REGION: str | None = Field(
         description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
         default=None,
     )
 
-    OCI_BUCKET_NAME: Optional[str] = Field(
+    OCI_BUCKET_NAME: str | None = Field(
         description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
         default=None,
     )
 
-    OCI_ACCESS_KEY: Optional[str] = Field(
+    OCI_ACCESS_KEY: str | None = Field(
         description="Access key (also known as API key) for authenticating with OCI Object Storage",
         default=None,
     )
 
-    OCI_SECRET_KEY: Optional[str] = Field(
+    OCI_SECRET_KEY: str | None = Field(
         description="Secret key associated with the access key for authenticating with OCI Object Storage",
         default=None,
     )

api/configs/middleware/storage/supabase_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class SupabaseStorageConfig(BaseSettings):
     Configuration settings for Supabase Object Storage Service
     """
 
-    SUPABASE_BUCKET_NAME: Optional[str] = Field(
+    SUPABASE_BUCKET_NAME: str | None = Field(
         description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
         default=None,
     )
 
-    SUPABASE_API_KEY: Optional[str] = Field(
+    SUPABASE_API_KEY: str | None = Field(
         description="API KEY for authenticating with Supabase",
         default=None,
     )
 
-    SUPABASE_URL: Optional[str] = Field(
+    SUPABASE_URL: str | None = Field(
         description="URL of the Supabase",
         default=None,
     )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TencentCloudCOSStorageConfig(BaseSettings):
     Configuration settings for Tencent Cloud Object Storage (COS)
     """
 
-    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+    TENCENT_COS_BUCKET_NAME: str | None = Field(
         description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
         default=None,
     )
 
-    TENCENT_COS_REGION: Optional[str] = Field(
+    TENCENT_COS_REGION: str | None = Field(
         description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
         default=None,
     )
 
-    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+    TENCENT_COS_SECRET_ID: str | None = Field(
         description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+    TENCENT_COS_SECRET_KEY: str | None = Field(
         description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SCHEME: Optional[str] = Field(
+    TENCENT_COS_SCHEME: str | None = Field(
         description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class VolcengineTOSStorageConfig(BaseSettings):
     Configuration settings for Volcengine Tinder Object Storage (TOS)
     """
 
-    VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
+    VOLCENGINE_TOS_BUCKET_NAME: str | None = Field(
         description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
         default=None,
     )
 
-    VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
+    VOLCENGINE_TOS_ENDPOINT: str | None = Field(
         description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
         default=None,
     )
 
-    VOLCENGINE_TOS_REGION: Optional[str] = Field(
+    VOLCENGINE_TOS_REGION: str | None = Field(
         description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
         default=None,
     )

api/configs/middleware/vdb/analyticdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -11,37 +9,37 @@ class AnalyticdbConfig(BaseSettings):
     https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
     """
 
-    ANALYTICDB_KEY_ID: Optional[str] = Field(
+    ANALYTICDB_KEY_ID: str | None = Field(
         default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
     )
-    ANALYTICDB_KEY_SECRET: Optional[str] = Field(
+    ANALYTICDB_KEY_SECRET: str | None = Field(
         default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
     )
-    ANALYTICDB_REGION_ID: Optional[str] = Field(
+    ANALYTICDB_REGION_ID: str | None = Field(
         default=None,
         description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
     )
-    ANALYTICDB_INSTANCE_ID: Optional[str] = Field(
+    ANALYTICDB_INSTANCE_ID: str | None = Field(
         default=None,
         description="The unique identifier of the AnalyticDB instance you want to connect to.",
     )
-    ANALYTICDB_ACCOUNT: Optional[str] = Field(
+    ANALYTICDB_ACCOUNT: str | None = Field(
         default=None,
         description="The account name used to log in to the AnalyticDB instance"
         " (usually the initial account created with the instance).",
     )
-    ANALYTICDB_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_PASSWORD: str | None = Field(
         default=None, description="The password associated with the AnalyticDB account for database authentication."
     )
-    ANALYTICDB_NAMESPACE: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE: str | None = Field(
         default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
     )
-    ANALYTICDB_NAMESPACE_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE_PASSWORD: str | None = Field(
         default=None,
         description="The password for accessing the specified namespace within the AnalyticDB instance"
         " (if namespace feature is enabled).",
     )
-    ANALYTICDB_HOST: Optional[str] = Field(
+    ANALYTICDB_HOST: str | None = Field(
         default=None, description="The host of the AnalyticDB instance you want to connect to."
     )
     ANALYTICDB_PORT: PositiveInt = Field(

api/configs/middleware/vdb/baidu_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class BaiduVectorDBConfig(BaseSettings):
     Configuration settings for Baidu Vector Database
     """
 
-    BAIDU_VECTOR_DB_ENDPOINT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ENDPOINT: str | None = Field(
         description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class BaiduVectorDBConfig(BaseSettings):
         default=30000,
     )
 
-    BAIDU_VECTOR_DB_ACCOUNT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ACCOUNT: str | None = Field(
         description="Account for authenticating with the Baidu Vector Database",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_API_KEY: Optional[str] = Field(
+    BAIDU_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Baidu Vector Database service",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_DATABASE: Optional[str] = Field(
+    BAIDU_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Baidu Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/chroma_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class ChromaConfig(BaseSettings):
     Configuration settings for Chroma vector database
     """
 
-    CHROMA_HOST: Optional[str] = Field(
+    CHROMA_HOST: str | None = Field(
         description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
         default=None,
     )
@@ -19,22 +17,22 @@ class ChromaConfig(BaseSettings):
         default=8000,
     )
 
-    CHROMA_TENANT: Optional[str] = Field(
+    CHROMA_TENANT: str | None = Field(
         description="Tenant identifier for multi-tenancy support in Chroma",
         default=None,
     )
 
-    CHROMA_DATABASE: Optional[str] = Field(
+    CHROMA_DATABASE: str | None = Field(
         description="Name of the Chroma database to connect to",
         default=None,
     )
 
-    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+    CHROMA_AUTH_PROVIDER: str | None = Field(
         description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
         default=None,
     )
 
-    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+    CHROMA_AUTH_CREDENTIALS: str | None = Field(
         description="Authentication credentials for Chroma (format depends on the auth provider)",
         default=None,
     )

api/configs/middleware/vdb/clickzetta_config.py

@@ -1,69 +1,68 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class ClickzettaConfig(BaseModel):
+class ClickzettaConfig(BaseSettings):
     """
     Clickzetta Lakehouse vector database configuration
     """
 
-    CLICKZETTA_USERNAME: Optional[str] = Field(
+    CLICKZETTA_USERNAME: str | None = Field(
         description="Username for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_PASSWORD: str | None = Field(
         description="Password for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_INSTANCE: str | None = Field(
         description="Clickzetta Lakehouse instance ID",
         default=None,
     )
 
-    CLICKZETTA_SERVICE: Optional[str] = Field(
+    CLICKZETTA_SERVICE: str | None = Field(
         description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
         default="api.clickzetta.com",
     )
 
-    CLICKZETTA_WORKSPACE: Optional[str] = Field(
+    CLICKZETTA_WORKSPACE: str | None = Field(
         description="Clickzetta workspace name",
         default="default",
     )
 
-    CLICKZETTA_VCLUSTER: Optional[str] = Field(
+    CLICKZETTA_VCLUSTER: str | None = Field(
         description="Clickzetta virtual cluster name",
         default="default_ap",
     )
 
-    CLICKZETTA_SCHEMA: Optional[str] = Field(
+    CLICKZETTA_SCHEMA: str | None = Field(
         description="Database schema name in Clickzetta",
         default="public",
     )
 
-    CLICKZETTA_BATCH_SIZE: Optional[int] = Field(
+    CLICKZETTA_BATCH_SIZE: int | None = Field(
         description="Batch size for bulk insert operations",
         default=100,
     )
 
-    CLICKZETTA_ENABLE_INVERTED_INDEX: Optional[bool] = Field(
+    CLICKZETTA_ENABLE_INVERTED_INDEX: bool | None = Field(
         description="Enable inverted index for full-text search capabilities",
         default=True,
     )
 
-    CLICKZETTA_ANALYZER_TYPE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_TYPE: str | None = Field(
         description="Analyzer type for full-text search: keyword, english, chinese, unicode",
         default="chinese",
     )
 
-    CLICKZETTA_ANALYZER_MODE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_MODE: str | None = Field(
         description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
         default="smart",
     )
 
-    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: Optional[str] = Field(
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: str | None = Field(
         description="Distance function for vector similarity: l2_distance or cosine_distance",
         default="cosine_distance",
     )

api/configs/middleware/vdb/couchbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class CouchbaseConfig(BaseSettings):
     Couchbase configs
     """
 
-    COUCHBASE_CONNECTION_STRING: Optional[str] = Field(
+    COUCHBASE_CONNECTION_STRING: str | None = Field(
         description="COUCHBASE connection string",
         default=None,
     )
 
-    COUCHBASE_USER: Optional[str] = Field(
+    COUCHBASE_USER: str | None = Field(
         description="COUCHBASE user",
         default=None,
     )
 
-    COUCHBASE_PASSWORD: Optional[str] = Field(
+    COUCHBASE_PASSWORD: str | None = Field(
         description="COUCHBASE password",
         default=None,
     )
 
-    COUCHBASE_BUCKET_NAME: Optional[str] = Field(
+    COUCHBASE_BUCKET_NAME: str | None = Field(
         description="COUCHBASE bucket name",
         default=None,
     )
 
-    COUCHBASE_SCOPE_NAME: Optional[str] = Field(
+    COUCHBASE_SCOPE_NAME: str | None = Field(
         description="COUCHBASE scope name",
         default=None,
     )

api/configs/middleware/vdb/elasticsearch_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt, model_validator
 from pydantic_settings import BaseSettings
 
@@ -10,7 +8,7 @@ class ElasticsearchConfig(BaseSettings):
     Can load from environment variables or .env files.
     """
 
-    ELASTICSEARCH_HOST: Optional[str] = Field(
+    ELASTICSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
         default="127.0.0.1",
     )
@@ -20,30 +18,28 @@ class ElasticsearchConfig(BaseSettings):
         default=9200,
     )
 
-    ELASTICSEARCH_USERNAME: Optional[str] = Field(
+    ELASTICSEARCH_USERNAME: str | None = Field(
         description="Username for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
-    ELASTICSEARCH_PASSWORD: Optional[str] = Field(
+    ELASTICSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
     # Elastic Cloud (optional)
-    ELASTICSEARCH_USE_CLOUD: Optional[bool] = Field(
+    ELASTICSEARCH_USE_CLOUD: bool | None = Field(
         description="Set to True to use Elastic Cloud instead of self-hosted Elasticsearch", default=False
     )
-    ELASTICSEARCH_CLOUD_URL: Optional[str] = Field(
+    ELASTICSEARCH_CLOUD_URL: str | None = Field(
         description="Full URL for Elastic Cloud deployment (e.g., 'https://example.es.region.aws.found.io:443')",
         default=None,
     )
-    ELASTICSEARCH_API_KEY: Optional[str] = Field(
-        description="API key for authenticating with Elastic Cloud", default=None
-    )
+    ELASTICSEARCH_API_KEY: str | None = Field(description="API key for authenticating with Elastic Cloud", default=None)
 
     # Common options
-    ELASTICSEARCH_CA_CERTS: Optional[str] = Field(
+    ELASTICSEARCH_CA_CERTS: str | None = Field(
         description="Path to CA certificate file for SSL verification", default=None
     )
     ELASTICSEARCH_VERIFY_CERTS: bool = Field(

api/configs/middleware/vdb/huawei_cloud_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class HuaweiCloudConfig(BaseSettings):
     Configuration settings for Huawei cloud search service
     """
 
-    HUAWEI_CLOUD_HOSTS: Optional[str] = Field(
+    HUAWEI_CLOUD_HOSTS: str | None = Field(
         description="Hostname or IP address of the Huawei cloud search service instance",
         default=None,
     )
 
-    HUAWEI_CLOUD_USER: Optional[str] = Field(
+    HUAWEI_CLOUD_USER: str | None = Field(
         description="Username for authenticating with Huawei cloud search service",
         default=None,
     )
 
-    HUAWEI_CLOUD_PASSWORD: Optional[str] = Field(
+    HUAWEI_CLOUD_PASSWORD: str | None = Field(
         description="Password for authenticating with Huawei cloud search service",
         default=None,
     )

api/configs/middleware/vdb/lindorm_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class LindormConfig(BaseSettings):
     Lindorm configs
     """
 
-    LINDORM_URL: Optional[str] = Field(
+    LINDORM_URL: str | None = Field(
         description="Lindorm url",
         default=None,
     )
-    LINDORM_USERNAME: Optional[str] = Field(
+    LINDORM_USERNAME: str | None = Field(
         description="Lindorm user",
         default=None,
     )
-    LINDORM_PASSWORD: Optional[str] = Field(
+    LINDORM_PASSWORD: str | None = Field(
         description="Lindorm password",
         default=None,
     )
-    DEFAULT_INDEX_TYPE: Optional[str] = Field(
+    LINDORM_INDEX_TYPE: str | None = Field(
         description="Lindorm Vector Index Type, hnsw or flat is available in dify",
         default="hnsw",
     )
-    DEFAULT_DISTANCE_TYPE: Optional[str] = Field(
+    LINDORM_DISTANCE_TYPE: str | None = Field(
         description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
     )
-    USING_UGC_INDEX: Optional[bool] = Field(
-        description="Using UGC index will store the same type of Index in a single index but can retrieve separately.",
-        default=False,
+    LINDORM_USING_UGC: bool | None = Field(
+        description="Using UGC index will store indexes with the same IndexType/Dimension in a single big index.",
+        default=True,
     )
-    LINDORM_QUERY_TIMEOUT: Optional[float] = Field(description="The lindorm search request timeout (s)", default=2.0)
+    LINDORM_QUERY_TIMEOUT: float | None = Field(description="The lindorm search request timeout (s)", default=2.0)

api/configs/middleware/vdb/matrixone_config.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class MatrixoneConfig(BaseModel):
+class MatrixoneConfig(BaseSettings):
     """Matrixone vector database configuration."""
 
     MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server")

api/configs/middleware/vdb/milvus_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class MilvusConfig(BaseSettings):
     Configuration settings for Milvus vector database
     """
 
-    MILVUS_URI: Optional[str] = Field(
+    MILVUS_URI: str | None = Field(
         description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
         default="http://127.0.0.1:19530",
     )
 
-    MILVUS_TOKEN: Optional[str] = Field(
+    MILVUS_TOKEN: str | None = Field(
         description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
 
-    MILVUS_USER: Optional[str] = Field(
+    MILVUS_USER: str | None = Field(
         description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
-    MILVUS_PASSWORD: Optional[str] = Field(
+    MILVUS_PASSWORD: str | None = Field(
         description="Password for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
@@ -40,7 +38,7 @@ class MilvusConfig(BaseSettings):
         default=True,
     )
 
-    MILVUS_ANALYZER_PARAMS: Optional[str] = Field(
+    MILVUS_ANALYZER_PARAMS: str | None = Field(
         description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
         default=None,
     )

api/configs/middleware/vdb/oceanbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OceanBaseVectorConfig(BaseSettings):
     Configuration settings for OceanBase Vector database
     """
 
-    OCEANBASE_VECTOR_HOST: Optional[str] = Field(
+    OCEANBASE_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PORT: Optional[PositiveInt] = Field(
+    OCEANBASE_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the OceanBase Vector server is listening (default is 2881)",
         default=2881,
     )
 
-    OCEANBASE_VECTOR_USER: Optional[str] = Field(
+    OCEANBASE_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PASSWORD: Optional[str] = Field(
+    OCEANBASE_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_DATABASE: Optional[str] = Field(
+    OCEANBASE_VECTOR_DATABASE: str | None = Field(
         description="Name of the OceanBase Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/opengauss_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class OpenGaussConfig(BaseSettings):
     Configuration settings for OpenGauss
     """
 
-    OPENGAUSS_HOST: Optional[str] = Field(
+    OPENGAUSS_HOST: str | None = Field(
         description="Hostname or IP address of the OpenGauss server(e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class OpenGaussConfig(BaseSettings):
         default=6600,
     )
 
-    OPENGAUSS_USER: Optional[str] = Field(
+    OPENGAUSS_USER: str | None = Field(
         description="Username for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_PASSWORD: Optional[str] = Field(
+    OPENGAUSS_PASSWORD: str | None = Field(
         description="Password for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_DATABASE: Optional[str] = Field(
+    OPENGAUSS_DATABASE: str | None = Field(
         description="Name of the OpenGauss database to connect to",
         default=None,
     )

api/configs/middleware/vdb/opensearch_config.py

@@ -1,5 +1,5 @@
-import enum
-from typing import Literal, Optional
+from enum import Enum
+from typing import Literal
 
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
@@ -10,7 +10,7 @@ class OpenSearchConfig(BaseSettings):
     Configuration settings for OpenSearch
     """
 
-    class AuthMethod(enum.StrEnum):
+    class AuthMethod(Enum):
         """
         Authentication method for OpenSearch
         """
@@ -18,7 +18,7 @@ class OpenSearchConfig(BaseSettings):
         BASIC = "basic"
         AWS_MANAGED_IAM = "aws_managed_iam"
 
-    OPENSEARCH_HOST: Optional[str] = Field(
+    OPENSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
     )
@@ -43,21 +43,21 @@ class OpenSearchConfig(BaseSettings):
         default=AuthMethod.BASIC,
     )
 
-    OPENSEARCH_USER: Optional[str] = Field(
+    OPENSEARCH_USER: str | None = Field(
         description="Username for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_PASSWORD: Optional[str] = Field(
+    OPENSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_AWS_REGION: Optional[str] = Field(
+    OPENSEARCH_AWS_REGION: str | None = Field(
         description="AWS region for OpenSearch (e.g. 'us-west-2')",
         default=None,
     )
 
-    OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+    OPENSEARCH_AWS_SERVICE: Literal["es", "aoss"] | None = Field(
         description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
     )

api/configs/middleware/vdb/oracle_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,33 +7,33 @@ class OracleConfig(BaseSettings):
     Configuration settings for Oracle database
     """
 
-    ORACLE_USER: Optional[str] = Field(
+    ORACLE_USER: str | None = Field(
         description="Username for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_PASSWORD: Optional[str] = Field(
+    ORACLE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_DSN: Optional[str] = Field(
+    ORACLE_DSN: str | None = Field(
         description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
         "For autonomous database, use the service name from tnsnames.ora in the wallet",
         default=None,
     )
 
-    ORACLE_CONFIG_DIR: Optional[str] = Field(
+    ORACLE_CONFIG_DIR: str | None = Field(
         description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
         default=None,
     )
 
-    ORACLE_WALLET_LOCATION: Optional[str] = Field(
+    ORACLE_WALLET_LOCATION: str | None = Field(
         description="Oracle wallet directory path containing the wallet files for secure connection",
         default=None,
     )
 
-    ORACLE_WALLET_PASSWORD: Optional[str] = Field(
+    ORACLE_WALLET_PASSWORD: str | None = Field(
         description="Password to decrypt the Oracle wallet, if it is encrypted",
         default=None,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectorConfig(BaseSettings):
     Configuration settings for PGVector (PostgreSQL with vector extension)
     """
 
-    PGVECTOR_HOST: Optional[str] = Field(
+    PGVECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectorConfig(BaseSettings):
         default=5433,
     )
 
-    PGVECTOR_USER: Optional[str] = Field(
+    PGVECTOR_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_PASSWORD: Optional[str] = Field(
+    PGVECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_DATABASE: Optional[str] = Field(
+    PGVECTOR_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database to connect to",
         default=None,
     )

api/configs/middleware/vdb/pgvectors_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectoRSConfig(BaseSettings):
     Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
     """
 
-    PGVECTO_RS_HOST: Optional[str] = Field(
+    PGVECTO_RS_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectoRSConfig(BaseSettings):
         default=5431,
     )
 
-    PGVECTO_RS_USER: Optional[str] = Field(
+    PGVECTO_RS_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+    PGVECTO_RS_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_DATABASE: Optional[str] = Field(
+    PGVECTO_RS_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
         default=None,
     )

api/configs/middleware/vdb/qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class QdrantConfig(BaseSettings):
     Configuration settings for Qdrant vector database
     """
 
-    QDRANT_URL: Optional[str] = Field(
+    QDRANT_URL: str | None = Field(
         description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
         default=None,
     )
 
-    QDRANT_API_KEY: Optional[str] = Field(
+    QDRANT_API_KEY: str | None = Field(
         description="API key for authenticating with the Qdrant server",
         default=None,
     )

api/configs/middleware/vdb/relyt_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class RelytConfig(BaseSettings):
     Configuration settings for Relyt database
     """
 
-    RELYT_HOST: Optional[str] = Field(
+    RELYT_HOST: str | None = Field(
         description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class RelytConfig(BaseSettings):
         default=9200,
     )
 
-    RELYT_USER: Optional[str] = Field(
+    RELYT_USER: str | None = Field(
         description="Username for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_PASSWORD: Optional[str] = Field(
+    RELYT_PASSWORD: str | None = Field(
         description="Password for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_DATABASE: Optional[str] = Field(
+    RELYT_DATABASE: str | None = Field(
         description="Name of the Relyt database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/tablestore_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class TableStoreConfig(BaseSettings):
     Configuration settings for TableStore.
     """
 
-    TABLESTORE_ENDPOINT: Optional[str] = Field(
+    TABLESTORE_ENDPOINT: str | None = Field(
         description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')",
         default=None,
     )
 
-    TABLESTORE_INSTANCE_NAME: Optional[str] = Field(
+    TABLESTORE_INSTANCE_NAME: str | None = Field(
         description="Instance name to access TableStore server (eg. 'instance-name')",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_ID: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_ID: str | None = Field(
         description="AccessKey id for the instance name",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_SECRET: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_SECRET: str | None = Field(
         description="AccessKey secret for the instance name",
         default=None,
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TencentVectorDBConfig(BaseSettings):
     Configuration settings for Tencent Vector Database
     """
 
-    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+    TENCENT_VECTOR_DB_URL: str | None = Field(
         description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+    TENCENT_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Tencent Vector Database service",
         default=None,
     )
@@ -24,12 +22,12 @@ class TencentVectorDBConfig(BaseSettings):
         default=30,
     )
 
-    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+    TENCENT_VECTOR_DB_USERNAME: str | None = Field(
         description="Username for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+    TENCENT_VECTOR_DB_PASSWORD: str | None = Field(
         description="Password for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
@@ -44,7 +42,7 @@ class TencentVectorDBConfig(BaseSettings):
         default=2,
     )
 
-    TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
+    TENCENT_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Tencent Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TidbOnQdrantConfig(BaseSettings):
     Tidb on Qdrant configs
     """
 
-    TIDB_ON_QDRANT_URL: Optional[str] = Field(
+    TIDB_ON_QDRANT_URL: str | None = Field(
         description="Tidb on Qdrant url",
         default=None,
     )
 
-    TIDB_ON_QDRANT_API_KEY: Optional[str] = Field(
+    TIDB_ON_QDRANT_API_KEY: str | None = Field(
         description="Tidb on Qdrant api key",
         default=None,
     )
@@ -34,37 +32,37 @@ class TidbOnQdrantConfig(BaseSettings):
         default=6334,
     )
 
-    TIDB_PUBLIC_KEY: Optional[str] = Field(
+    TIDB_PUBLIC_KEY: str | None = Field(
         description="Tidb account public key",
         default=None,
     )
 
-    TIDB_PRIVATE_KEY: Optional[str] = Field(
+    TIDB_PRIVATE_KEY: str | None = Field(
         description="Tidb account private key",
         default=None,
     )
 
-    TIDB_API_URL: Optional[str] = Field(
+    TIDB_API_URL: str | None = Field(
         description="Tidb API url",
         default=None,
     )
 
-    TIDB_IAM_API_URL: Optional[str] = Field(
+    TIDB_IAM_API_URL: str | None = Field(
         description="Tidb IAM API url",
         default=None,
     )
 
-    TIDB_REGION: Optional[str] = Field(
+    TIDB_REGION: str | None = Field(
         description="Tidb serverless region",
         default="regions/aws-us-east-1",
     )
 
-    TIDB_PROJECT_ID: Optional[str] = Field(
+    TIDB_PROJECT_ID: str | None = Field(
         description="Tidb project id",
         default=None,
     )
 
-    TIDB_SPEND_LIMIT: Optional[int] = Field(
+    TIDB_SPEND_LIMIT: int | None = Field(
         description="Tidb spend limit",
         default=100,
     )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TiDBVectorConfig(BaseSettings):
     Configuration settings for TiDB Vector database
     """
 
-    TIDB_VECTOR_HOST: Optional[str] = Field(
+    TIDB_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
         default=None,
     )
 
-    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+    TIDB_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the TiDB Vector server is listening (default is 4000)",
         default=4000,
     )
 
-    TIDB_VECTOR_USER: Optional[str] = Field(
+    TIDB_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+    TIDB_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+    TIDB_VECTOR_DATABASE: str | None = Field(
         description="Name of the TiDB Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/upstash_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class UpstashConfig(BaseSettings):
     Configuration settings for Upstash vector database
     """
 
-    UPSTASH_VECTOR_URL: Optional[str] = Field(
+    UPSTASH_VECTOR_URL: str | None = Field(
         description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
         default=None,
     )
 
-    UPSTASH_VECTOR_TOKEN: Optional[str] = Field(
+    UPSTASH_VECTOR_TOKEN: str | None = Field(
         description="Token for authenticating with the upstash server",
         default=None,
     )

api/configs/middleware/vdb/vastbase_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class VastbaseVectorConfig(BaseSettings):
     Configuration settings for Vector (Vastbase with vector extension)
     """
 
-    VASTBASE_HOST: Optional[str] = Field(
+    VASTBASE_HOST: str | None = Field(
         description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class VastbaseVectorConfig(BaseSettings):
         default=5432,
     )
 
-    VASTBASE_USER: Optional[str] = Field(
+    VASTBASE_USER: str | None = Field(
         description="Username for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_PASSWORD: Optional[str] = Field(
+    VASTBASE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_DATABASE: Optional[str] = Field(
+    VASTBASE_DATABASE: str | None = Field(
         description="Name of the Vastbase database to connect to",
         default=None,
     )

api/configs/middleware/vdb/vikingdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -11,14 +9,14 @@ class VikingDBConfig(BaseSettings):
     https://www.volcengine.com/docs/6291/65568
     """
 
-    VIKINGDB_ACCESS_KEY: Optional[str] = Field(
+    VIKINGDB_ACCESS_KEY: str | None = Field(
         description="The Access Key provided by Volcengine VikingDB for API authentication."
         "Refer to the following documentation for details on obtaining credentials:"
         "https://www.volcengine.com/docs/6291/65568",
         default=None,
     )
 
-    VIKINGDB_SECRET_KEY: Optional[str] = Field(
+    VIKINGDB_SECRET_KEY: str | None = Field(
         description="The Secret Key provided by Volcengine VikingDB for API authentication.",
         default=None,
     )

api/configs/middleware/vdb/weaviate_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class WeaviateConfig(BaseSettings):
     Configuration settings for Weaviate vector database
     """
 
-    WEAVIATE_ENDPOINT: Optional[str] = Field(
+    WEAVIATE_ENDPOINT: str | None = Field(
         description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
         default=None,
     )
 
-    WEAVIATE_API_KEY: Optional[str] = Field(
+    WEAVIATE_API_KEY: str | None = Field(
         description="API key for authenticating with the Weaviate server",
         default=None,
     )

api/configs/packaging/__init__.py

@@ -1,6 +1,6 @@
 from pydantic import Field
 
-from configs.packaging.pyproject import PyProjectConfig, PyProjectTomlConfig
+from configs.packaging.pyproject import PyProjectTomlConfig
 
 
 class PackagingInfo(PyProjectTomlConfig):

api/configs/remote_settings_sources/apollo/__init__.py

@@ -1,5 +1,5 @@
 from collections.abc import Mapping
-from typing import Any, Optional
+from typing import Any
 
 from pydantic import Field
 from pydantic.fields import FieldInfo
@@ -15,22 +15,22 @@ class ApolloSettingsSourceInfo(BaseSettings):
     Packaging build information
     """
 
-    APOLLO_APP_ID: Optional[str] = Field(
+    APOLLO_APP_ID: str | None = Field(
         description="apollo app_id",
         default=None,
     )
 
-    APOLLO_CLUSTER: Optional[str] = Field(
+    APOLLO_CLUSTER: str | None = Field(
         description="apollo cluster",
         default=None,
     )
 
-    APOLLO_CONFIG_URL: Optional[str] = Field(
+    APOLLO_CONFIG_URL: str | None = Field(
         description="apollo config url",
         default=None,
     )
 
-    APOLLO_NAMESPACE: Optional[str] = Field(
+    APOLLO_NAMESPACE: str | None = Field(
         description="apollo namespace",
         default=None,
     )

api/configs/remote_settings_sources/apollo/client.py

@@ -4,8 +4,9 @@ import logging
 import os
 import threading
 import time
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from pathlib import Path
+from typing import Any
 
 from .python_3x import http_request, makedirs_wrapper
 from .utils import (
@@ -25,13 +26,13 @@ logger = logging.getLogger(__name__)
 class ApolloClient:
     def __init__(
         self,
-        config_url,
-        app_id,
-        cluster="default",
-        secret="",
-        start_hot_update=True,
-        change_listener=None,
-        _notification_map=None,
+        config_url: str,
+        app_id: str,
+        cluster: str = "default",
+        secret: str = "",
+        start_hot_update: bool = True,
+        change_listener: Callable[[str, str, str, Any], None] | None = None,
+        _notification_map: dict[str, int] | None = None,
     ):
         # Core routing parameters
         self.config_url = config_url
@@ -47,17 +48,17 @@ class ApolloClient:
         # Private control variables
         self._cycle_time = 5
         self._stopping = False
-        self._cache = {}
-        self._no_key = {}
-        self._hash = {}
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._no_key: dict[str, str] = {}
+        self._hash: dict[str, str] = {}
         self._pull_timeout = 75
         self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
-        self._long_poll_thread = None
+        self._long_poll_thread: threading.Thread | None = None
         self._change_listener = change_listener  # "add" "delete" "update"
         if _notification_map is None:
             _notification_map = {"application": -1}
         self._notification_map = _notification_map
-        self.last_release_key = None
+        self.last_release_key: str | None = None
         # Private startup method
         self._path_checker()
         if start_hot_update:
@@ -68,7 +69,7 @@ class ApolloClient:
         heartbeat.daemon = True
         heartbeat.start()
 
-    def get_json_from_net(self, namespace="application"):
+    def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None:
         url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
             self.config_url, self.app_id, self.cluster, namespace, "", self.ip
         )
@@ -88,7 +89,7 @@ class ApolloClient:
             logger.exception("an error occurred in get_json_from_net")
             return None
 
-    def get_value(self, key, default_val=None, namespace="application"):
+    def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any:
         try:
             # read memory configuration
             namespace_cache = self._cache.get(namespace)
@@ -104,7 +105,8 @@ class ApolloClient:
             namespace_data = self.get_json_from_net(namespace)
             val = get_value_from_dict(namespace_data, key)
             if val is not None:
-                self._update_cache_and_file(namespace_data, namespace)
+                if namespace_data is not None:
+                    self._update_cache_and_file(namespace_data, namespace)
                 return val
 
             # read the file configuration
@@ -126,23 +128,23 @@ class ApolloClient:
     # to ensure the real-time correctness of the function call.
     # If the user does not have the same default val twice
     # and the default val is used here, there may be a problem.
-    def _set_local_cache_none(self, namespace, key):
+    def _set_local_cache_none(self, namespace: str, key: str) -> None:
         no_key = no_key_cache_key(namespace, key)
         self._no_key[no_key] = key
 
-    def _start_hot_update(self):
+    def _start_hot_update(self) -> None:
         self._long_poll_thread = threading.Thread(target=self._listener)
         # When the asynchronous thread is started, the daemon thread will automatically exit
         # when the main thread is launched.
         self._long_poll_thread.daemon = True
         self._long_poll_thread.start()
 
-    def stop(self):
+    def stop(self) -> None:
         self._stopping = True
         logger.info("Stopping listener...")
 
     # Call the set callback function, and if it is abnormal, try it out
-    def _call_listener(self, namespace, old_kv, new_kv):
+    def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None:
         if self._change_listener is None:
             return
         if old_kv is None:
@@ -168,12 +170,12 @@ class ApolloClient:
         except BaseException as e:
             logger.warning(str(e))
 
-    def _path_checker(self):
+    def _path_checker(self) -> None:
         if not os.path.isdir(self._cache_file_path):
             makedirs_wrapper(self._cache_file_path)
 
     # update the local cache and file cache
-    def _update_cache_and_file(self, namespace_data, namespace="application"):
+    def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None:
         # update the local cache
         self._cache[namespace] = namespace_data
         # update the file cache
@@ -187,7 +189,7 @@ class ApolloClient:
             self._hash[namespace] = new_hash
 
     # get the configuration from the local file
-    def _get_local_cache(self, namespace="application"):
+    def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]:
         cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
         if os.path.isfile(cache_file_path):
             with open(cache_file_path) as f:
@@ -195,8 +197,8 @@ class ApolloClient:
             return result
         return {}
 
-    def _long_poll(self):
-        notifications = []
+    def _long_poll(self) -> None:
+        notifications: list[dict[str, Any]] = []
         for key in self._cache:
             namespace_data = self._cache[key]
             notification_id = -1
@@ -236,7 +238,7 @@ class ApolloClient:
         except Exception as e:
             logger.warning(str(e))
 
-    def _get_net_and_set_local(self, namespace, n_id, call_change=False):
+    def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None:
         namespace_data = self.get_json_from_net(namespace)
         if not namespace_data:
             return
@@ -248,7 +250,7 @@ class ApolloClient:
             new_kv = namespace_data.get(CONFIGURATIONS)
             self._call_listener(namespace, old_kv, new_kv)
 
-    def _listener(self):
+    def _listener(self) -> None:
         logger.info("start long_poll")
         while not self._stopping:
             self._long_poll()
@@ -266,13 +268,13 @@ class ApolloClient:
         headers["Timestamp"] = time_unix_now
         return headers
 
-    def _heart_beat(self):
+    def _heart_beat(self) -> None:
         while not self._stopping:
             for namespace in self._notification_map:
                 self._do_heart_beat(namespace)
             time.sleep(60 * 10)  # 10 minutes
 
-    def _do_heart_beat(self, namespace):
+    def _do_heart_beat(self, namespace: str) -> None:
         url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}"
         try:
             code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
@@ -292,7 +294,7 @@ class ApolloClient:
             logger.exception("an error occurred in _do_heart_beat")
             return None
 
-    def get_all_dicts(self, namespace):
+    def get_all_dicts(self, namespace: str) -> dict[str, Any] | None:
         namespace_data = self._cache.get(namespace)
         if namespace_data is None:
             net_namespace_data = self.get_json_from_net(namespace)

api/configs/remote_settings_sources/apollo/python_3x.py

@@ -2,6 +2,8 @@ import logging
 import os
 import ssl
 import urllib.request
+from collections.abc import Mapping
+from typing import Any
 from urllib import parse
 from urllib.error import HTTPError
 
@@ -19,9 +21,9 @@ urllib.request.install_opener(opener)
 logger = logging.getLogger(__name__)
 
 
-def http_request(url, timeout, headers={}):
+def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]:
     try:
-        request = urllib.request.Request(url, headers=headers)
+        request = urllib.request.Request(url, headers=dict(headers))
         res = urllib.request.urlopen(request, timeout=timeout)
         body = res.read().decode("utf-8")
         return res.code, body
@@ -33,9 +35,9 @@ def http_request(url, timeout, headers={}):
         raise e
 
 
-def url_encode(params):
+def url_encode(params: dict[str, Any]) -> str:
     return parse.urlencode(params)
 
 
-def makedirs_wrapper(path):
+def makedirs_wrapper(path: str) -> None:
     os.makedirs(path, exist_ok=True)

api/configs/remote_settings_sources/apollo/utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import socket
+from typing import Any
 
 from .python_3x import url_encode
 
@@ -10,7 +11,7 @@ NAMESPACE_NAME = "namespaceName"
 
 
 # add timestamps uris and keys
-def signature(timestamp, uri, secret):
+def signature(timestamp: str, uri: str, secret: str) -> str:
     import base64
     import hmac
 
@@ -19,16 +20,16 @@ def signature(timestamp, uri, secret):
     return base64.b64encode(hmac_code).decode()
 
 
-def url_encode_wrapper(params):
+def url_encode_wrapper(params: dict[str, Any]) -> str:
     return url_encode(params)
 
 
-def no_key_cache_key(namespace, key):
+def no_key_cache_key(namespace: str, key: str) -> str:
     return f"{namespace}{len(namespace)}{key}"
 
 
 # Returns whether the obtained value is obtained, and None if it does not
-def get_value_from_dict(namespace_cache, key):
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any:
     if namespace_cache:
         kv_data = namespace_cache.get(CONFIGURATIONS)
         if kv_data is None:
@@ -38,7 +39,7 @@ def get_value_from_dict(namespace_cache, key):
     return None
 
 
-def init_ip():
+def init_ip() -> str:
     ip = ""
     s = None
     try:

api/configs/remote_settings_sources/base.py

@@ -11,5 +11,5 @@ class RemoteSettingsSource:
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
         raise NotImplementedError
 
-    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool):
         return value

api/configs/remote_settings_sources/nacos/__init__.py

@@ -11,16 +11,16 @@ logger = logging.getLogger(__name__)
 
 from configs.remote_settings_sources.base import RemoteSettingsSource
 
-from .utils import _parse_config
+from .utils import parse_config
 
 
 class NacosSettingsSource(RemoteSettingsSource):
     def __init__(self, configs: Mapping[str, Any]):
         self.configs = configs
-        self.remote_configs: dict[str, Any] = {}
+        self.remote_configs: dict[str, str] = {}
         self.async_init()
 
-    def async_init(self):
+    def async_init(self) -> None:
         data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
         group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
         tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
@@ -29,22 +29,19 @@ class NacosSettingsSource(RemoteSettingsSource):
         try:
             content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
             self.remote_configs = self._parse_config(content)
-        except Exception as e:
+        except Exception:
             logger.exception("[get-access-token] exception occurred")
             raise
 
-    def _parse_config(self, content: str) -> dict:
+    def _parse_config(self, content: str) -> dict[str, str]:
         if not content:
             return {}
         try:
-            return _parse_config(self, content)
+            return parse_config(content)
         except Exception as e:
             raise RuntimeError(f"Failed to parse config: {e}")
 
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        if not isinstance(self.remote_configs, dict):
-            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
-
         field_value = self.remote_configs.get(field_name)
         if field_value is None:
             return None, field_name, False

api/configs/remote_settings_sources/nacos/http_request.py

@@ -17,20 +17,26 @@ class NacosHttpClient:
         self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
         self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
         self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
-        self.token = None
+        self.token: str | None = None
         self.token_ttl = 18000
         self.token_expire_time: float = 0
 
-    def http_request(self, url, method="GET", headers=None, params=None):
+    def http_request(
+        self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None
+    ) -> str:
+        if headers is None:
+            headers = {}
+        if params is None:
+            params = {}
         try:
             self._inject_auth_info(headers, params)
             response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
             response.raise_for_status()
             return response.text
-        except requests.exceptions.RequestException as e:
+        except requests.RequestException as e:
             return f"Request to Nacos failed: {e}"
 
-    def _inject_auth_info(self, headers, params, module="config"):
+    def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
         headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
 
         if module == "login":
@@ -45,16 +51,17 @@ class NacosHttpClient:
             headers["timeStamp"] = ts
         if self.username and self.password:
             self.get_access_token(force_refresh=False)
-            params["accessToken"] = self.token
+            if self.token is not None:
+                params["accessToken"] = self.token
 
-    def __do_sign(self, sign_str, sk):
+    def __do_sign(self, sign_str: str, sk: str) -> str:
         return (
             base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
             .decode()
             .strip()
         )
 
-    def get_sign_str(self, group, tenant, ts):
+    def get_sign_str(self, group: str, tenant: str, ts: str) -> str:
         sign_str = ""
         if tenant:
             sign_str = tenant + "+"
@@ -63,7 +70,7 @@ class NacosHttpClient:
         sign_str += ts  # Directly concatenate ts without conditional checks, because the nacos auth header forced it.
         return sign_str
 
-    def get_access_token(self, force_refresh=False):
+    def get_access_token(self, force_refresh: bool = False) -> str | None:
         current_time = time.time()
         if self.token and not force_refresh and self.token_expire_time > current_time:
             return self.token
@@ -77,6 +84,7 @@ class NacosHttpClient:
             self.token = response_data.get("accessToken")
             self.token_ttl = response_data.get("tokenTtl", 18000)
             self.token_expire_time = current_time + self.token_ttl - 10
-        except Exception as e:
+            return self.token
+        except Exception:
             logger.exception("[get-access-token] exception occur")
             raise

api/configs/remote_settings_sources/nacos/utils.py

@@ -1,4 +1,4 @@
-def _parse_config(self, content: str) -> dict[str, str]:
+def parse_config(content: str) -> dict[str, str]:
     config: dict[str, str] = {}
     if not content:
         return config

api/constants/__init__.py

@@ -16,14 +16,14 @@ AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "amr", "mpga"]
 AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
 
+_doc_extensions: list[str]
 if dify_config.ETL_TYPE == "Unstructured":
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "vtt", "properties"]
-    DOCUMENT_EXTENSIONS.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
+    _doc_extensions = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "vtt", "properties"]
+    _doc_extensions.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
     if dify_config.UNSTRUCTURED_API_URL:
-        DOCUMENT_EXTENSIONS.append("ppt")
-    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+        _doc_extensions.append("ppt")
 else:
-    DOCUMENT_EXTENSIONS = [
+    _doc_extensions = [
         "txt",
         "markdown",
         "md",
@@ -38,4 +38,4 @@ else:
         "vtt",
         "properties",
     ]
-    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+DOCUMENT_EXTENSIONS = _doc_extensions + [ext.upper() for ext in _doc_extensions]

api/constants/languages.py

@@ -19,6 +19,7 @@ language_timezone_mapping = {
     "fa-IR": "Asia/Tehran",
     "sl-SI": "Europe/Ljubljana",
     "th-TH": "Asia/Bangkok",
+    "id-ID": "Asia/Jakarta",
 }
 
 languages = list(language_timezone_mapping.keys())

api/constants/model_template.py

@@ -7,7 +7,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # workflow default mode
     AppMode.WORKFLOW: {
         "app": {
-            "mode": AppMode.WORKFLOW.value,
+            "mode": AppMode.WORKFLOW,
             "enable_site": True,
             "enable_api": True,
         }
@@ -15,7 +15,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # completion default mode
     AppMode.COMPLETION: {
         "app": {
-            "mode": AppMode.COMPLETION.value,
+            "mode": AppMode.COMPLETION,
             "enable_site": True,
             "enable_api": True,
         },
@@ -44,7 +44,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # chat default mode
     AppMode.CHAT: {
         "app": {
-            "mode": AppMode.CHAT.value,
+            "mode": AppMode.CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -60,7 +60,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # advanced-chat default mode
     AppMode.ADVANCED_CHAT: {
         "app": {
-            "mode": AppMode.ADVANCED_CHAT.value,
+            "mode": AppMode.ADVANCED_CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -68,7 +68,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # agent-chat default mode
     AppMode.AGENT_CHAT: {
         "app": {
-            "mode": AppMode.AGENT_CHAT.value,
+            "mode": AppMode.AGENT_CHAT,
             "enable_site": True,
             "enable_api": True,
         },

api/contexts/__init__.py

@@ -5,10 +5,10 @@ from typing import TYPE_CHECKING
 from contexts.wrapper import RecyclableContextVar
 
 if TYPE_CHECKING:
+    from core.datasource.__base.datasource_provider import DatasourcePluginProviderController
     from core.model_runtime.entities.model_entities import AIModelEntity
     from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
     from core.tools.plugin_tool.provider import PluginToolProviderController
-    from core.workflow.entities.variable_pool import VariablePool
 
 
 """
@@ -33,3 +33,11 @@ plugin_model_schema_lock: RecyclableContextVar[Lock] = RecyclableContextVar(Cont
 plugin_model_schemas: RecyclableContextVar[dict[str, "AIModelEntity"]] = RecyclableContextVar(
     ContextVar("plugin_model_schemas")
 )
+
+datasource_plugin_providers: RecyclableContextVar[dict[str, "DatasourcePluginProviderController"]] = (
+    RecyclableContextVar(ContextVar("datasource_plugin_providers"))
+)
+
+datasource_plugin_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("datasource_plugin_providers_lock")
+)

api/controllers/console/__init__.py

@@ -1,4 +1,5 @@
 from flask import Blueprint
+from flask_restx import Namespace
 
 from libs.external_api import ExternalApi
 
@@ -26,7 +27,16 @@ from .files import FileApi, FilePreviewApi, FileSupportTypeApi
 from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
 
 bp = Blueprint("console", __name__, url_prefix="/console/api")
-api = ExternalApi(bp)
+
+api = ExternalApi(
+    bp,
+    version="1.0",
+    title="Console API",
+    description="Console management APIs for app configuration, monitoring, and administration",
+)
+
+# Create namespace
+console_ns = Namespace("console", description="Console management API operations", path="/")
 
 # File
 api.add_resource(FileApi, "/files/upload")
@@ -43,7 +53,17 @@ api.add_resource(AppImportConfirmApi, "/apps/imports/<string:import_id>/confirm"
 api.add_resource(AppImportCheckDependenciesApi, "/apps/imports/<string:app_id>/check-dependencies")
 
 # Import other controllers
-from . import admin, apikey, extension, feature, ping, setup, version
+from . import (
+    admin,
+    apikey,
+    extension,
+    feature,
+    init_validate,
+    ping,
+    setup,
+    spec,
+    version,
+)
 
 # Import app controllers
 from .app import (
@@ -70,7 +90,16 @@ from .app import (
 )
 
 # Import auth controllers
-from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_password, login, oauth
+from .auth import (
+    activate,
+    data_source_bearer_auth,
+    data_source_oauth,
+    email_register,
+    forgot_password,
+    login,
+    oauth,
+    oauth_server,
+)
 
 # Import billing controllers
 from .billing import billing, compliance
@@ -86,6 +115,15 @@ from .datasets import (
     metadata,
     website,
 )
+from .datasets.rag_pipeline import (
+    datasource_auth,
+    datasource_content_preview,
+    rag_pipeline,
+    rag_pipeline_datasets,
+    rag_pipeline_draft_variable,
+    rag_pipeline_import,
+    rag_pipeline_workflow,
+)
 
 # Import explore controllers
 from .explore import (
@@ -95,6 +133,23 @@ from .explore import (
     saved_message,
 )
 
+# Import tag controllers
+from .tag import tags
+
+# Import workspace controllers
+from .workspace import (
+    account,
+    agent_providers,
+    endpoint,
+    load_balancing_config,
+    members,
+    model_providers,
+    models,
+    plugin,
+    tool_providers,
+    workspace,
+)
+
 # Explore Audio
 api.add_resource(ChatAudioApi, "/installed-apps/<uuid:installed_app_id>/audio-to-text", endpoint="installed_app_audio")
 api.add_resource(ChatTextApi, "/installed-apps/<uuid:installed_app_id>/text-to-audio", endpoint="installed_app_text")
@@ -166,19 +221,79 @@ api.add_resource(
     InstalledAppWorkflowTaskStopApi, "/installed-apps/<uuid:installed_app_id>/workflows/tasks/<string:task_id>/stop"
 )
 
-# Import tag controllers
-from .tag import tags
+api.add_namespace(console_ns)
 
-# Import workspace controllers
-from .workspace import (
-    account,
-    agent_providers,
-    endpoint,
-    load_balancing_config,
-    members,
-    model_providers,
-    models,
-    plugin,
-    tool_providers,
-    workspace,
-)
+__all__ = [
+    "account",
+    "activate",
+    "admin",
+    "advanced_prompt_template",
+    "agent",
+    "agent_providers",
+    "annotation",
+    "api",
+    "apikey",
+    "app",
+    "audio",
+    "billing",
+    "bp",
+    "completion",
+    "compliance",
+    "console_ns",
+    "conversation",
+    "conversation_variables",
+    "data_source",
+    "data_source_bearer_auth",
+    "data_source_oauth",
+    "datasets",
+    "datasets_document",
+    "datasets_segments",
+    "datasource_auth",
+    "datasource_content_preview",
+    "email_register",
+    "endpoint",
+    "extension",
+    "external",
+    "feature",
+    "forgot_password",
+    "generator",
+    "hit_testing",
+    "init_validate",
+    "installed_app",
+    "load_balancing_config",
+    "login",
+    "mcp_server",
+    "members",
+    "message",
+    "metadata",
+    "model_config",
+    "model_providers",
+    "models",
+    "oauth",
+    "oauth_server",
+    "ops_trace",
+    "parameter",
+    "ping",
+    "plugin",
+    "rag_pipeline",
+    "rag_pipeline_datasets",
+    "rag_pipeline_draft_variable",
+    "rag_pipeline_import",
+    "rag_pipeline_workflow",
+    "recommended_app",
+    "saved_message",
+    "setup",
+    "site",
+    "spec",
+    "statistic",
+    "tags",
+    "tool_providers",
+    "version",
+    "website",
+    "workflow",
+    "workflow_app_log",
+    "workflow_draft_variable",
+    "workflow_run",
+    "workflow_statistic",
+    "workspace",
+]

api/controllers/console/admin.py

@@ -1,22 +1,26 @@
+from collections.abc import Callable
 from functools import wraps
+from typing import ParamSpec, TypeVar
 
 from flask import request
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound, Unauthorized
 
+P = ParamSpec("P")
+R = TypeVar("R")
 from configs import dify_config
 from constants.languages import supported_language
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
 from models.model import App, InstalledApp, RecommendedApp
 
 
-def admin_required(view):
+def admin_required(view: Callable[P, R]):
     @wraps(view)
-    def decorated(*args, **kwargs):
+    def decorated(*args: P.args, **kwargs: P.kwargs):
         if not dify_config.ADMIN_API_KEY:
             raise Unauthorized("API key is invalid.")
 
@@ -41,7 +45,28 @@ def admin_required(view):
     return decorated
 
 
+@console_ns.route("/admin/insert-explore-apps")
 class InsertExploreAppListApi(Resource):
+    @api.doc("insert_explore_app")
+    @api.doc(description="Insert or update an app in the explore list")
+    @api.expect(
+        api.model(
+            "InsertExploreAppRequest",
+            {
+                "app_id": fields.String(required=True, description="Application ID"),
+                "desc": fields.String(description="App description"),
+                "copyright": fields.String(description="Copyright information"),
+                "privacy_policy": fields.String(description="Privacy policy"),
+                "custom_disclaimer": fields.String(description="Custom disclaimer"),
+                "language": fields.String(required=True, description="Language code"),
+                "category": fields.String(required=True, description="App category"),
+                "position": fields.Integer(required=True, description="Display position"),
+            },
+        )
+    )
+    @api.response(200, "App updated successfully")
+    @api.response(201, "App inserted successfully")
+    @api.response(404, "App not found")
     @only_edition_cloud
     @admin_required
     def post(self):
@@ -111,7 +136,12 @@ class InsertExploreAppListApi(Resource):
                 return {"result": "success"}, 200
 
 
+@console_ns.route("/admin/insert-explore-apps/<uuid:app_id>")
 class InsertExploreAppApi(Resource):
+    @api.doc("delete_explore_app")
+    @api.doc(description="Remove an app from the explore list")
+    @api.doc(params={"app_id": "Application ID to remove"})
+    @api.response(204, "App removed successfully")
     @only_edition_cloud
     @admin_required
     def delete(self, app_id):
@@ -130,21 +160,21 @@ class InsertExploreAppApi(Resource):
             app.is_public = False
 
         with Session(db.engine) as session:
-            installed_apps = session.execute(
-                select(InstalledApp).where(
-                    InstalledApp.app_id == recommended_app.app_id,
-                    InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id,
+            installed_apps = (
+                session.execute(
+                    select(InstalledApp).where(
+                        InstalledApp.app_id == recommended_app.app_id,
+                        InstalledApp.tenant_id != InstalledApp.app_owner_tenant_id,
+                    )
                 )
-            ).all()
+                .scalars()
+                .all()
+            )
 
-        for installed_app in installed_apps:
-            db.session.delete(installed_app)
+            for installed_app in installed_apps:
+                session.delete(installed_app)
 
         db.session.delete(recommended_app)
         db.session.commit()
 
         return {"result": "success"}, 204
-
-
-api.add_resource(InsertExploreAppListApi, "/admin/insert-explore-apps")
-api.add_resource(InsertExploreAppApi, "/admin/insert-explore-apps/<uuid:app_id>")

api/controllers/console/apikey.py

@@ -1,8 +1,7 @@
-from typing import Any, Optional
-
 import flask_restx
 from flask_login import current_user
 from flask_restx import Resource, fields, marshal_with
+from flask_restx._http import HTTPStatus
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
@@ -13,7 +12,7 @@ from libs.login import login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
 
-from . import api
+from . import api, console_ns
 from .wraps import account_initialization_required, setup_required
 
 api_key_fields = {
@@ -40,7 +39,7 @@ def _get_resource(resource_id, tenant_id, resource_model):
             ).scalar_one_or_none()
 
     if resource is None:
-        flask_restx.abort(404, message=f"{resource_model.__name__} not found.")
+        flask_restx.abort(HTTPStatus.NOT_FOUND, message=f"{resource_model.__name__} not found.")
 
     return resource
 
@@ -49,7 +48,7 @@ class BaseApiKeyListResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
     resource_type: str | None = None
-    resource_model: Optional[Any] = None
+    resource_model: type | None = None
     resource_id_field: str | None = None
     token_prefix: str | None = None
     max_keys = 10
@@ -59,11 +58,11 @@ class BaseApiKeyListResource(Resource):
         assert self.resource_id_field is not None, "resource_id_field must be set"
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id, self.resource_model)
-        keys = (
-            db.session.query(ApiToken)
-            .where(ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id)
-            .all()
-        )
+        keys = db.session.scalars(
+            select(ApiToken).where(
+                ApiToken.type == self.resource_type, getattr(ApiToken, self.resource_id_field) == resource_id
+            )
+        ).all()
         return {"items": keys}
 
     @marshal_with(api_key_fields)
@@ -82,12 +81,12 @@ class BaseApiKeyListResource(Resource):
 
         if current_key_count >= self.max_keys:
             flask_restx.abort(
-                400,
+                HTTPStatus.BAD_REQUEST,
                 message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
-                code="max_keys_exceeded",
+                custom="max_keys_exceeded",
             )
 
-        key = ApiToken.generate_api_key(self.token_prefix, 24)
+        key = ApiToken.generate_api_key(self.token_prefix or "", 24)
         api_token = ApiToken()
         setattr(api_token, self.resource_id_field, resource_id)
         api_token.tenant_id = current_user.current_tenant_id
@@ -102,7 +101,7 @@ class BaseApiKeyResource(Resource):
     method_decorators = [account_initialization_required, login_required, setup_required]
 
     resource_type: str | None = None
-    resource_model: Optional[Any] = None
+    resource_model: type | None = None
     resource_id_field: str | None = None
 
     def delete(self, resource_id, api_key_id):
@@ -126,7 +125,7 @@ class BaseApiKeyResource(Resource):
         )
 
         if key is None:
-            flask_restx.abort(404, message="API key not found")
+            flask_restx.abort(HTTPStatus.NOT_FOUND, message="API key not found")
 
         db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
         db.session.commit()
@@ -134,7 +133,25 @@ class BaseApiKeyResource(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:resource_id>/api-keys")
 class AppApiKeyListResource(BaseApiKeyListResource):
+    @api.doc("get_app_api_keys")
+    @api.doc(description="Get all API keys for an app")
+    @api.doc(params={"resource_id": "App ID"})
+    @api.response(200, "Success", api_key_list)
+    def get(self, resource_id):
+        """Get all API keys for an app"""
+        return super().get(resource_id)
+
+    @api.doc("create_app_api_key")
+    @api.doc(description="Create a new API key for an app")
+    @api.doc(params={"resource_id": "App ID"})
+    @api.response(201, "API key created successfully", api_key_fields)
+    @api.response(400, "Maximum keys exceeded")
+    def post(self, resource_id):
+        """Create a new API key for an app"""
+        return super().post(resource_id)
+
     def after_request(self, resp):
         resp.headers["Access-Control-Allow-Origin"] = "*"
         resp.headers["Access-Control-Allow-Credentials"] = "true"
@@ -146,7 +163,16 @@ class AppApiKeyListResource(BaseApiKeyListResource):
     token_prefix = "app-"
 
 
+@console_ns.route("/apps/<uuid:resource_id>/api-keys/<uuid:api_key_id>")
 class AppApiKeyResource(BaseApiKeyResource):
+    @api.doc("delete_app_api_key")
+    @api.doc(description="Delete an API key for an app")
+    @api.doc(params={"resource_id": "App ID", "api_key_id": "API key ID"})
+    @api.response(204, "API key deleted successfully")
+    def delete(self, resource_id, api_key_id):
+        """Delete an API key for an app"""
+        return super().delete(resource_id, api_key_id)
+
     def after_request(self, resp):
         resp.headers["Access-Control-Allow-Origin"] = "*"
         resp.headers["Access-Control-Allow-Credentials"] = "true"
@@ -157,7 +183,25 @@ class AppApiKeyResource(BaseApiKeyResource):
     resource_id_field = "app_id"
 
 
+@console_ns.route("/datasets/<uuid:resource_id>/api-keys")
 class DatasetApiKeyListResource(BaseApiKeyListResource):
+    @api.doc("get_dataset_api_keys")
+    @api.doc(description="Get all API keys for a dataset")
+    @api.doc(params={"resource_id": "Dataset ID"})
+    @api.response(200, "Success", api_key_list)
+    def get(self, resource_id):
+        """Get all API keys for a dataset"""
+        return super().get(resource_id)
+
+    @api.doc("create_dataset_api_key")
+    @api.doc(description="Create a new API key for a dataset")
+    @api.doc(params={"resource_id": "Dataset ID"})
+    @api.response(201, "API key created successfully", api_key_fields)
+    @api.response(400, "Maximum keys exceeded")
+    def post(self, resource_id):
+        """Create a new API key for a dataset"""
+        return super().post(resource_id)
+
     def after_request(self, resp):
         resp.headers["Access-Control-Allow-Origin"] = "*"
         resp.headers["Access-Control-Allow-Credentials"] = "true"
@@ -169,7 +213,16 @@ class DatasetApiKeyListResource(BaseApiKeyListResource):
     token_prefix = "ds-"
 
 
+@console_ns.route("/datasets/<uuid:resource_id>/api-keys/<uuid:api_key_id>")
 class DatasetApiKeyResource(BaseApiKeyResource):
+    @api.doc("delete_dataset_api_key")
+    @api.doc(description="Delete an API key for a dataset")
+    @api.doc(params={"resource_id": "Dataset ID", "api_key_id": "API key ID"})
+    @api.response(204, "API key deleted successfully")
+    def delete(self, resource_id, api_key_id):
+        """Delete an API key for a dataset"""
+        return super().delete(resource_id, api_key_id)
+
     def after_request(self, resp):
         resp.headers["Access-Control-Allow-Origin"] = "*"
         resp.headers["Access-Control-Allow-Credentials"] = "true"
@@ -178,9 +231,3 @@ class DatasetApiKeyResource(BaseApiKeyResource):
     resource_type = "dataset"
     resource_model = Dataset
     resource_id_field = "dataset_id"
-
-
-api.add_resource(AppApiKeyListResource, "/apps/<uuid:resource_id>/api-keys")
-api.add_resource(AppApiKeyResource, "/apps/<uuid:resource_id>/api-keys/<uuid:api_key_id>")
-api.add_resource(DatasetApiKeyListResource, "/datasets/<uuid:resource_id>/api-keys")
-api.add_resource(DatasetApiKeyResource, "/datasets/<uuid:resource_id>/api-keys/<uuid:api_key_id>")

api/controllers/console/app/advanced_prompt_template.py

@@ -1,12 +1,26 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 
 
+@console_ns.route("/app/prompt-templates")
 class AdvancedPromptTemplateList(Resource):
+    @api.doc("get_advanced_prompt_templates")
+    @api.doc(description="Get advanced prompt templates based on app mode and model configuration")
+    @api.expect(
+        api.parser()
+        .add_argument("app_mode", type=str, required=True, location="args", help="Application mode")
+        .add_argument("model_mode", type=str, required=True, location="args", help="Model mode")
+        .add_argument("has_context", type=str, default="true", location="args", help="Whether has context")
+        .add_argument("model_name", type=str, required=True, location="args", help="Model name")
+    )
+    @api.response(
+        200, "Prompt templates retrieved successfully", fields.List(fields.Raw(description="Prompt template data"))
+    )
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -19,6 +33,3 @@ class AdvancedPromptTemplateList(Resource):
         args = parser.parse_args()
 
         return AdvancedPromptTemplateService.get_prompt(args)
-
-
-api.add_resource(AdvancedPromptTemplateList, "/app/prompt-templates")

api/controllers/console/app/agent.py

@@ -1,6 +1,6 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.helper import uuid_value
@@ -9,7 +9,18 @@ from models.model import AppMode
 from services.agent_service import AgentService
 
 
+@console_ns.route("/apps/<uuid:app_id>/agent/logs")
 class AgentLogApi(Resource):
+    @api.doc("get_agent_logs")
+    @api.doc(description="Get agent execution logs for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("message_id", type=str, required=True, location="args", help="Message UUID")
+        .add_argument("conversation_id", type=str, required=True, location="args", help="Conversation UUID")
+    )
+    @api.response(200, "Agent logs retrieved successfully", fields.List(fields.Raw(description="Agent log entries")))
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -23,6 +34,3 @@ class AgentLogApi(Resource):
         args = parser.parse_args()
 
         return AgentService.get_agent_logs(app_model, args["conversation_id"], args["message_id"])
-
-
-api.add_resource(AgentLogApi, "/apps/<uuid:app_id>/agent/logs")

api/controllers/console/app/annotation.py

@@ -2,11 +2,11 @@ from typing import Literal
 
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.common.errors import NoFileUploadedError, TooManyFilesError
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_resource_check,
@@ -21,7 +21,23 @@ from libs.login import login_required
 from services.annotation_service import AppAnnotationService
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>")
 class AnnotationReplyActionApi(Resource):
+    @api.doc("annotation_reply_action")
+    @api.doc(description="Enable or disable annotation reply for an app")
+    @api.doc(params={"app_id": "Application ID", "action": "Action to perform (enable/disable)"})
+    @api.expect(
+        api.model(
+            "AnnotationReplyActionRequest",
+            {
+                "score_threshold": fields.Float(required=True, description="Score threshold for annotation matching"),
+                "embedding_provider_name": fields.String(required=True, description="Embedding provider name"),
+                "embedding_model_name": fields.String(required=True, description="Embedding model name"),
+            },
+        )
+    )
+    @api.response(200, "Action completed successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -43,7 +59,13 @@ class AnnotationReplyActionApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-setting")
 class AppAnnotationSettingDetailApi(Resource):
+    @api.doc("get_annotation_setting")
+    @api.doc(description="Get annotation settings for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Annotation settings retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -56,7 +78,23 @@ class AppAnnotationSettingDetailApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-settings/<uuid:annotation_setting_id>")
 class AppAnnotationSettingUpdateApi(Resource):
+    @api.doc("update_annotation_setting")
+    @api.doc(description="Update annotation settings for an app")
+    @api.doc(params={"app_id": "Application ID", "annotation_setting_id": "Annotation setting ID"})
+    @api.expect(
+        api.model(
+            "AnnotationSettingUpdateRequest",
+            {
+                "score_threshold": fields.Float(required=True, description="Score threshold"),
+                "embedding_provider_name": fields.String(required=True, description="Embedding provider"),
+                "embedding_model_name": fields.String(required=True, description="Embedding model"),
+            },
+        )
+    )
+    @api.response(200, "Settings updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -75,7 +113,13 @@ class AppAnnotationSettingUpdateApi(Resource):
         return result, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>/status/<uuid:job_id>")
 class AnnotationReplyActionStatusApi(Resource):
+    @api.doc("get_annotation_reply_action_status")
+    @api.doc(description="Get status of annotation reply action job")
+    @api.doc(params={"app_id": "Application ID", "job_id": "Job ID", "action": "Action type"})
+    @api.response(200, "Job status retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -99,7 +143,19 @@ class AnnotationReplyActionStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations")
 class AnnotationApi(Resource):
+    @api.doc("list_annotations")
+    @api.doc(description="Get annotations for an app with pagination")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size")
+        .add_argument("keyword", type=str, location="args", default="", help="Search keyword")
+    )
+    @api.response(200, "Annotations retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -122,6 +178,21 @@ class AnnotationApi(Resource):
         }
         return response, 200
 
+    @api.doc("create_annotation")
+    @api.doc(description="Create a new annotation for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "CreateAnnotationRequest",
+            {
+                "question": fields.String(required=True, description="Question text"),
+                "answer": fields.String(required=True, description="Answer text"),
+                "annotation_reply": fields.Raw(description="Annotation reply data"),
+            },
+        )
+    )
+    @api.response(201, "Annotation created successfully", annotation_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -168,7 +239,13 @@ class AnnotationApi(Resource):
             return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/export")
 class AnnotationExportApi(Resource):
+    @api.doc("export_annotations")
+    @api.doc(description="Export all annotations for an app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Annotations exported successfully", fields.List(fields.Nested(annotation_fields)))
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -182,7 +259,14 @@ class AnnotationExportApi(Resource):
         return response, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>")
 class AnnotationUpdateDeleteApi(Resource):
+    @api.doc("update_delete_annotation")
+    @api.doc(description="Update or delete an annotation")
+    @api.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
+    @api.response(200, "Annotation updated successfully", annotation_fields)
+    @api.response(204, "Annotation deleted successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -214,7 +298,14 @@ class AnnotationUpdateDeleteApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/batch-import")
 class AnnotationBatchImportApi(Resource):
+    @api.doc("batch_import_annotations")
+    @api.doc(description="Batch import annotations from CSV file")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Batch import started successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "No file uploaded or too many files")
     @setup_required
     @login_required
     @account_initialization_required
@@ -239,7 +330,13 @@ class AnnotationBatchImportApi(Resource):
         return AppAnnotationService.batch_import_app_annotations(app_id, file)
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/batch-import-status/<uuid:job_id>")
 class AnnotationBatchImportStatusApi(Resource):
+    @api.doc("get_batch_import_status")
+    @api.doc(description="Get status of batch import job")
+    @api.doc(params={"app_id": "Application ID", "job_id": "Job ID"})
+    @api.response(200, "Job status retrieved successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -262,7 +359,20 @@ class AnnotationBatchImportStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/<uuid:annotation_id>/hit-histories")
 class AnnotationHitHistoryListApi(Resource):
+    @api.doc("list_annotation_hit_histories")
+    @api.doc(description="Get hit histories for an annotation")
+    @api.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size")
+    )
+    @api.response(
+        200, "Hit histories retrieved successfully", fields.List(fields.Nested(annotation_hit_history_fields))
+    )
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -285,17 +395,3 @@ class AnnotationHitHistoryListApi(Resource):
             "page": page,
         }
         return response
-
-
-api.add_resource(AnnotationReplyActionApi, "/apps/<uuid:app_id>/annotation-reply/<string:action>")
-api.add_resource(
-    AnnotationReplyActionStatusApi, "/apps/<uuid:app_id>/annotation-reply/<string:action>/status/<uuid:job_id>"
-)
-api.add_resource(AnnotationApi, "/apps/<uuid:app_id>/annotations")
-api.add_resource(AnnotationExportApi, "/apps/<uuid:app_id>/annotations/export")
-api.add_resource(AnnotationUpdateDeleteApi, "/apps/<uuid:app_id>/annotations/<uuid:annotation_id>")
-api.add_resource(AnnotationBatchImportApi, "/apps/<uuid:app_id>/annotations/batch-import")
-api.add_resource(AnnotationBatchImportStatusApi, "/apps/<uuid:app_id>/annotations/batch-import-status/<uuid:job_id>")
-api.add_resource(AnnotationHitHistoryListApi, "/apps/<uuid:app_id>/annotations/<uuid:annotation_id>/hit-histories")
-api.add_resource(AppAnnotationSettingDetailApi, "/apps/<uuid:app_id>/annotation-setting")
-api.add_resource(AppAnnotationSettingUpdateApi, "/apps/<uuid:app_id>/annotation-settings/<uuid:annotation_setting_id>")

api/controllers/console/app/app.py

@@ -2,12 +2,12 @@ import uuid
 from typing import cast
 
 from flask_login import current_user
-from flask_restx import Resource, inputs, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, inputs, marshal, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden, abort
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     account_initialization_required,
@@ -34,7 +34,27 @@ def _validate_description_length(description):
     return description
 
 
+@console_ns.route("/apps")
 class AppListApi(Resource):
+    @api.doc("list_apps")
+    @api.doc(description="Get list of applications with pagination and filtering")
+    @api.expect(
+        api.parser()
+        .add_argument("page", type=int, location="args", help="Page number (1-99999)", default=1)
+        .add_argument("limit", type=int, location="args", help="Page size (1-100)", default=20)
+        .add_argument(
+            "mode",
+            type=str,
+            location="args",
+            choices=["completion", "chat", "advanced-chat", "workflow", "agent-chat", "channel", "all"],
+            default="all",
+            help="App mode filter",
+        )
+        .add_argument("name", type=str, location="args", help="Filter by app name")
+        .add_argument("tag_ids", type=str, location="args", help="Comma-separated tag IDs")
+        .add_argument("is_created_by_me", type=bool, location="args", help="Filter by creator")
+    )
+    @api.response(200, "Success", app_pagination_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -91,6 +111,24 @@ class AppListApi(Resource):
 
         return marshal(app_pagination, app_pagination_fields), 200
 
+    @api.doc("create_app")
+    @api.doc(description="Create a new application")
+    @api.expect(
+        api.model(
+            "CreateAppRequest",
+            {
+                "name": fields.String(required=True, description="App name"),
+                "description": fields.String(description="App description (max 400 chars)"),
+                "mode": fields.String(required=True, enum=ALLOW_CREATE_APP_MODES, description="App mode"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(201, "App created successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -115,12 +153,21 @@ class AppListApi(Resource):
             raise BadRequest("mode is required")
 
         app_service = AppService()
+        if not isinstance(current_user, Account):
+            raise ValueError("current_user must be an Account instance")
+        if current_user.current_tenant_id is None:
+            raise ValueError("current_user.current_tenant_id cannot be None")
         app = app_service.create_app(current_user.current_tenant_id, args, current_user)
 
         return app, 201
 
 
+@console_ns.route("/apps/<uuid:app_id>")
 class AppApi(Resource):
+    @api.doc("get_app_detail")
+    @api.doc(description="Get application details")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Success", app_detail_fields_with_site)
     @setup_required
     @login_required
     @account_initialization_required
@@ -139,6 +186,26 @@ class AppApi(Resource):
 
         return app_model
 
+    @api.doc("update_app")
+    @api.doc(description="Update application details")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "UpdateAppRequest",
+            {
+                "name": fields.String(required=True, description="App name"),
+                "description": fields.String(description="App description (max 400 chars)"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+                "use_icon_as_answer_icon": fields.Boolean(description="Use icon as answer icon"),
+                "max_active_requests": fields.Integer(description="Maximum active requests"),
+            },
+        )
+    )
+    @api.response(200, "App updated successfully", app_detail_fields_with_site)
+    @api.response(403, "Insufficient permissions")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -161,14 +228,31 @@ class AppApi(Resource):
         args = parser.parse_args()
 
         app_service = AppService()
-        app_model = app_service.update_app(app_model, args)
+        # Construct ArgsDict from parsed arguments
+        from services.app_service import AppService as AppServiceType
+
+        args_dict: AppServiceType.ArgsDict = {
+            "name": args["name"],
+            "description": args.get("description", ""),
+            "icon_type": args.get("icon_type", ""),
+            "icon": args.get("icon", ""),
+            "icon_background": args.get("icon_background", ""),
+            "use_icon_as_answer_icon": args.get("use_icon_as_answer_icon", False),
+            "max_active_requests": args.get("max_active_requests", 0),
+        }
+        app_model = app_service.update_app(app_model, args_dict)
 
         return app_model
 
+    @api.doc("delete_app")
+    @api.doc(description="Delete application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(204, "App deleted successfully")
+    @api.response(403, "Insufficient permissions")
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def delete(self, app_model):
         """Delete app"""
         # The role of the current user in the ta table must be admin, owner, or editor
@@ -181,7 +265,25 @@ class AppApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/copy")
 class AppCopyApi(Resource):
+    @api.doc("copy_app")
+    @api.doc(description="Create a copy of an existing application")
+    @api.doc(params={"app_id": "Application ID to copy"})
+    @api.expect(
+        api.model(
+            "CopyAppRequest",
+            {
+                "name": fields.String(description="Name for the copied app"),
+                "description": fields.String(description="Description for the copied app"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon": fields.String(description="Icon"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(201, "App copied successfully", app_detail_fields_with_site)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -223,11 +325,26 @@ class AppCopyApi(Resource):
         return app, 201
 
 
+@console_ns.route("/apps/<uuid:app_id>/export")
 class AppExportApi(Resource):
+    @api.doc("export_app")
+    @api.doc(description="Export application configuration as DSL")
+    @api.doc(params={"app_id": "Application ID to export"})
+    @api.expect(
+        api.parser()
+        .add_argument("include_secret", type=bool, location="args", default=False, help="Include secrets in export")
+        .add_argument("workflow_id", type=str, location="args", help="Specific workflow ID to export")
+    )
+    @api.response(
+        200,
+        "App exported successfully",
+        api.model("AppExportResponse", {"data": fields.String(description="DSL export data")}),
+    )
+    @api.response(403, "Insufficient permissions")
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def get(self, app_model):
         """Export app"""
         # The role of the current user in the ta table must be admin, owner, or editor
@@ -237,12 +354,23 @@ class AppExportApi(Resource):
         # Add include_secret params
         parser = reqparse.RequestParser()
         parser.add_argument("include_secret", type=inputs.boolean, default=False, location="args")
+        parser.add_argument("workflow_id", type=str, location="args")
         args = parser.parse_args()
 
-        return {"data": AppDslService.export_dsl(app_model=app_model, include_secret=args["include_secret"])}
+        return {
+            "data": AppDslService.export_dsl(
+                app_model=app_model, include_secret=args["include_secret"], workflow_id=args.get("workflow_id")
+            )
+        }
 
 
+@console_ns.route("/apps/<uuid:app_id>/name")
 class AppNameApi(Resource):
+    @api.doc("check_app_name")
+    @api.doc(description="Check if app name is available")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(api.parser().add_argument("name", type=str, required=True, location="args", help="Name to check"))
+    @api.response(200, "Name availability checked")
     @setup_required
     @login_required
     @account_initialization_required
@@ -258,12 +386,28 @@ class AppNameApi(Resource):
         args = parser.parse_args()
 
         app_service = AppService()
-        app_model = app_service.update_app_name(app_model, args.get("name"))
+        app_model = app_service.update_app_name(app_model, args["name"])
 
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/icon")
 class AppIconApi(Resource):
+    @api.doc("update_app_icon")
+    @api.doc(description="Update application icon")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppIconRequest",
+            {
+                "icon": fields.String(required=True, description="Icon data"),
+                "icon_type": fields.String(description="Icon type"),
+                "icon_background": fields.String(description="Icon background color"),
+            },
+        )
+    )
+    @api.response(200, "Icon updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -280,12 +424,23 @@ class AppIconApi(Resource):
         args = parser.parse_args()
 
         app_service = AppService()
-        app_model = app_service.update_app_icon(app_model, args.get("icon"), args.get("icon_background"))
+        app_model = app_service.update_app_icon(app_model, args.get("icon") or "", args.get("icon_background") or "")
 
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/site-enable")
 class AppSiteStatus(Resource):
+    @api.doc("update_app_site_status")
+    @api.doc(description="Enable or disable app site")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppSiteStatusRequest", {"enable_site": fields.Boolean(required=True, description="Enable or disable site")}
+        )
+    )
+    @api.response(200, "Site status updated successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -301,12 +456,23 @@ class AppSiteStatus(Resource):
         args = parser.parse_args()
 
         app_service = AppService()
-        app_model = app_service.update_app_site_status(app_model, args.get("enable_site"))
+        app_model = app_service.update_app_site_status(app_model, args["enable_site"])
 
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/api-enable")
 class AppApiStatus(Resource):
+    @api.doc("update_app_api_status")
+    @api.doc(description="Enable or disable app API")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppApiStatusRequest", {"enable_api": fields.Boolean(required=True, description="Enable or disable API")}
+        )
+    )
+    @api.response(200, "API status updated successfully", app_detail_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -322,12 +488,17 @@ class AppApiStatus(Resource):
         args = parser.parse_args()
 
         app_service = AppService()
-        app_model = app_service.update_app_api_status(app_model, args.get("enable_api"))
+        app_model = app_service.update_app_api_status(app_model, args["enable_api"])
 
         return app_model
 
 
+@console_ns.route("/apps/<uuid:app_id>/trace")
 class AppTraceApi(Resource):
+    @api.doc("get_app_trace")
+    @api.doc(description="Get app tracing configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Trace configuration retrieved successfully")
     @setup_required
     @login_required
     @account_initialization_required
@@ -337,6 +508,20 @@ class AppTraceApi(Resource):
 
         return app_trace_config
 
+    @api.doc("update_app_trace")
+    @api.doc(description="Update app tracing configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "AppTraceRequest",
+            {
+                "enabled": fields.Boolean(required=True, description="Enable or disable tracing"),
+                "tracing_provider": fields.String(required=True, description="Tracing provider"),
+            },
+        )
+    )
+    @api.response(200, "Trace configuration updated successfully")
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -356,14 +541,3 @@ class AppTraceApi(Resource):
         )
 
         return {"result": "success"}
-
-
-api.add_resource(AppListApi, "/apps")
-api.add_resource(AppApi, "/apps/<uuid:app_id>")
-api.add_resource(AppCopyApi, "/apps/<uuid:app_id>/copy")
-api.add_resource(AppExportApi, "/apps/<uuid:app_id>/export")
-api.add_resource(AppNameApi, "/apps/<uuid:app_id>/name")
-api.add_resource(AppIconApi, "/apps/<uuid:app_id>/icon")
-api.add_resource(AppSiteStatus, "/apps/<uuid:app_id>/site-enable")
-api.add_resource(AppApiStatus, "/apps/<uuid:app_id>/api-enable")
-api.add_resource(AppTraceApi, "/apps/<uuid:app_id>/trace")

api/controllers/console/app/audio.py

@@ -1,11 +1,11 @@
 import logging
 
 from flask import request
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 from werkzeug.exceptions import InternalServerError
 
 import services
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
     AudioTooLargeError,
@@ -34,7 +34,18 @@ from services.errors.audio import (
 logger = logging.getLogger(__name__)
 
 
+@console_ns.route("/apps/<uuid:app_id>/audio-to-text")
 class ChatMessageAudioApi(Resource):
+    @api.doc("chat_message_audio_transcript")
+    @api.doc(description="Transcript audio to text for chat messages")
+    @api.doc(params={"app_id": "App ID"})
+    @api.response(
+        200,
+        "Audio transcription successful",
+        api.model("AudioTranscriptResponse", {"text": fields.String(description="Transcribed text from audio")}),
+    )
+    @api.response(400, "Bad request - No audio uploaded or unsupported type")
+    @api.response(413, "Audio file too large")
     @setup_required
     @login_required
     @account_initialization_required
@@ -76,11 +87,28 @@ class ChatMessageAudioApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/text-to-audio")
 class ChatMessageTextApi(Resource):
+    @api.doc("chat_message_text_to_speech")
+    @api.doc(description="Convert text to speech for chat messages")
+    @api.doc(params={"app_id": "App ID"})
+    @api.expect(
+        api.model(
+            "TextToSpeechRequest",
+            {
+                "message_id": fields.String(description="Message ID"),
+                "text": fields.String(required=True, description="Text to convert to speech"),
+                "voice": fields.String(description="Voice to use for TTS"),
+                "streaming": fields.Boolean(description="Whether to stream the audio"),
+            },
+        )
+    )
+    @api.response(200, "Text to speech conversion successful")
+    @api.response(400, "Bad request - Invalid parameters")
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def post(self, app_model: App):
         try:
             parser = reqparse.RequestParser()
@@ -124,11 +152,18 @@ class ChatMessageTextApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/text-to-audio/voices")
 class TextModesApi(Resource):
+    @api.doc("get_text_to_speech_voices")
+    @api.doc(description="Get available TTS voices for a specific language")
+    @api.doc(params={"app_id": "App ID"})
+    @api.expect(api.parser().add_argument("language", type=str, required=True, location="args", help="Language code"))
+    @api.response(200, "TTS voices retrieved successfully", fields.List(fields.Raw(description="Available voices")))
+    @api.response(400, "Invalid language parameter")
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def get(self, app_model):
         try:
             parser = reqparse.RequestParser()
@@ -164,8 +199,3 @@ class TextModesApi(Resource):
         except Exception as e:
             logger.exception("Failed to handle get request to TextModesApi")
             raise InternalServerError()
-
-
-api.add_resource(ChatMessageAudioApi, "/apps/<uuid:app_id>/audio-to-text")
-api.add_resource(ChatMessageTextApi, "/apps/<uuid:app_id>/text-to-audio")
-api.add_resource(TextModesApi, "/apps/<uuid:app_id>/text-to-audio/voices")

api/controllers/console/app/completion.py

@@ -1,12 +1,11 @@
 import logging
 
-import flask_login
 from flask import request
-from flask_restx import Resource, reqparse
-from werkzeug.exceptions import InternalServerError, NotFound
+from flask_restx import Resource, fields, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
     CompletionRequestError,
@@ -29,7 +28,8 @@ from core.helper.trace_id_helper import get_external_trace_id
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
-from libs.login import login_required
+from libs.login import current_user, login_required
+from models import Account
 from models.model import AppMode
 from services.app_generate_service import AppGenerateService
 from services.errors.llm import InvokeRateLimitError
@@ -38,7 +38,27 @@ logger = logging.getLogger(__name__)
 
 
 # define completion message api for user
+@console_ns.route("/apps/<uuid:app_id>/completion-messages")
 class CompletionMessageApi(Resource):
+    @api.doc("create_completion_message")
+    @api.doc(description="Generate completion message for debugging")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "CompletionMessageRequest",
+            {
+                "inputs": fields.Raw(required=True, description="Input variables"),
+                "query": fields.String(description="Query text", default=""),
+                "files": fields.List(fields.Raw(), description="Uploaded files"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "response_mode": fields.String(enum=["blocking", "streaming"], description="Response mode"),
+                "retriever_from": fields.String(default="dev", description="Retriever source"),
+            },
+        )
+    )
+    @api.response(200, "Completion generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(404, "App not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -56,11 +76,11 @@ class CompletionMessageApi(Resource):
         streaming = args["response_mode"] != "blocking"
         args["auto_generate_name"] = False
 
-        account = flask_login.current_user
-
         try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account or EndUser instance")
             response = AppGenerateService.generate(
-                app_model=app_model, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
+                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
             )
 
             return helper.compact_generate_response(response)
@@ -86,25 +106,58 @@ class CompletionMessageApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop")
 class CompletionMessageStopApi(Resource):
+    @api.doc("stop_completion_message")
+    @api.doc(description="Stop a running completion message generation")
+    @api.doc(params={"app_id": "Application ID", "task_id": "Task ID to stop"})
+    @api.response(200, "Task stopped successfully")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=AppMode.COMPLETION)
     def post(self, app_model, task_id):
-        account = flask_login.current_user
-
-        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
+        if not isinstance(current_user, Account):
+            raise ValueError("current_user must be an Account instance")
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
 
         return {"result": "success"}, 200
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages")
 class ChatMessageApi(Resource):
+    @api.doc("create_chat_message")
+    @api.doc(description="Generate chat message for debugging")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "ChatMessageRequest",
+            {
+                "inputs": fields.Raw(required=True, description="Input variables"),
+                "query": fields.String(required=True, description="User query"),
+                "files": fields.List(fields.Raw(), description="Uploaded files"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "conversation_id": fields.String(description="Conversation ID"),
+                "parent_message_id": fields.String(description="Parent message ID"),
+                "response_mode": fields.String(enum=["blocking", "streaming"], description="Response mode"),
+                "retriever_from": fields.String(default="dev", description="Retriever source"),
+            },
+        )
+    )
+    @api.response(200, "Chat message generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(404, "App or conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT])
     def post(self, app_model):
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        if not current_user.has_edit_permission:
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("inputs", type=dict, required=True, location="json")
         parser.add_argument("query", type=str, required=True, location="json")
@@ -123,11 +176,11 @@ class ChatMessageApi(Resource):
         if external_trace_id:
             args["external_trace_id"] = external_trace_id
 
-        account = flask_login.current_user
-
         try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account or EndUser instance")
             response = AppGenerateService.generate(
-                app_model=app_model, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
+                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
             )
 
             return helper.compact_generate_response(response)
@@ -155,20 +208,19 @@ class ChatMessageApi(Resource):
             raise InternalServerError()
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")
 class ChatMessageStopApi(Resource):
+    @api.doc("stop_chat_message")
+    @api.doc(description="Stop a running chat message generation")
+    @api.doc(params={"app_id": "Application ID", "task_id": "Task ID to stop"})
+    @api.response(200, "Task stopped successfully")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     def post(self, app_model, task_id):
-        account = flask_login.current_user
-
-        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, account.id)
+        if not isinstance(current_user, Account):
+            raise ValueError("current_user must be an Account instance")
+        AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
 
         return {"result": "success"}, 200
-
-
-api.add_resource(CompletionMessageApi, "/apps/<uuid:app_id>/completion-messages")
-api.add_resource(CompletionMessageStopApi, "/apps/<uuid:app_id>/completion-messages/<string:task_id>/stop")
-api.add_resource(ChatMessageApi, "/apps/<uuid:app_id>/chat-messages")
-api.add_resource(ChatMessageStopApi, "/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")

api/controllers/console/app/conversation.py

@@ -8,7 +8,7 @@ from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
 from werkzeug.exceptions import Forbidden, NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -22,13 +22,35 @@ from fields.conversation_fields import (
 from libs.datetime_utils import naive_utc_now
 from libs.helper import DatetimeString
 from libs.login import login_required
-from models import Conversation, EndUser, Message, MessageAnnotation
+from models import Account, Conversation, EndUser, Message, MessageAnnotation
 from models.model import AppMode
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-conversations")
 class CompletionConversationApi(Resource):
+    @api.doc("list_completion_conversations")
+    @api.doc(description="Get completion conversations with pagination and filtering")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("keyword", type=str, location="args", help="Search keyword")
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+        .add_argument(
+            "annotation_status",
+            type=str,
+            location="args",
+            choices=["annotated", "not_annotated", "all"],
+            default="all",
+            help="Annotation status filter",
+        )
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size (1-100)")
+    )
+    @api.response(200, "Success", conversation_pagination_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -101,7 +123,14 @@ class CompletionConversationApi(Resource):
         return conversations
 
 
+@console_ns.route("/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>")
 class CompletionConversationDetailApi(Resource):
+    @api.doc("get_completion_conversation")
+    @api.doc(description="Get completion conversation details with messages")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(200, "Success", conversation_message_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -114,16 +143,24 @@ class CompletionConversationDetailApi(Resource):
 
         return _get_conversation(app_model, conversation_id)
 
+    @api.doc("delete_completion_conversation")
+    @api.doc(description="Delete a completion conversation")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(204, "Conversation deleted successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
+    @get_app_model(mode=AppMode.COMPLETION)
     def delete(self, app_model, conversation_id):
         if not current_user.is_editor:
             raise Forbidden()
         conversation_id = str(conversation_id)
 
         try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
             ConversationService.delete(app_model, conversation_id, current_user)
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -131,7 +168,38 @@ class CompletionConversationDetailApi(Resource):
         return {"result": "success"}, 204
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-conversations")
 class ChatConversationApi(Resource):
+    @api.doc("list_chat_conversations")
+    @api.doc(description="Get chat conversations with pagination, filtering and summary")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("keyword", type=str, location="args", help="Search keyword")
+        .add_argument("start", type=str, location="args", help="Start date (YYYY-MM-DD HH:MM)")
+        .add_argument("end", type=str, location="args", help="End date (YYYY-MM-DD HH:MM)")
+        .add_argument(
+            "annotation_status",
+            type=str,
+            location="args",
+            choices=["annotated", "not_annotated", "all"],
+            default="all",
+            help="Annotation status filter",
+        )
+        .add_argument("message_count_gte", type=int, location="args", help="Minimum message count")
+        .add_argument("page", type=int, location="args", default=1, help="Page number")
+        .add_argument("limit", type=int, location="args", default=20, help="Page size (1-100)")
+        .add_argument(
+            "sort_by",
+            type=str,
+            location="args",
+            choices=["created_at", "-created_at", "updated_at", "-updated_at"],
+            default="-updated_at",
+            help="Sort field and direction",
+        )
+    )
+    @api.response(200, "Success", conversation_with_summary_pagination_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -239,7 +307,7 @@ class ChatConversationApi(Resource):
                 .having(func.count(Message.id) >= args["message_count_gte"])
             )
 
-        if app_model.mode == AppMode.ADVANCED_CHAT.value:
+        if app_model.mode == AppMode.ADVANCED_CHAT:
             query = query.where(Conversation.invoke_from != InvokeFrom.DEBUGGER.value)
 
         match args["sort_by"]:
@@ -259,7 +327,14 @@ class ChatConversationApi(Resource):
         return conversations
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>")
 class ChatConversationDetailApi(Resource):
+    @api.doc("get_chat_conversation")
+    @api.doc(description="Get chat conversation details")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(200, "Success", conversation_detail_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -272,6 +347,12 @@ class ChatConversationDetailApi(Resource):
 
         return _get_conversation(app_model, conversation_id)
 
+    @api.doc("delete_chat_conversation")
+    @api.doc(description="Delete a chat conversation")
+    @api.doc(params={"app_id": "Application ID", "conversation_id": "Conversation ID"})
+    @api.response(204, "Conversation deleted successfully")
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
@@ -282,6 +363,8 @@ class ChatConversationDetailApi(Resource):
         conversation_id = str(conversation_id)
 
         try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
             ConversationService.delete(app_model, conversation_id, current_user)
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -289,12 +372,6 @@ class ChatConversationDetailApi(Resource):
         return {"result": "success"}, 204
 
 
-api.add_resource(CompletionConversationApi, "/apps/<uuid:app_id>/completion-conversations")
-api.add_resource(CompletionConversationDetailApi, "/apps/<uuid:app_id>/completion-conversations/<uuid:conversation_id>")
-api.add_resource(ChatConversationApi, "/apps/<uuid:app_id>/chat-conversations")
-api.add_resource(ChatConversationDetailApi, "/apps/<uuid:app_id>/chat-conversations/<uuid:conversation_id>")
-
-
 def _get_conversation(app_model, conversation_id):
     conversation = (
         db.session.query(Conversation)

api/controllers/console/app/conversation_variables.py

@@ -2,7 +2,7 @@ from flask_restx import Resource, marshal_with, reqparse
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -12,7 +12,17 @@ from models import ConversationVariable
 from models.model import AppMode
 
 
+@console_ns.route("/apps/<uuid:app_id>/conversation-variables")
 class ConversationVariablesApi(Resource):
+    @api.doc("get_conversation_variables")
+    @api.doc(description="Get conversation variables for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "conversation_id", type=str, location="args", help="Conversation ID to filter variables"
+        )
+    )
+    @api.response(200, "Conversation variables retrieved successfully", paginated_conversation_variable_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -55,6 +65,3 @@ class ConversationVariablesApi(Resource):
                 for row in rows
             ],
         }
-
-
-api.add_resource(ConversationVariablesApi, "/apps/<uuid:app_id>/conversation-variables")

api/controllers/console/app/generator.py

@@ -1,9 +1,9 @@
 from collections.abc import Sequence
 
 from flask_login import current_user
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -16,10 +16,29 @@ from core.helper.code_executor.javascript.javascript_code_provider import Javasc
 from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
+from extensions.ext_database import db
 from libs.login import login_required
+from models import App
+from services.workflow_service import WorkflowService
 
 
+@console_ns.route("/rule-generate")
 class RuleGenerateApi(Resource):
+    @api.doc("generate_rule_config")
+    @api.doc(description="Generate rule configuration using LLM")
+    @api.expect(
+        api.model(
+            "RuleGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Rule generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "no_variable": fields.Boolean(required=True, default=False, description="Whether to exclude variables"),
+            },
+        )
+    )
+    @api.response(200, "Rule configuration generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -50,7 +69,26 @@ class RuleGenerateApi(Resource):
         return rules
 
 
+@console_ns.route("/rule-code-generate")
 class RuleCodeGenerateApi(Resource):
+    @api.doc("generate_rule_code")
+    @api.doc(description="Generate code rules using LLM")
+    @api.expect(
+        api.model(
+            "RuleCodeGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Code generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "no_variable": fields.Boolean(required=True, default=False, description="Whether to exclude variables"),
+                "code_language": fields.String(
+                    default="javascript", description="Programming language for code generation"
+                ),
+            },
+        )
+    )
+    @api.response(200, "Code rules generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -82,7 +120,22 @@ class RuleCodeGenerateApi(Resource):
         return code_result
 
 
+@console_ns.route("/rule-structured-output-generate")
 class RuleStructuredOutputGenerateApi(Resource):
+    @api.doc("generate_structured_output")
+    @api.doc(description="Generate structured output rules using LLM")
+    @api.expect(
+        api.model(
+            "StructuredOutputGenerateRequest",
+            {
+                "instruction": fields.String(required=True, description="Structured output generation instruction"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+            },
+        )
+    )
+    @api.response(200, "Structured output generated successfully")
+    @api.response(400, "Invalid request parameters")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -111,7 +164,27 @@ class RuleStructuredOutputGenerateApi(Resource):
         return structured_output
 
 
+@console_ns.route("/instruction-generate")
 class InstructionGenerateApi(Resource):
+    @api.doc("generate_instruction")
+    @api.doc(description="Generate instruction for workflow nodes or general use")
+    @api.expect(
+        api.model(
+            "InstructionGenerateRequest",
+            {
+                "flow_id": fields.String(required=True, description="Workflow/Flow ID"),
+                "node_id": fields.String(description="Node ID for workflow context"),
+                "current": fields.String(description="Current instruction text"),
+                "language": fields.String(default="javascript", description="Programming language (javascript/python)"),
+                "instruction": fields.String(required=True, description="Instruction for generation"),
+                "model_config": fields.Raw(required=True, description="Model configuration"),
+                "ideal_output": fields.String(description="Expected ideal output"),
+            },
+        )
+    )
+    @api.response(200, "Instruction generated successfully")
+    @api.response(400, "Invalid request parameters or flow/workflow not found")
+    @api.response(402, "Provider quota exceeded")
     @setup_required
     @login_required
     @account_initialization_required
@@ -135,9 +208,6 @@ class InstructionGenerateApi(Resource):
         try:
             # Generate from nothing for a workflow node
             if (args["current"] == code_template or args["current"] == "") and args["node_id"] != "":
-                from models import App, db
-                from services.workflow_service import WorkflowService
-
                 app = db.session.query(App).where(App.id == args["flow_id"]).first()
                 if not app:
                     return {"error": f"app {args['flow_id']} not found"}, 400
@@ -191,6 +261,7 @@ class InstructionGenerateApi(Resource):
                     instruction=args["instruction"],
                     model_config=args["model_config"],
                     ideal_output=args["ideal_output"],
+                    workflow_service=WorkflowService(),
                 )
             return {"error": "incompatible parameters"}, 400
         except ProviderTokenNotInitError as ex:
@@ -203,11 +274,25 @@ class InstructionGenerateApi(Resource):
             raise CompletionRequestError(e.description)
 
 
+@console_ns.route("/instruction-generate/template")
 class InstructionGenerationTemplateApi(Resource):
+    @api.doc("get_instruction_template")
+    @api.doc(description="Get instruction generation template")
+    @api.expect(
+        api.model(
+            "InstructionTemplateRequest",
+            {
+                "instruction": fields.String(required=True, description="Template instruction"),
+                "ideal_output": fields.String(description="Expected ideal output"),
+            },
+        )
+    )
+    @api.response(200, "Template retrieved successfully")
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
-    def post(self) -> dict:
+    def post(self):
         parser = reqparse.RequestParser()
         parser.add_argument("type", type=str, required=True, default=False, location="json")
         args = parser.parse_args()
@@ -222,10 +307,3 @@ class InstructionGenerationTemplateApi(Resource):
                 return {"data": INSTRUCTION_GENERATE_TEMPLATE_CODE}
             case _:
                 raise ValueError(f"Invalid type: {args['type']}")
-
-
-api.add_resource(RuleGenerateApi, "/rule-generate")
-api.add_resource(RuleCodeGenerateApi, "/rule-code-generate")
-api.add_resource(RuleStructuredOutputGenerateApi, "/rule-structured-output-generate")
-api.add_resource(InstructionGenerateApi, "/instruction-generate")
-api.add_resource(InstructionGenerationTemplateApi, "/instruction-generate/template")

api/controllers/console/app/mcp_server.py

@@ -2,10 +2,10 @@ import json
 from enum import StrEnum
 
 from flask_login import current_user
-from flask_restx import Resource, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from extensions.ext_database import db
@@ -19,7 +19,12 @@ class AppMCPServerStatus(StrEnum):
     INACTIVE = "inactive"
 
 
+@console_ns.route("/apps/<uuid:app_id>/server")
 class AppMCPServerController(Resource):
+    @api.doc("get_app_mcp_server")
+    @api.doc(description="Get MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "MCP server configuration retrieved successfully", app_server_fields)
     @setup_required
     @login_required
     @account_initialization_required
@@ -29,6 +34,20 @@ class AppMCPServerController(Resource):
         server = db.session.query(AppMCPServer).where(AppMCPServer.app_id == app_model.id).first()
         return server
 
+    @api.doc("create_app_mcp_server")
+    @api.doc(description="Create MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MCPServerCreateRequest",
+            {
+                "description": fields.String(description="Server description"),
+                "parameters": fields.Raw(required=True, description="Server parameters configuration"),
+            },
+        )
+    )
+    @api.response(201, "MCP server configuration created successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -59,6 +78,23 @@ class AppMCPServerController(Resource):
         db.session.commit()
         return server
 
+    @api.doc("update_app_mcp_server")
+    @api.doc(description="Update MCP server configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MCPServerUpdateRequest",
+            {
+                "id": fields.String(required=True, description="Server ID"),
+                "description": fields.String(description="Server description"),
+                "parameters": fields.Raw(required=True, description="Server parameters configuration"),
+                "status": fields.String(description="Server status"),
+            },
+        )
+    )
+    @api.response(200, "MCP server configuration updated successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Server not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -94,7 +130,14 @@ class AppMCPServerController(Resource):
         return server
 
 
+@console_ns.route("/apps/<uuid:server_id>/server/refresh")
 class AppMCPServerRefreshController(Resource):
+    @api.doc("refresh_app_mcp_server")
+    @api.doc(description="Refresh MCP server configuration and regenerate server code")
+    @api.doc(params={"server_id": "Server ID"})
+    @api.response(200, "MCP server refreshed successfully", app_server_fields)
+    @api.response(403, "Insufficient permissions")
+    @api.response(404, "Server not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -113,7 +156,3 @@ class AppMCPServerRefreshController(Resource):
         server.server_code = AppMCPServer.generate_server_code(16)
         db.session.commit()
         return server
-
-
-api.add_resource(AppMCPServerController, "/apps/<uuid:app_id>/server")
-api.add_resource(AppMCPServerRefreshController, "/apps/<uuid:server_id>/server/refresh")

api/controllers/console/app/message.py

@@ -1,12 +1,11 @@
 import logging
 
-from flask_login import current_user
 from flask_restx import Resource, fields, marshal_with, reqparse
 from flask_restx.inputs import int_range
 from sqlalchemy import exists, select
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -27,7 +26,8 @@ from extensions.ext_database import db
 from fields.conversation_fields import annotation_fields, message_detail_fields
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
-from libs.login import login_required
+from libs.login import current_user, login_required
+from models.account import Account
 from models.model import AppMode, Conversation, Message, MessageAnnotation, MessageFeedback
 from services.annotation_service import AppAnnotationService
 from services.errors.conversation import ConversationNotExistsError
@@ -37,6 +37,7 @@ from services.message_service import MessageService
 logger = logging.getLogger(__name__)
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages")
 class ChatMessageListApi(Resource):
     message_infinite_scroll_pagination_fields = {
         "limit": fields.Integer,
@@ -44,6 +45,17 @@ class ChatMessageListApi(Resource):
         "data": fields.List(fields.Nested(message_detail_fields)),
     }
 
+    @api.doc("list_chat_messages")
+    @api.doc(description="Get chat messages for a conversation with pagination")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser()
+        .add_argument("conversation_id", type=str, required=True, location="args", help="Conversation ID")
+        .add_argument("first_id", type=str, location="args", help="First message ID for pagination")
+        .add_argument("limit", type=int, location="args", default=20, help="Number of messages to return (1-100)")
+    )
+    @api.response(200, "Success", message_infinite_scroll_pagination_fields)
+    @api.response(404, "Conversation not found")
     @setup_required
     @login_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
@@ -95,30 +107,53 @@ class ChatMessageListApi(Resource):
                 .all()
             )
 
+        # Initialize has_more based on whether we have a full page
         if len(history_messages) == args["limit"]:
             current_page_first_message = history_messages[-1]
-
-        has_more = db.session.scalar(
-            select(
-                exists().where(
-                    Message.conversation_id == conversation.id,
-                    Message.created_at < current_page_first_message.created_at,
-                    Message.id != current_page_first_message.id,
+            # Check if there are more messages before the current page
+            has_more = db.session.scalar(
+                select(
+                    exists().where(
+                        Message.conversation_id == conversation.id,
+                        Message.created_at < current_page_first_message.created_at,
+                        Message.id != current_page_first_message.id,
+                    )
                 )
             )
-        )
+        else:
+            # If we don't have a full page, there are no more messages
+            has_more = False
 
         history_messages = list(reversed(history_messages))
 
         return InfiniteScrollPagination(data=history_messages, limit=args["limit"], has_more=has_more)
 
 
+@console_ns.route("/apps/<uuid:app_id>/feedbacks")
 class MessageFeedbackApi(Resource):
+    @api.doc("create_message_feedback")
+    @api.doc(description="Create or update message feedback (like/dislike)")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MessageFeedbackRequest",
+            {
+                "message_id": fields.String(required=True, description="Message ID"),
+                "rating": fields.String(enum=["like", "dislike"], description="Feedback rating"),
+            },
+        )
+    )
+    @api.response(200, "Feedback updated successfully")
+    @api.response(404, "Message not found")
+    @api.response(403, "Insufficient permissions")
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def post(self, app_model):
+        if current_user is None:
+            raise Forbidden()
+
         parser = reqparse.RequestParser()
         parser.add_argument("message_id", required=True, type=uuid_value, location="json")
         parser.add_argument("rating", type=str, choices=["like", "dislike", None], location="json")
@@ -126,7 +161,7 @@ class MessageFeedbackApi(Resource):
 
         message_id = str(args["message_id"])
 
-        message = db.session.query(Message).filter(Message.id == message_id, Message.app_id == app_model.id).first()
+        message = db.session.query(Message).where(Message.id == message_id, Message.app_id == app_model.id).first()
 
         if not message:
             raise NotFound("Message Not Exists.")
@@ -155,7 +190,24 @@ class MessageFeedbackApi(Resource):
         return {"result": "success"}
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations")
 class MessageAnnotationApi(Resource):
+    @api.doc("create_message_annotation")
+    @api.doc(description="Create message annotation")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "MessageAnnotationRequest",
+            {
+                "message_id": fields.String(description="Message ID"),
+                "question": fields.String(required=True, description="Question text"),
+                "answer": fields.String(required=True, description="Answer text"),
+                "annotation_reply": fields.Raw(description="Annotation reply"),
+            },
+        )
+    )
+    @api.response(200, "Annotation created successfully", annotation_fields)
+    @api.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
@@ -163,7 +215,9 @@ class MessageAnnotationApi(Resource):
     @get_app_model
     @marshal_with(annotation_fields)
     def post(self, app_model):
-        if not current_user.is_editor:
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+        if not current_user.has_edit_permission:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -177,18 +231,37 @@ class MessageAnnotationApi(Resource):
         return annotation
 
 
+@console_ns.route("/apps/<uuid:app_id>/annotations/count")
 class MessageAnnotationCountApi(Resource):
+    @api.doc("get_annotation_count")
+    @api.doc(description="Get count of message annotations for the app")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(
+        200,
+        "Annotation count retrieved successfully",
+        api.model("AnnotationCountResponse", {"count": fields.Integer(description="Number of annotations")}),
+    )
+    @get_app_model
     @setup_required
     @login_required
     @account_initialization_required
-    @get_app_model
     def get(self, app_model):
         count = db.session.query(MessageAnnotation).where(MessageAnnotation.app_id == app_model.id).count()
 
         return {"count": count}
 
 
+@console_ns.route("/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions")
 class MessageSuggestedQuestionApi(Resource):
+    @api.doc("get_message_suggested_questions")
+    @api.doc(description="Get suggested questions for a message")
+    @api.doc(params={"app_id": "Application ID", "message_id": "Message ID"})
+    @api.response(
+        200,
+        "Suggested questions retrieved successfully",
+        api.model("SuggestedQuestionsResponse", {"data": fields.List(fields.String(description="Suggested question"))}),
+    )
+    @api.response(404, "Message or conversation not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -221,7 +294,13 @@ class MessageSuggestedQuestionApi(Resource):
         return {"data": questions}
 
 
+@console_ns.route("/apps/<uuid:app_id>/messages/<uuid:message_id>")
 class MessageApi(Resource):
+    @api.doc("get_message")
+    @api.doc(description="Get message details by ID")
+    @api.doc(params={"app_id": "Application ID", "message_id": "Message ID"})
+    @api.response(200, "Message retrieved successfully", message_detail_fields)
+    @api.response(404, "Message not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -236,11 +315,3 @@ class MessageApi(Resource):
             raise NotFound("Message Not Exists.")
 
         return message
-
-
-api.add_resource(MessageSuggestedQuestionApi, "/apps/<uuid:app_id>/chat-messages/<uuid:message_id>/suggested-questions")
-api.add_resource(ChatMessageListApi, "/apps/<uuid:app_id>/chat-messages", endpoint="console_chat_messages")
-api.add_resource(MessageFeedbackApi, "/apps/<uuid:app_id>/feedbacks")
-api.add_resource(MessageAnnotationApi, "/apps/<uuid:app_id>/annotations")
-api.add_resource(MessageAnnotationCountApi, "/apps/<uuid:app_id>/annotations/count")
-api.add_resource(MessageApi, "/apps/<uuid:app_id>/messages/<uuid:message_id>", endpoint="console_message")

api/controllers/console/app/model_config.py

@@ -3,9 +3,10 @@ from typing import cast
 
 from flask import request
 from flask_login import current_user
-from flask_restx import Resource
+from flask_restx import Resource, fields
+from werkzeug.exceptions import Forbidden
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.agent.entities import AgentToolEntity
@@ -14,17 +15,51 @@ from core.tools.utils.configuration import ToolParameterConfigurationManager
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
 from libs.login import login_required
+from models.account import Account
 from models.model import AppMode, AppModelConfig
 from services.app_model_config_service import AppModelConfigService
 
 
+@console_ns.route("/apps/<uuid:app_id>/model-config")
 class ModelConfigResource(Resource):
+    @api.doc("update_app_model_config")
+    @api.doc(description="Update application model configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "ModelConfigRequest",
+            {
+                "provider": fields.String(description="Model provider"),
+                "model": fields.String(description="Model name"),
+                "configs": fields.Raw(description="Model configuration parameters"),
+                "opening_statement": fields.String(description="Opening statement"),
+                "suggested_questions": fields.List(fields.String(), description="Suggested questions"),
+                "more_like_this": fields.Raw(description="More like this configuration"),
+                "speech_to_text": fields.Raw(description="Speech to text configuration"),
+                "text_to_speech": fields.Raw(description="Text to speech configuration"),
+                "retrieval_model": fields.Raw(description="Retrieval model configuration"),
+                "tools": fields.List(fields.Raw(), description="Available tools"),
+                "dataset_configs": fields.Raw(description="Dataset configurations"),
+                "agent_mode": fields.Raw(description="Agent mode configuration"),
+            },
+        )
+    )
+    @api.response(200, "Model configuration updated successfully")
+    @api.response(400, "Invalid configuration")
+    @api.response(404, "App not found")
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
     def post(self, app_model):
         """Modify app model config"""
+        if not isinstance(current_user, Account):
+            raise Forbidden()
+
+        if not current_user.has_edit_permission:
+            raise Forbidden()
+
+        assert current_user.current_tenant_id is not None, "The tenant information should be loaded."
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(
             tenant_id=current_user.current_tenant_id,
@@ -39,7 +74,7 @@ class ModelConfigResource(Resource):
         )
         new_app_model_config = new_app_model_config.from_model_config_dict(model_configuration)
 
-        if app_model.mode == AppMode.AGENT_CHAT.value or app_model.is_agent:
+        if app_model.mode == AppMode.AGENT_CHAT or app_model.is_agent:
             # get original app model config
             original_app_model_config = (
                 db.session.query(AppModelConfig).where(AppModelConfig.id == app_model.app_model_config_id).first()
@@ -142,6 +177,3 @@ class ModelConfigResource(Resource):
         app_model_config_was_updated.send(app_model, app_model_config=new_app_model_config)
 
         return {"result": "success"}
-
-
-api.add_resource(ModelConfigResource, "/apps/<uuid:app_id>/model-config")

api/controllers/console/app/ops_trace.py

@@ -1,18 +1,31 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource, fields, reqparse
 from werkzeug.exceptions import BadRequest
 
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
 from controllers.console.wraps import account_initialization_required, setup_required
 from libs.login import login_required
 from services.ops_service import OpsService
 
 
+@console_ns.route("/apps/<uuid:app_id>/trace-config")
 class TraceAppConfigApi(Resource):
     """
     Manage trace app configurations
     """
 
+    @api.doc("get_trace_app_config")
+    @api.doc(description="Get tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "tracing_provider", type=str, required=True, location="args", help="Tracing provider name"
+        )
+    )
+    @api.response(
+        200, "Tracing configuration retrieved successfully", fields.Raw(description="Tracing configuration data")
+    )
+    @api.response(400, "Invalid request parameters")
     @setup_required
     @login_required
     @account_initialization_required
@@ -29,6 +42,22 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("create_trace_app_config")
+    @api.doc(description="Create a new tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "TraceConfigCreateRequest",
+            {
+                "tracing_provider": fields.String(required=True, description="Tracing provider name"),
+                "tracing_config": fields.Raw(required=True, description="Tracing configuration data"),
+            },
+        )
+    )
+    @api.response(
+        201, "Tracing configuration created successfully", fields.Raw(description="Created configuration data")
+    )
+    @api.response(400, "Invalid request parameters or configuration already exists")
     @setup_required
     @login_required
     @account_initialization_required
@@ -51,6 +80,20 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("update_trace_app_config")
+    @api.doc(description="Update an existing tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.model(
+            "TraceConfigUpdateRequest",
+            {
+                "tracing_provider": fields.String(required=True, description="Tracing provider name"),
+                "tracing_config": fields.Raw(required=True, description="Updated tracing configuration data"),
+            },
+        )
+    )
+    @api.response(200, "Tracing configuration updated successfully", fields.Raw(description="Success response"))
+    @api.response(400, "Invalid request parameters or configuration not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -71,6 +114,16 @@ class TraceAppConfigApi(Resource):
         except Exception as e:
             raise BadRequest(str(e))
 
+    @api.doc("delete_trace_app_config")
+    @api.doc(description="Delete an existing tracing configuration for an application")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.expect(
+        api.parser().add_argument(
+            "tracing_provider", type=str, required=True, location="args", help="Tracing provider name"
+        )
+    )
+    @api.response(204, "Tracing configuration deleted successfully")
+    @api.response(400, "Invalid request parameters or configuration not found")
     @setup_required
     @login_required
     @account_initialization_required
@@ -87,6 +140,3 @@ class TraceAppConfigApi(Resource):
             return {"result": "success"}, 204
         except Exception as e:
             raise BadRequest(str(e))
-
-
-api.add_resource(TraceAppConfigApi, "/apps/<uuid:app_id>/trace-config")